diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c00507c96..17387a1a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: PULSAR_ADMIN_PORT: 28080 PULSAR_BROKER_PORT: 26650 DEKAF_PORT: 28090 + # Isolate the Library's data dir per run so items don't accumulate and skew counts. + DEKAF_FRESH_DATA: 1 steps: - uses: actions/checkout@v4 with: diff --git a/.gitignore b/.gitignore index 11fbfeb84..1a304e33e 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,10 @@ result # Docker **/slim.*.json + +# macOS +.DS_Store + +# Local design assets (~24MB of PNG/GIF concepts) - not part of the product build. +# Anchored to the repo root so it does not also swallow any nested `design/` dir elsewhere. +/design/ diff --git a/AGENTS.md b/AGENTS.md index 27acdb557..fdb4dc883 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ Dekaf is an open-source UI for Apache Pulsar. It's a single deployable binary th - A **Scala 3 / ZIO** backend exposing a gRPC API (`server/`) - An embedded **Envoy proxy** that translates browser gRPC-Web ↔ native gRPC -The UI and server communicate over **Protobuf / gRPC-Web**. Proto definitions live in `proto/` and are the source of truth for the API contract — generated code is committed into `ui/grpc-web/` and `server/src/main/scala/pb/`. +The UI and server communicate over **Protobuf / gRPC-Web**. Proto definitions live in `proto/` and are the source of truth for the API contract — generated code lands in `ui/grpc-web/` and `server/src/main/scala/pb/`, which are gitignored and regenerated by `cd proto && make build`. ## Development environment @@ -69,7 +69,7 @@ Dekaf stores saved sessions and other user artifacts as "managed items" on disk ## Conventions & notes - The backend is intentionally **straightforward Scala** — avoid heavy FP / type-level acrobatics (per `CONTRIBUTING.md`). -- After changing any `.proto`, you **must** run `cd proto && make build` and rebuild both sides; the generated code is committed. +- After changing any `.proto`, you **must** run `cd proto && make build` and rebuild both sides. The generated output is gitignored, so a clean checkout has no `pb/` or `grpc-web/` until you run it. - Generated directories (`ui/grpc-web/`, `server/src/main/scala/pb/`) should not be hand-edited. - `demoapp/` is a sample producer app used by the quick-start docker-compose to populate demo data. - `desktop/` contains an Electron wrapper; `helm/` and `deployment/` are for k8s; `docker/` holds image builds and the quick-start compose file. diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index c9c1604a4..aff6f155a 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -76,8 +76,8 @@ Also set the appropriate cookie settings. |Field |Description | |--- |--- | -|cookieSecure | `true` or `false`. Set it to `true` if you use the `https` protocol. | -|cookieSameSite | `true` or `false`. Set it to `true` if you use the `https` protocol. | +|cookieSecure | `true` or `false`. Set it to `true` if you use the `https` protocol. Adds the `Secure` attribute, so the browser only sends the cookie over HTTPS. | +|cookieSameSite | `lax`, `strict` or `none` (case-insensitive). Controls the cookie's `SameSite` attribute, which tells the browser whether to send the cookie on cross-site requests - the built-in CSRF protection. Leave it unset to use the browser default. `none` additionally requires `cookieSecure: true`, because browsers reject `SameSite=None` on a non-`Secure` cookie; if you set it without `cookieSecure`, the attribute is omitted and a warning is logged. An unrecognised value is also omitted with a warning. | ### Default Pulsar Auth diff --git a/docs/consume/consumer-session-tutorial.md b/docs/consume/consumer-session-tutorial.md index 9a685bb8e..1300e2945 100644 --- a/docs/consume/consumer-session-tutorial.md +++ b/docs/consume/consumer-session-tutorial.md @@ -6,7 +6,7 @@ - Navigate to a Pulsar topic you're interested in - Click the **Consume** button -- By default, **Consumer Session** starts from the **Latest Message** in the topic. If no producers produce new messages in realtime to the selected topic, switch the **Start From** field to **Earliest Message** +- By default, **Consumer Session** starts from the **Earliest Message** in the topic, so pressing play shows the data that is already there. On a **non-persistent** topic, which keeps no history at all, it starts from the **Latest Message** instead and the history-based positions are unavailable. - Click the **▶** button to start the consumer session. ![initial consumer session](./img/start-consumer-session.png) @@ -67,7 +67,7 @@ All the loaded messages data will be erased on the UI. If you want to save the c The **Search in Loaded Messages** feature that we used above, may be not enough for precise search and search in Pulsar topics with a lot of messages for the following reasons: - In case you're searching for a small subset of messages in comparison with all messages in the topic, the network bandwidth may easily became a bottleneck. -- The amount of messages loaded to your browser simultaneously is limited by RAM available for a single browser tab. By default, the message count limit is `10,000` messages, but you can adjust it in consumer session's **advanced settings** depending on the average message size. +- The amount of messages loaded to your browser simultaneously is limited by RAM available for a single browser tab. By default, the message count limit is `10,000` messages, but you can adjust it with **Limit num. display messages** in the consumer session settings depending on the average message size. The **Message Filter** feature solve both mentioned problems. They work on server-side, therefore processed messages that don't pass message filters, aren't being loaded to your browser. @@ -226,7 +226,7 @@ Each consumed message represents a row in the table. Table has the following columns: -- **#**: Index number of the message in the current view. It is determined by the order in which the message was consumed and delivered to the UI. The order of messages may differ from the **Publish Time** order in case of consuming a partitioning topic or multiple non-partitioned topics. +- **#**: Index number of the message in the current view. It is determined by the order in which the message was consumed and delivered to the UI. Across a partitioned topic or several topics that depends on the session's delivery order: **Guaranteed** and **Best effort** (the default) merge by the selected timestamp - Guaranteed adds no disorder of its own, while under **Best effort** a message arriving after the reorder window is numbered out of order. **Fastest** delivers each topic and partition independently, so its numbering follows arrival rather than **Publish Time**. - **Publish Time**: The timestamp of when the message is published. The timestamp is automatically applied by the producer. - **Key**: The key of the message. Messages are optionally tagged with keys, which is useful for features like topic compaction and key-shared subscriptions. - **Value**: Value of the message serialized as JSON. It may be inefficient to observe values this way because some values may not fit the column width. To fix that, you can map specific value field to a table column by using **Projections**. diff --git a/docs/consume/index.md b/docs/consume/index.md index ae9cfe004..5d940b4b7 100644 --- a/docs/consume/index.md +++ b/docs/consume/index.md @@ -6,6 +6,22 @@ Dekaf allows you to explore Apache Pulsar topic data by using **consumer session - Browse live stream data - Consume multiple topics at once +- Choose how Dekaf combines multiple topics or partitions. Pulsar itself does not guarantee order + across topics. + - **Guaranteed** (the default) is an exact replay: it delivers everything recorded up to Play (of what + retention still holds) in strict selected-timestamp order, then auto-pauses with a caught-up + banner, whose "Load new messages up to now" extends the replay to the present; an ordering violation across a pause seam + (producer clock skew) is delivered loudly flagged - a row marker plus a session counter - + never silently. Multi-stream sessions require persistent topics. + - **Best effort** merges by the selected timestamp within ~0.75 s. Late messages can + appear out of order; none are dropped. + - **Fastest** delivers each topic or partition independently, with no reordering delay. + - See the [comparison of start positions and modes](./modes-comparison.md) for how every + start-from mode, delivery order and consumption mode combine. +- **Order by** uses Pulsar timestamps: publish time is added automatically by the producer, broker + publish time is broker entry metadata recorded when a message arrives, and event time is an + optional timestamp set by the application. Missing broker publish time or event time falls back + to publish time and is reported. - Filter messages using user-friendly basic, or advanced JavaScript filters - Search for specific value in loaded messages - Map specific message field to the search results table column @@ -20,8 +36,33 @@ Dekaf allows you to explore Apache Pulsar topic data by using **consumer session The amount of messages that can be processed during the consumer session is unlimited, but the amount of messages that are loaded and displayed in the user-faced UI at once has limits. -By default this limit equals to `10,000` messages. Depending on the average message size, you can configure this number for specific topic. +By default this limit equals to `1,000,000` messages. Depending on the average message size in the consumed topics, you can lower it per consumer session - the help beside the toggle says when to. Operations that user can perform on the loaded data (e.g. sorting, export, or search in found) are limited to these messages. We're looking for the best way to get rid of this limitation. + +### Pausing loses nothing only on persistent topics + +Pausing closes the session's intake and hands back the messages it had already prefetched, so on a +persistent topic the broker redelivers them and the session picks up where it stopped. A +non-persistent topic stores nothing, so there is nothing for the broker to redeliver: whatever is +published while the session is paused - including the messages it had already prefetched and handed +back - is gone, and resuming shows only what is published from then on. A paused session says so on +screen whenever it reads such topics. + +### A session affects retention while it exists + +Dekaf browses through its own non-durable subscription, so the broker holds a live cursor at the +position the session has read up to, and a topic's data cannot be reclaimed ahead of that cursor. +An open session therefore delays ledger deletion and storage reclamation on the topics it reads +whenever its cursor sits behind data the broker would otherwise be free to drop - and it keeps +doing so for as long as the session stays open. The window is longest where you would expect: a +session that is paused - including one auto-paused at its **Guaranteed** replay boundary - or that +started from an old position holds its cursor where it is until it moves on. + +Other consumers are not affected: every other subscription keeps its own cursor and backlog, and no +topic or subscription policy is changed. Stopping or deleting the session removes its subscription, +and the cursor with it. A session nobody is watching any more - a closed tab whose cleanup never +reached the server - is stopped automatically after 1 hour without a live stream, which is the +longest an abandoned session can hold a cursor. diff --git a/docs/consume/modes-comparison.md b/docs/consume/modes-comparison.md new file mode 100644 index 000000000..69f62d502 --- /dev/null +++ b/docs/consume/modes-comparison.md @@ -0,0 +1,72 @@ +# Start positions and modes - comparison + +A consumer session has three independent choices: + +1. **Start from** - where in the topic the session begins. +2. **Delivery order** - how messages from several topics or partitions are combined into one stream. +3. **Consumption mode** (per target) - whether the session reads the raw log or the compacted view. + +This page compares the options and what they are compatible with. + +## Start from + +| Mode | What you get | Exact? | Speed | Notes | +|---|---|---|---|---| +| Earliest message | Everything the topic still retains | Exact | Instant | | +| Latest message | Only messages published after Play | Exact | Instant | The only mode available on non-persistent topics | +| Skip first n messages | Everything except the globally-oldest n (by publish time, across all partitions) | Exact - counts real messages, not storage entries | Proportional to n; a progress bar appears for very large skips | For jumping deep into a huge topic, a time-based or percentage mode is instant instead | +| Latest n messages | Exactly the globally-newest n (by publish time, across all partitions) | Exact | Fast at any topic size (walks only the tail) | | +| Message with specific ID | From one exact message | Exact | Instant | | +| Specific time | From a moment in time | To the millisecond | Instant at any topic size | | +| Relative time ago | Same, phrased as "2 hours ago" | To the millisecond | Instant | Optional rounding to the unit start | +| Approximate position (% of data) | Roughly this far through the stored data | Approximate - counted in storage entries, so batching skews the message percentage | Instant at any topic size - it asks the broker a fixed number of questions however much the topic holds | Each topic and partition is positioned independently | +| Approximate position (% of time) | Roughly this far between the oldest and newest publish times | Approximate - producer clocks set the timestamps | Instant at any topic size | One shared cutoff instant for the whole session, so multi-topic sessions align | + +## Compatibility: start from x what you are reading + +| Start from | Non-persistent topic | Read-compacted target | Topic with chunked messages | Partitioned topic | +|---|---|---|---|---| +| Earliest message | disabled - nothing is retained | yes | yes | yes | +| Latest message | **yes - the only mode** | yes | yes | yes | +| Skip first n | disabled | yes - counts what is actually delivered | yes - a chunked message counts as one message | yes - the skipped n are global, not per partition | +| Latest n | disabled | **refused** - it counts the raw log, which compaction rewrites | **refused** - storage entries are not messages there; the error names the alternatives | yes - the n are global, not n per partition | +| Message ID | disabled | yes | yes | yes | +| Specific / Relative time | disabled | yes | yes | yes | +| % of data | disabled | **refused** - it measures the raw log | yes, but chunk entries skew the percentage | yes - each partition positioned independently | +| % of time | disabled | **refused** - its range comes from the raw log | yes | yes - one shared cutoff across all partitions | + +A refused combination fails with a message naming the reason and a working alternative - it never +silently returns a different result than asked for. + +## Delivery order (when a session merges several topics or partitions) + +| | Fastest | Best effort | Guaranteed (the default) | +|---|---|---|---| +| Order claim | None - messages appear as they arrive | Sorted by the selected timestamp, waiting up to ~0.75 s for stragglers | An exact replay of everything recorded up to Play (of what retention still holds), strictly sorted; it auto-pauses when caught up, and "Load new messages up to now" extends the replay to the present | +| A partition goes silent | No effect | After the grace, delivery continues; a straggler arriving later is shown late and counted | Silence itself does not matter - the replay delivers each stream's recorded range and auto-pauses at the boundary; a silent stream holds delivery only while part of its recorded range is still undelivered | +| A silent partition stays silent | No effect | Delivery simply continues without it; only while a counted start position (Skip first n) is still resolving does a ~30 s give-up set the stream aside and mark the start position approximate | If a stream's recorded range can no longer be delivered - trimmed away by retention, or the start position seeked past its end - no caught-up fires: the session discloses the wait and offers a one-click switch to Best effort. Otherwise the caught-up pause stands, and "Load new messages up to now" replays whatever was recorded since | +| Late/out-of-order messages | Shown as they come | Delivered and counted, never dropped | Cannot occur within a replay chunk; across a pause seam a producer-clock reversal is delivered flagged (row marker + counter), never silently | +| Non-persistent topics | yes | yes | Single stream only - nothing is retained, so Play announces caught-up instantly; merging non-persistent topics is refused | +| Best for | Raw throughput, single topics | Everyday browsing and live following | Forensic reading of recorded history where order matters more than immediacy | + +**Order by** applies to Best effort and Guaranteed: publish time (producer clock), broker publish +time (broker clock, entry metadata), or event time (application-set). A message missing the chosen +timestamp falls back to publish time, and the session reports how often that happened. + +**Latest message x Guaranteed** replays nothing - a replay of recorded history starting at "now" +has nothing behind it. The combination is selectable; Play simply answers caught-up immediately, +and the caught-up panel offers the two ways on (load what has arrived since, or switch to Best +effort and follow live). + +## Consumption mode (per target) + +| | Regular | Read compacted | +|---|---|---| +| What you read | The raw log - every retained message | The newest message per key up to the compaction horizon, then the raw tail | +| Use it for | Everything by default | Key-value style topics where only the latest value per key matters | +| Start-from limits | None | Latest n and both percentage modes are refused (they measure the raw log); Earliest, Latest, Skip first n, Message ID and the time modes work | + +## See also + +- [Consumer sessions overview](./index.md) +- [Consumer session tutorial](./consumer-session-tutorial.md) diff --git a/e2e/README.md b/e2e/README.md index 342126e31..ec97a83b9 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -62,19 +62,59 @@ sbt "testOnly features.consumersession.*" # one feature area sbt "testOnly features.library.*" # the Library feature sbt "testOnly routes.NavigationTreeSpec" # one spec sbt "testOnly *CsFiltersSpec -- -z CS-10" # a single test by name substring (-z) + +sbt "testOnly configuration.*" # the DEKAF_* configuration lane (§6) ``` -### The `KnownBug` mechanism (lane currently EMPTY - all bugs fixed 2026-07-19) +The `configuration.*` specs are the one group that does **not** use the shared `:8090` stack: each +starts its own Dekaf with the `DEKAF_*` overrides it is about (see §6). They need Pulsar and the +staged server build, not `run-dekaf.sh`. + +### The `KnownBug` mechanism (lane currently EMPTY) While an app bug is open, its regression test asserts the **correct** behavior, is tagged `KnownBug`, and stays **red on purpose** - excluded from `sbt test` (`Test / test / testOptions` in `build.sbt`) so the normal run is green. When the bug is fixed, the test is **untagged** and joins the green lane -as an ordinary regression. All 19 catalogued bugs were fixed on 2026-07-19, so the -`knownbugs/*Spec` tests now run green in the normal lane (6 remain `ignore`d - fixed app-side but not -driveable from this harness; rationale inline, and see §6). The tag + exclusion stay wired for the next bug: +as an ordinary regression. All 19 originally catalogued bugs were fixed on 2026-07-19, so the +`knownbugs/*Spec` tests now run green in the normal lane. (`CsTopicKindsSpec` CS-TK-6 was untagged +on 2026-07-25 when both counting Start-From modes became GLOBAL - §6, "Start-From: batching, and +the entry-vs-message trap".) + +The lane was last used on 2026-08-09, during the Guaranteed exact-replay alignment, and emptied +the same day: `CsDeliveryModesSpec` **CS-DM-R3B** caught the first-play boundary re-capture +clobbering a live-edge (Latest) boundary - fixed server-side (the first Play now consumes the +create-time boundaries) and untagged; `CsFlowControlSpec` **CS-FC-5**'s intermittent red turned +out to be an oracle defect in the TEST (the broker's dispatch counter counts dispatches and +resets with the consumer on a transient reconnect, so a redelivered held window read as +over-admission) - the oracle is now connection-epoch-bracketed, the server's admission +serialization got its own unit pins, and the cell is untagged. No test carries the tag today, so +the bug lane runs **0 tests** - that is the healthy state, not a broken filter. The named census +in `SuiteFactsSpec` FACTS-2 and the marker below must move together with any future tag. + +Nothing in this suite is `ignore`d or `pending` either. The bugs that are fixed app-side but not +driveable from Playwright have no e2e test *at all* - they are covered in the jest / server tiers +instead (§6), with an inline pointer where they would have lived. (This paragraph used to claim six +`ignore`d tests; `rg '\bignore\s*\(' e2e/src/test` has found none for some time.) + +Two topic-policy specs are the one exception, and an honest one: `TopicPolicySpec` (TOP-8/9) and +`TopicPolicyBreadthSpec` (TOP-13/14) `assume(...)` on the broker's `topicLevelPoliciesEnabled` +setting, so on any given stack the branch that does not match the broker RUNTIME-cancels - a canceled +test is neither a pass nor a failure. On the dev stack (policies ON) TOP-8 always cancels. That is +config-gated coverage rather than a hidden red test, so it is named here and pinned: the census below +counts every `assume(...)` and fails if one appears OUTSIDE those two specs, which is the one way the +same mechanism could quietly drop a test from the green run. + +All of these are pinned by `harness.SuiteFactsSpec` against the marker below - the `ignored`, +`pending` and `known-bug` counts, the `assume` count and its location, and that build.sbt excludes +exactly the one `KnownBug` tag from `sbt test` (`excluded-tags`) - so none of the claims can drift +again. + + + +The tag and its exclusion stay wired for the next bug: ```bash -sbt "testOnly * -- -n KnownBug" # the bug lane - currently runs nothing (no open bugs) +sbt "testOnly * -- -n KnownBug" # the bug lane - red by design; currently empty (0 tests) sbt test # green lane, includes the fixed-bug regressions ``` @@ -166,16 +206,22 @@ e2e/src ├── main/scala │ ├── harness/ Config, PwRuntime (shared Playwright+Browser), PulsarFixtures (admin oracle + │ │ producer + per-test teardown), Eventually (poll the oracle, no fixed sleeps), +│ │ DekafInstance (a short-lived Dekaf with DEKAF_* overrides - §6), │ │ Cli (ShowTrace / Codegen mains) │ ├── ui/ reusable component objects (ConfirmationDialog, SubscriptionOverviewPage, …) │ └── features/ component objects for the priority features │ ├── library/ LibrarySidebar, LibrarySaveDialog, LibraryBrowser -│ └── consumersession/ ConsumerSessionPage, FilterPanel, TargetSelector, ExportModal, ToolsPanel +│ └── consumersession/ ConsumerSessionPage, FilterPanel, TargetSelector, ExportModal, ToolsPanel, +│ DeliveredMessages (the exact-set oracle - §6) └── test/scala - ├── harness/ DekafSuite (base trait: fresh BrowserContext + trace + fixtures per test) + ├── harness/ DekafSuite (base trait: fresh BrowserContext + trace + fixtures per test), + │ BatchingFixtureSpec (the broker facts under Start-From), StackScriptsSpec + │ (scripts/), SuiteFactsSpec (this README's countable claims), + │ DeliveredMessagesSpec (the exact-set oracle really does beat a count - §6) ├── smoke/ routes/ instance/ navigation, chrome, and per-page specs ├── primitives/ cross-cutting form primitives (X-1/2/3) ├── features/ library/ + consumersession/ + producer/ specs + ├── configuration/ the DEKAF_* properties, each on its own short-lived Dekaf (§6) └── knownbugs/ the KnownBug-tagged regressions (§6) ``` @@ -188,21 +234,183 @@ e2e/src Honest backlog - none block the green lane; each is a place the suite proves less than it might. +**Running the SERVER suite needs `bin//` on PATH.** Not an e2e matter, but it bites here +first: four `schema.protobufnative` tests and one `convertersTest` case shell out to `protoc.bin`, +which lives in the repo's per-arch `bin` dir. `run-dekaf.sh` exports it; a bare +`cd server && sbt test` does not, and the five fail with `Cannot run program "protoc.bin"` - which +reads exactly like a regression and is not one. Use +`export PATH="$PATH:$(node ../bin/get-bin-dir.js)"`. + +**A table's `autoRefresh.intervalMs` below 2000ms is silently floored** (found 2026-08-09 while +root-causing CS-TP-5). SWR's default `dedupingInterval` is 2000ms, so a Table asking to poll every +1000ms - Topic Positions does - actually polls every ~2000ms. Measured, not inferred: request +counts sampled at 500ms landed the first refresh at ~2.5-3s and then every 2s. CS-TP-5 therefore +asserts the contract in its title (a visible tab keeps polling) with a tolerant window instead of +a fixed one; its old 2200ms wait expired just before the first refresh and failed deterministically. +Making the prop honest would double a deliberately broker-heavy panel's poll rate, so it is +recorded as an owner decision rather than changed. + +**A toast dismissed and re-announced is briefly TWO elements - or silently NONE** (found 2026-08-12 +root-causing CS-DM-R2 and R3B). react-toastify's removal is animation-mediated: a dismissed toast +stays in the DOM until its exit animation ends, and a toast created under a still-exiting id is +silently dropped. So a panel that is dismissed and re-announced moments later - the caught-up +banner across a "Load new messages" hand-off - loses both ways: one fixed id and the re-announce +is swallowed (a 20s timeout that reads like the banner never appeared - R2); fresh ids per episode +and the retiring panel coexists with its successor, which every strict locator rejects in +milliseconds with "resolved to 2 elements" (R3B - an instant catch-up re-announces faster than +ANY exit animation, so no exit duration is short enough). The fix was structural: the caught-up +panel is not a toast at all - the session component renders it conditionally +(`ConsumerSession.tsx`, `ReplayCaughtUpDock`), so at most one exists by construction. If a +notification-like panel ever needs dismiss-then-re-announce again, render it from the owning +component; do not put it in the toast layer, and do not `.first()`/`.last()` around the ambiguity. + **Infrastructure** -- **Playwright browsers aren't pinned/cached** - a clean CI runner downloads Chromium on first use - (the CI `e2e` job installs it each run). -- **Library state accumulates.** `run-dekaf.sh` doesn't set `DEKAF_DATA_DIR`, so items land in - `server/data/library` and survive runs. Tests are context-scoped so they still pass, but - instance-scoped items (e.g. LIB-16's note) leak and counts drift. Isolating the data dir also needs - `js/dist/libs.js` + `proto/` seeded into it (see `run-dekaf.sh`). - **No independent Library oracle.** LIB CRUD arranges *and* verifies through the same UI path, so a shared serialization/render defect could pass both. A generated `LibraryService` gRPC stub would fix - this (and let the `ignore`d BUG-8/9/17 regressions drive the server directly). + this - the one remaining piece of test infrastructure worth building. (One partial exception since + 2026-08-08: CFG-DD-1 reads the item's bytes back off disk, because it starts an instance whose + `dataDir` the test itself owns. That is an oracle outside the UI, but only for "an item with this + name was persisted here" - not for its contents.) + +*Resolved:* Playwright browsers are no longer downloaded at all - they come pre-patched from the nix +store, version-locked to the Java client via the `nixpkgs-playwright` flake input, so a clean runner +needs neither a download nor sudo. And `run-dekaf.sh` now honours **`DEKAF_FRESH_DATA=1`**: it seeds a +throwaway data dir with `js/` + `proto/` and points `DEKAF_DATA_DIR` at it, which CI sets - so Library +items no longer accumulate across runs. Local dev keeps its persistent library by default. + +That throwaway tree used to be a `mktemp -d`, and **leaked on every CI build**: `run-dekaf.sh` ends in +`exec sbt run`, so the process that created it is replaced by the server, and CI then kills the whole +process tree - no trap or shutdown hook in the server's own lifetime can fire, and the random name +left nothing findable afterwards. The path now comes from **`scripts/fresh-data-dir.sh`** (one +deterministic directory **per stack** under `$RUNNER_TEMP`, falling back to `$TMPDIR`): +`run-dekaf.sh` clears the previous run's tree before seeding, and **`stack-down.sh`** - the teardown +step CI already runs with `if: always()` - removes it. + +*Per stack* is the second half of the fix and arrived after the first: a single deterministic name is +shared by every stack on the box, and since startup `rm -rf`s it and teardown `rm -rf`s it again, two +stacks side by side - a second Dekaf on its own port, which is exactly how you run two - destroyed +each other's **live** data dir. The identity is derived rather than configured, from the two +variables that already distinguish the stacks and are already in both scripts' environment: +`DEKAF_PORT` and `PULSAR_CONTAINER_NAME`. Teardown must therefore be run with the same values the +stack came up with (CI sets `DEKAF_PORT` at job level, so every step of the job agrees); getting it +wrong now leaks a tree instead of destroying a live one, and `clean` prints any tree it deliberately +left behind so that leak is visible. `harness.StackScriptsSpec` (STACK-1..4) pins the determinism, +the removal, that two identities cannot collide - cleaning one leaves the other's seeded data byte +for byte - and that both scripts still go through the shared path. + +Two more fixes that had no permanent regression now have one, in both cases because the failing +condition does not occur on a healthy local stack and had to be **created**: + +- **LIB-22** delays `ListLibraryItems` from inside the page (a wrapped `XMLHttpRequest.send`, not a + Playwright route handler - a Java handler that sleeps blocks the driver's own dispatch loop and + would stall the test's `isVisible` call too, hiding the race). It asserts the Notes panel really is + unsettled and that `LibrarySidebar.createNote` still works. Reverting `createNote` to its + pre-fix `isVisible` branch makes it time out on `lib-new-note`, which is the original symptom. +- **CS-33** blocks `cdn.jsdelivr.net` outright and asserts a Monaco editor still mounts, that the + files came from `/ui/static/dist/vs/`, and that nothing reached for the CDN. Blocking rather than + merely observing is the point: on a runner with internet, a regression would silently succeed + through the CDN and prove nothing about the offline case. *Still unproven:* the loader path is + built against `document.baseURI` so that it survives a non-root `DEKAF_PUBLIC_BASE_URL` / + `basePath`, and this stack serves Dekaf at the root - where the base-relative and origin-rooted + forms are the same string. The second Dekaf on a sub-path that this needs now EXISTS + (`harness.DekafInstance`; CFG-BP-1 asserts every asset the app requests resolves under the + sub-path and nothing 404s), but no test mounts a Monaco editor on such an instance, so the AMD + loader path specifically is still uncovered. + +**Configuration properties (`DEKAF_*`)** + +Until 2026-08-08 no test in this suite set any `DEKAF_*` variable beyond the three connection +basics, so every path that turns an operator's SETTING into behaviour was unexercised end to end. +That gap was worse than a coverage count suggests: the two cookie-hardening bugs that shipped - +`Secure` computed into a local and never interpolated into the header, and `cookieSameSite` matched +against lowercase literals only, so a capitalised `Lax` emitted NO attribute at all - were both +config-to-behaviour WIRING failures, and both were invisible to every tier because each test called +the cookie writer with hand-passed parameters, bypassing the plumbing entirely. + +`harness.DekafInstance` is the affordance that closes it: it starts a short-lived Dekaf with +arbitrary `DEKAF_*` overrides on a **probed** free port (8700-8799, deliberately clear of `:8080` +and of the shared stack's `:8090`) with a seeded `DEKAF_DATA_DIR` of its own, polls `/health` +through the embedded Envoy for real readiness, and always tears the whole thing down - the JVM, its +Envoy child, the port, and the data dir - including on failure. Instance logs are kept under +`target/config-instances/`; they are the only diagnosis when a server refuses to boot. Specs live in +`configuration/` and navigate ABSOLUTE URLs, because `DekafSuite`'s BrowserContext baseURL still +points at the shared stack. Two harness traps found while building it, both now fixed in +`DekafInstance` and worth knowing about: + +- probing a free port by BINDING it is not enough - Java's `ServerSocket` sets `SO_REUSEADDR`, so the + wildcard bind succeeds on macOS while another process holds the same port on `127.0.0.1`. Envoy + then binds the wildcard too, loopback traffic goes to the *other* program, and the readiness poll + gets a 401 from something that is not Dekaf. It now connects first and only trusts a refusal. +- the JDK `HttpClient` defaults to HTTP/2, i.e. an `Upgrade: h2c` probe on the first request to each + new origin - and every instance here is a new origin. Jetty answers that `400 Invalid Upgrade + header`, which the readiness poll cannot tell from "not ready yet". It is pinned to HTTP/1.1. + +Covered, every assertion through the browser and across the Envoy proxy: + +- **`cookieSameSite` x `cookieSecure`** - `CfgCookieSpec` CFG-COOKIE-1..4 add a credential through + the real UI and assert the intercepted `Set-Cookie`'s attribute list EXACTLY (sorted, so order is + free but a missing `Secure` or a smuggled `Domain=` cannot hide). CFG-COOKIE-1 configures + `Lax` in mixed case on purpose - the exact spelling that shipped broken - and additionally asserts + the browser STORED the cookie hardened; -2 covers `strict` with `cookieSecure` unset; -3 covers + `NONE` with `Secure`; -4 pins the guard rail that `none` WITHOUT `Secure` emits no `SameSite` at + all, because a browser would reject the whole cookie and break authentication. +- **`basePath`** - `CfgBasePathSpec` CFG-BP-1 serves the app at `/dekaf-e2e` and asserts it WORKS + there: a deep link loads, breadcrumbs render from a gRPC-web round trip, in-app navigation stays + under the sub-path (the router's basename), every `/api/` call resolved under it, static assets + were served from it with nothing 4xx, and the cookie `Path` is the sub-path - plus that the origin + ROOT serves nothing, which is what separates "the app moved" from "the app answers everywhere". + This is also the only coverage the embedded Envoy's routing has ever had. +- **`publicBaseUrl`, isolated from `basePath`** - `CfgPublicBaseUrlSpec` CFG-PBU-1 uses the shape + where the two genuinely differ, a reverse proxy that STRIPS the prefix: Dekaf served at the + internal root, `publicBaseUrl` carrying the public mount point. It asserts `document.baseURI`, + that assets were really REQUESTED under the public prefix, and the cookie `Path`. +- **`pulsarName` / `pulsarColor`** - `CfgInstanceIdentitySpec` CFG-ID-1/2. The name must render in + the navigation tree root and the instance-overview row; the colour must reach the DOM as the + accent element's COMPUTED `box-shadow`. +- **`dataDir`** - `CfgDataDirSpec` CFG-DD-1 saves a Library item on one data dir and asserts a second + instance on another cannot see it, a THIRD instance on the first one can (so the absence is + isolation rather than a lookup that never finds anything), and the item's bytes are on disk under + the configured directory - an oracle independent of the UI that wrote it, which the Library + otherwise has none of. +- **`defaultPulsarAuth`** - `CfgDefaultAuthSpec` CFG-AUTH-1 configures a JWT-shaped credential whose + token is the literal words `e2e.dummy.token` (no secret, real or fake, belongs in this repo) and + asserts it is the Default credential's TYPE in the manager, is CURRENT, is in the `pulsar_auth` + cookie, and actually builds working Pulsar clients. `empty` would have been benign too, but it is + also the built-in default, so it could not tell a working configuration from an ignored one. + +**Not covered, and why:** + +- **The ~15 `pulsarTls*` properties, plus `protocol` / `tlsCertificateFilePath` / `tlsKeyFilePath`** + need a TLS-enabled Pulsar and a certificate fixture. `dekaf-e2e-pulsar` is plaintext and this + harness ships no certs; `protocol: https` additionally makes Envoy terminate TLS, so the browser + side would need its own trust setup. No tier covers their EFFECT - `server`'s `mergeConfigsTest` + only proves the values survive config merging. +- **`bindAddress`, `port`, `internalHttpPort`, `internalGrpcPort`** are infrastructure with no + distinct browser-observable behaviour. `port` is in fact exercised on every instance + `DekafInstance` starts (each one is a different port), just never asserted as itself. +- **`pulsarWebUrl` / `pulsarBrokerUrl`** are exercised implicitly by the entire suite - every test + reads and writes Pulsar through them - and again by every config instance. +- **`pulsarListenerName`** needs advertised listeners configured on the broker. The standalone + container declares none, so setting it can only produce a connection failure, never a behaviour. +- **Envoy's other settings** (CORS, `max_request_headers_kb`, the gRPC stream timeouts) have no + coverage; CFG-BP-1 exercises its ROUTING only. +- **Staleness of the staged build - a real caveat, not a nicety.** These specs run + `server/target/universal/stage/bin/dekaf` rather than `sbt run`, so that the test holds the + server's own process handle (sbt forks the app into a grandchild, and killing sbt can leave it + alive holding the port). That build is staged only when MISSING, or on + `DEKAF_SERVER_STAGE=force`: an `sbt stage` launched from inside a running e2e run shares this + run's `~/.ivy2` locks and intermittently exits 1 with no diagnostic at all, which made staging on + every run flakier than the staleness it prevents. **After editing `server/`, re-run these specs + with `DEKAF_SERVER_STAGE=force`** (or `cd server && sbt stage` first), or they test the previous + build. If staging fails three times and a build already exists, the harness falls back to it and + says so loudly on stderr. **Assertions thinner than the feature they name** (acknowledged, not defects) -- **CS-16/17** don't assert the full counter/state machine incl. broker-side consumer presence; - **CS-23** asserts the details panel opens, not its tab contents; **CS-28** asserts formats + `.zip` - entry indices, not exact exported values. +- **CS-23** asserts the details panel opens, not its tab contents. (CS-16/17 used to sit here for + broker-side consumer presence; the lifecycle specs now assert broker consumer counts on stop.) (**CS-28** used to sit here too; + it now parses the exported `.zip` and compares exact `(index, key, value, topic)` records in order, + so only the fields the broker owns - message id, publish/event time, size, producer name - are + left unpinned.) - **TOP-3/4, SUB-3** assert success toasts rather than a polled state change; **RES-2** supplies a missing id, not a malformed persisted config; **LIB-18** proves the `?id=` URL loads, not that the exact saved config restored. @@ -212,6 +420,292 @@ Honest backlog - none block the green lane; each is a place the suite proves les split + clear-backlog but not unload/unload-all; TOP-7 doesn't verify Earliest/Latest cursor semantics; Producer properties/event-time have testIds but no dedicated test yet. +**Counting is not identity - the high-pressure delivery specs** (strengthened 2026-08-08) + +`CsPauseLoopSpec` CS-PL-1 and `CsFlowControlSpec` CS-FC-1 used to conclude "zero loss" from the +final message COUNT. A count cannot: one message lost anywhere plus one delivered twice anywhere +reaches exactly the same number, and 1,005 *wrong* messages satisfy "1,005 survived the cut" as +readily as the right ones. `cs-loaded` is not even read from the delivered rows - it is the server's +own `numMessageSent` - so it cannot see a message counted but never shown, or shown but never +counted. Both specs now produce **unique payloads** (`bk-*`/`burst-*`; `a-*`/`b-*` per topic), take +their expectation from a **broker read-back**, and compare the exact MULTISET through +`features.consumersession.DeliveredMessages`. `harness.DeliveredMessagesSpec` (SET-1..4) is the +permanent record that the comparison catches what the count does not: it runs both oracles over one +loss plus one duplicate and pins the count accepting it. That spec is pure - no browser, no broker. + +Three things make this work, and each is a trap if you write another one: + +- **Read the set through the app's EXPORT** (`ConsumerSessionPage.exportedValues`), not the table. + `allColumnValues` walks the virtualized list and gives up after 300 viewport steps - a few + thousand rows on this 1280x800 context - so it cannot reach a 55,000-message set at all. +- **The session must be PAUSED to export.** While running, the app renders and exports only its last + 250 messages, so a running export silently compares a tail; `exportedValues` refuses to run. +- **Display retention is part of the arrangement.** A session keeps 10,000 messages on screen by + default and drops its oldest past that, so beyond 10,000 the delivered set is *unobservable* and + only counts remain. CS-PL-1 raises the session's own `numDisplayItems` limit above its total + (`cs.setNumDisplayItems`) and then asserts `data-cs-retained` equals it, so a limit that did not + take effect fails as itself rather than as a mysteriously missing 45,000. + +Counts are KEPT alongside the sets, on purpose, and two places say so inline so a later reader does +not "fix" them away: they come from a different source (the server's counter) and are read after a +quiet window, which catches a straggling extra delivery arriving *after* the set first matched. +CS-FC-5's plateau bounds and CS-FC-2's `loaded == 0` stay counts outright - a byte watermark is a +statement about how many bytes were admitted, and zero already is an exact set. + +**The delivery-order x liveness matrix - the exact-replay contract, cell by cell** (`CsDeliveryModesSpec`, aligned 2026-08-09) + +The three delivery-order modes crossed with every way a topic can be alive, one clearly-labelled +cell per combination, and one CARDINAL RULE: every cell pins its mode EXPLICITLY through the +`cs-delivery-order` control - no cell depends on the session default, so the matrix keeps meaning +the same thing whichever mode the owner makes the default. (`CsMergeOrderSpec` CS-MO-0 is the one +test that pins what the default IS - Best effort, per the owner decision of 2026-08-09, which +supersedes the 2026-08-08 decision that said Guaranteed.) + +**Guaranteed is an EXACT REPLAY** (owner decision 2026-08-09, landed with server + UI on this +branch): Play captures every stream's recorded end; the session delivers everything recorded up +to that boundary in strict key order, then AUTO-PAUSES on the ordinary paused state with the +caught-up banner (`cs-replay-caught-up`). A message published after Play belongs to the NEXT +chunk - handed back un-acked, its consumer held - and an explicit "Load new messages up to now" extends the boundary to +now and replays the delta exactly. "Continue live with Best effort" (on the banner) is the +designed transition out of the replay into live following. The hold-forever behavior these cells +used to pin - and the `PINS THE CURRENT PRODUCT DECISION` markers that guarded it - are gone; +every old hold expectation was first CONFIRMED failing against the redesigned stack (full sets +delivered, sessions auto-paused) before its cell was re-aimed. + +The liveness states, and what each Guaranteed cell now proves: + +- **A ALL LIVE** (CS-DM-AF/AB/AG) - two topics, both receiving during the session: two logical + topics give the test an independent live producer per stream. Fastest delivers everything exactly + once (no order asserted - Fastest is the absence of the merge); Best effort delivers in global + publish-time order; Guaranteed turns live traffic into CHUNKS - the recorded range replays and + pauses, post-Play traffic stays settled at zero until "Load new messages" replays it as the next chunk, + exactly once, order intact across both chunks. +- **B ALL STATIC** (CS-DM-BF/BB/BG) - one 3-partition topic, backlog on every partition: the + one-click "browse this topic" shape. Fastest and Best effort complete the backlog (the ~0.75s + reorder grace is what releases Best effort's tails); Guaranteed replays the WHOLE backlog - a + drained partition holds nothing back - then auto-pauses, with no stall chip, no newer-entries + line and no excluded-topics line (a zero is silence; R1 pins the positive halves). +- **C MIXED** (CS-DM-CF/CB/CG) - two topics, one live and one stopped before Play. Under + Guaranteed the stopped topic holds NOTHING back: the recorded range replays whole (including + what the old contract held behind the silent stream), the live tail waits past the boundary, + and "Load new messages" delivers the accumulated delta exactly once. +- **D EDGES** - empty-from-birth partitions (D1: backlog on one partition of three, the other two + never spoke - an empty recorded range is trivially FINISHED, so the backlog replays in full, + promptly, with no waiting chip; pause-window words on all three partitions then arrive together + as the next chunk); static-becomes-live (D2: the late word to a finished topic is recorded PAST + the boundary - banner first, then "Load new messages" delivers it, order intact); a single non-partitioned + topic (D3: one stream needs no MERGE - Fastest/Best effort stay chip-free - but the replay + boundary is not a merge property: Guaranteed builds its layer at ANY stream count, so the chip + announces the replay and the static single topic pauses caught-up like any other shape); + Guaranteed x non-persistent multi-stream (D4: refused at create, by name, with the remediation - + the session delivers nothing; the SINGLE non-persistent stream is CS-MO-0B's instant caught-up); + the one-click switch (D5, below); and ownership churn (D6, below). +- **D5 THE SWITCH** (CS-DM-D5G) - from the caught-up pause, the banner's "Continue live with Best + effort" (the only broker-level exercise of the SetDeliveryOrder RPC) releases the boundary and + the session follows live traffic: the pause-window words flow, a fresh round arrives with no + loading new messages, everything exactly once across the transition, and EXACTLY ONE divider badge + (`cs-order-switch-point`, "Best effort from here") sits on the first row delivered after the + switch - rows above are the exact replay, rows below the bounded-reorder live following. The + switch also writes Best effort back into the config (pinned at the jest tier). +- **D6 CHURN** (CS-DM-D6G/D6B/D6S) - `admin.topics().unload` on ONE stream's topic mid-session: + the broker drops and re-owns the topic, the session's consumer silently reconnects, and + everything received-but-unacknowledged is REDELIVERED - the standalone reproduction of what a + topic ownership move does on a multi-broker cluster, observed rather than assumed (the cell + waits for the consumer's `connectedSince` stamp to change). D6G is the replay's regression + gate: the unload lands mid-replay (6,000 messages give it a deterministic window), a + boundary-nacked word sits un-acked on the churned topic, and the chunk must stay EXACT across + the reconnect - every redelivered copy counted against the SAME replay range (the caught-up + count is exactly 6,000, never 6,001), the broker's bracketed dispatch counter proving + redeliveries really reached the session, and "Load new messages" then delivering the held next-chunk word + exactly once. D6S parks a skip-first-14 mid-count (12 seeded, dispatched, none shown), + unloads one stream, and asserts the two survivors BY NAME - a double-spent or under-spent + budget picks the wrong survivors, not just the wrong count. Fastest is skipped in D6: it makes + no ordering claim for churn to break, and its exactly-once under pressure is CS-PL/CS-FC's. +- **R REPLAY** (CS-DM-R1/R2/R3/R3B) - the redesign's own cells. R1: the banner names the boundary + instant ("Caught up to ") and, because words were recorded past the boundary mid-drain, the + approximate newer-data line ("~N newer entries" - the broker counts entries, so the banner + never claims an exact message count); the excluded-topics line stays silent with no regex in + play. R2: "Load new messages" extends the boundary REPEATABLY - three chunks through one session, each delta + exact, the banner returning at every new boundary. R3: Latest x Guaranteed UNGATED (owner + decision 2026-08-11, reversing the 2026-08-09 disabled-option gate) - "Latest message" stays + selectable under Guaranteed in BOTH directions, shows no note, and never rewrites the stored + value. R3B - the combination's PLAY behavior on a + topic that HOLDS a backlog (instant caught-up over an empty boundary, the pre-play backlog + unread, chunk verbs intact) - caught a real first-play boundary-clobbering bug on 2026-08-09 + and now pins the fixed contract: the create is the first Play, and the boundaries decided at + create are the first chunk's. + +Every cell's identity oracle is the exact MULTISET via `exportedValues` + +`DeliveredMessages.assertExactly` (a loss and a duplicate cannot cancel), with arrival order read +separately (`allColumnValues`) wherever the mode promises one, and `settledLoaded`'s bounded quiet +window only ever asserting "nothing more arrives". + +**The seam flag has no e2e cell, deliberately.** A seam violation needs a producer clock writing +an earlier timestamp into a pause window - controlled clock skew, unit-tier territory (the +hand-clock tests next to the merge). What the replay/resume cells DO assert, everywhere a +boundary is crossed: ZERO seam flags in ordinary flows - the session counter chip +(`cs-order-warning`, carrying the seam count since the 2026-08-11 toolbar rework) stays absent and no row carries `cs-out-of-order-marker` - so a +false positive cannot ship unnoticed. + +Alongside the matrix, the same redesign re-aimed the other Guaranteed pins: `CsMergeOrderSpec` +CS-MO-G1 (complete replay in exact key order, then the boundary pause), CS-MO-G2 (post-Play words +wait for the next chunk; "Load new messages" replays the delta), CS-MO-T1 (the documented tiebreak over real +publish-time ties, now across the WHOLE recorded set), CS-MO-0B (a single non-persistent stream +is an instant caught-up - nothing recorded, nothing replayed, and a word published into the pause +is honestly gone); `CsWideTopologySpec` CS-WT-2 (the barrier finishes at width 200 - the whole +20,000 replays, auto-pauses, and one closing word per partition arrives as a 200-message next chunk +delta); and `CsFlowControlSpec` CS-FC-5 (the byte watermark's hold is now an UNDELIVERABLE +recorded range - a force-deleted mid-drain topic - because an empty-at-boundary stream no longer +holds anything; its dispatch-counter oracle is CONNECTION-EPOCH-BRACKETED, since the broker's +counter counts dispatches, resets with the consumer, and a transient reconnect redelivers the +whole un-acked held window - read bare, that once looked like 400 MiB admitted past a 256 MiB +cap). CS-FC-2 kept its give-up contract untouched but needed one arranged premise: the tools +console now opens by default, so the collapsed-console layout oracle closes it first. + +**A live-edge stream resumed AGAIN with no delta re-arms the wait** (known corner, disclosed, no +e2e cell - owner decision pending). A second resume RPC - which is the FIRST 'Load new messages' the user +presses, since the create's Play consumed the boundary - of a Guaranteed session whose stream sits +at the live edge with NOTHING recorded since re-captures a boundary the extension cannot classify: +nothing was ever offered on that stream, so "cursor already at the end" and "cursor before the +end" look identical, and the barrier re-arms the wait. The session then stalls WITH the existing +stall disclosure (`cs-order-waiting`) until any traffic finishes the stream - honest, but not the +instant re-caught-up a user might expect. The fix would need a live-edge cursor floor recorded at +create - one extra read on the Latest fast path - and is deliberately not built until the owner +decides it is worth that cost. + +**Start-From: batching, and the entry-vs-message trap** (bug FIXED 2026-07-25; coverage now green) + +A Pulsar broker addresses its log by **entry**, and the Java producer packs many messages into one +entry by default. `PulsarAdmin.examineMessage` - which "Skip first n messages" and "Latest n +messages" used to be built on - therefore counts entries, not messages. Every fixture the suite had +produced *one message per entry* (a blocking `send` closes a one-message batch every time), so entry +positions and message positions always coincided and the difference was invisible: "skip the first +5" actually skipped 8 (or 50), and nothing caught it. + +`PulsarFixtures` now offers `produceBatched` / `produceUnbatched` / `numberOfEntries` / +`readAllMessages` / `messageIdHex`. **`produceBatched` throws** if the messages did not actually +share entries - an unbatched "batched" fixture would silently recreate the blind spot it exists to +close. `harness.BatchingFixtureSpec` (BATCH-1..4) pins the broker facts underneath: batching really +happens, `examineMessage` is entry-addressed, past the end it *clamps* on `"earliest"` but *throws* +on `"latest"`, and an empty partition cannot answer at all. + +Outcome coverage per mode lives in `CsStartFromOutcomesSpec` (CS-SF-1..19 and CS-SF-22, persistent +non-partitioned, batched and unbatched), `CsStartFromMatrixSpec` (skip-n / latest-n x batched / +unbatched x the whole `TopicKind` matrix, plus the spread-across-partitions cases) and +`CsApproximatePartitionedSpec` (CS-SF-20/21, the two approximate modes on a partitioned topic). The +contracts asserted, as implemented: + +- **Skip first n** drops n messages of the session's merged stream and delivers the rest - n in + TOTAL, across every physical topic. The merge takes each partition in its own append order and + compares publish times across the partitions' current heads, so on same-clock producers this is + the n globally-earliest; where producer clocks disagree, which n can shift while the count stays + exact. On a single ordered log it is exactly "start at message n + 1". `n = 0` shows everything. +- **Latest n** delivers **exactly n in total**, not n per partition, under the same head-comparison + ordering. + (Until 2026-07-25 it was resolved per physical topic, so "latest 2" on a 3-partition topic returned + six; CS-SFM-4 is the regression against that.) `n = 0` shows nothing and streams only what arrives + after play. One deliberate edge: if two ENABLED TARGETS select the same topic, each target delivers + its own counted set through its own subscription - see the duplicate-target contract note in + `handleStartFrom.scala`. +- **Approximate position (% of data)** is by ENTRY, not by message - it has to resolve in constant time at + any topic size - so 50% of twelve messages lands on m-07 written one per entry, while deliberately + uneven batches can put it at a very different message position. CS-SF-11/12 assert exactly that + difference from the broker's entry count. 0% is Earliest, 100% is Latest, and a percentage outside 0-100 is refused + without reaching the session - **CS-SF-14 presses Play with the rejected text still on screen**, so + what it pins is that the invalid value never reaches the session (the run comes out at the last + valid one), not merely that correcting the field afterwards works. +- **Approximate position (% of time)** interpolates between the first and last PUBLISH TIMES instead + and seeks to the resulting instant, so the same 50% lands somewhere else whenever messages did not + arrive evenly. CS-SF-16 pins the proportionality (25% and 75% of a 12-second range, each falling in + the middle of a gap); **CS-SF-17 asks one topic the same "50%" with both modes and asserts the two + different answers** - two old messages plus a burst of ten, where half the time is back in the + empty stretch while half the entries are inside the burst. Its endpoints are deliberately NOT the + entry mode's: 0% is Earliest but **100% includes every message tied at the latest publish + timestamp** (CS-SF-18), because the time range ends at that timestamp rather than past it. + CS-SF-19 presses Play on the rejected value the same way CS-SF-14 does. Separate logical topics + DO share one range (P2.15): CS-SF-22 gives two topics disjoint ranges and proves the session + resolves a single cutoff across both - it asserts the pooled answer and keeps the per-topic one + as the counterexample, so a regression to a range per topic fails there. +- **The two approximate modes diverge on a PARTITIONED topic**, and that is where their definitions + actually differ: the entry mode resolves **every physical topic independently** (each partition + leaves `floor(fraction x its own entry count)` entries behind) while the publish-time mode groups the + partitions by logical topic, pools `min(first publish time)` .. `max(last publish time)` across the + group, and seeks **every** partition to that one instant - so an idle partition cannot drag the + cutoff backwards and a late-starting one gets no range of its own. `CsApproximatePartitionedSpec` + CS-SF-20/21 arrange one topic skewed on both axes at once (partitions holding 8 / 4 / 2 / **0** + messages over three different stretches of a 12-second range, each pooled endpoint owned by exactly + one partition) and assert the exact set for each mode, having first asserted that the two modes - + and the mistake each is exposed to - really do give different answers on that arrangement. + The **empty** partition is part of the arrangement, not an accident: `examineMessage` does not + report an empty topic as an empty range, it *fails* (BATCH-4), so an empty partition reaches the + server as an error it has to tell apart from an operational one, and both modes then have to skip + it - counted as time zero it would drag the pooled start back to 1970 and turn 50% into Earliest. + Both tests also gate on the session holding flow permits on **every** partition, the empty one + included, so a server that quietly dropped it would fail rather than answer the same. The pure + arithmetic stays in `server/src/test/scala/consumer/session_runner/approximatePublishTimePositionTest.scala`. +- **Non-persistent targets** cannot honour any history mode, and the selector now says so: all eight + are rendered `disabled` with a note (`cs-start-from-non-persistent-note`), leaving only "Latest + message". CS-SFM-2 and CS-TK-5 assert the exact disabled set rather than watching a + permitted-but-meaningless selection behave. +- **Skip progress** (`cs-start-from-progress`) is deliberately silent at or below 1,000,000 messages + to skip. CS-SF-15 asks for a 2,000,000 skip on a 12-message topic - `messagesToSkip` is what was + asked for, not what exists - which is the one way to drive the server -> gRPC -> panel join that + neither the jest nor the server tests can reach. It requires `data-cs-skipped` to be **strictly + positive**: the server reports as soon as the discard claims its first message, so a zero would be + a frame the UI could have rendered from its own initial state rather than proof of a real callback. + +Honest limits: the **non-persistent** quadrants cannot prove batching at all (no managed ledger, so +no `numberOfEntries`). The single-log cells still funnel their payload through **one** partition - +not because the global contract needs it, but because that is where "the first n" is a plain slice of +the payload and the cell stays readable. + +`CsTopicKindsSpec` **CS-TK-6** was **untagged on 2026-07-25**: its `all.drop(2)` expectation is a +statement about a global order, which is exactly what Skip-N now promises. Verified green six runs +running. `CsStartFromMatrixSpec` CS-SFM-3/4 assert the same two contracts on a payload genuinely +spread over every partition, with the expectation derived from the **broker's own publish times** +(`globalOrder`) rather than from the produce order - and the arrangement spaces its publishes so that +no two messages in different partitions share a millisecond, which that helper asserts rather than +assumes. + +CS-SFM-3/4 configure **no message filter, value projection or coloring rule** on purpose, so a +failure there is about start-from and nothing else. + +What used to sit next door was a *concurrent-entry* hazard: a session gets **one** GraalVM JS context +(`ConsumerSessionContextPool` pins the pool to size 1) while a partitioned topic gives each partition +its own listener thread. GraalJS lets a context move between threads but not be entered by two at +once ("Multi threaded access ... is not allowed for language(s) js"), and nothing serialized them. +**That is now fixed**: `ConsumerSessionContext.exclusively` takes a reentrant lock and +`ConsumerSessionContextPool.withNextContext` leases it for a **whole message** - the filter chain, +the coloring rules, the projections and the `getStdout` drain all run inside one lease, so neither +the exception nor the subtler outcome (one message judged against another's `setCurrentMessage` +global) can happen. The browser console goes through the same lease. + +Both of the races this section used to list next to it are **fixed**: + +- *ordering* - `ConsumerListener.received` now resolves **and processes** inside + `startFromOrdering.inOrder`, so a vector reaches the target handler in the order the merge chose; +- *concurrent observer entry* - start-from progress no longer calls `StreamObserver.onNext` itself. + Every write goes through `ConsumerSessionRunner.sendResponse`, which holds `sendLock`. + +The two hazards this section used to file as still-open - about the ORDER and TERMINATION of what the +observer is handed rather than about entering it - are **fixed** as well, and pinned at the server +tier in `server/src/test/scala/consumer/session_runner/sessionOutputSerializationTest.scala` (suite +"progress never goes backwards, and nothing follows the end of the stream"): + +- `sendResponse` now **builds** its response - start-from progress included - **inside** `sendLock`, + in the same critical section as the `onNext` and behind an `if !streamCompleted` check, so two + threads can no longer snapshot an older, incomplete progress frame and send it after a newer + complete one. Pinned by "AN OLDER PROGRESS FRAME CANNOT OVERTAKE A NEWER COMPLETE ONE". +- `stop()` now calls `observer.onCompleted()` **inside** `sendLock`, behind a sticky `streamCompleted` + terminal gate, so nothing is written after completion and completion never interleaves with an + `onNext` still in flight. Pinned by "NO RESPONSE REACHES THE CLIENT AFTER THE STREAM HAS BEEN + COMPLETED" and "THE STREAM IS NOT COMPLETED WHILE A RESPONSE IS STILL BEING WRITTEN". + +Because those are unit-pinned at the server tier, this suite does not reproduce them: a browser +cannot see which of two frames the server built first, and the window is a few instructions wide. +Probed 2026-07-25 at 400 messages over 3 partitions, with and without a JS filter: no +multi-threaded-access error appeared then, and none has appeared in a full run since. + **Open product/config questions** (not test gaps) - **SUB-6**: Delete-Subscription's guard is the **topic FQN**, not the subscription name - the test encodes today's behavior; confirm it's intended. @@ -219,11 +713,33 @@ Honest backlog - none block the green lane; each is a place the suite proves les Pulsar 2.10.2 while overriding the image to 3.2.1 - template-validated only, never deployed. Align versions before relying on it. +**Message shapes that are entries but not messages.** Start-from's entry-addressed modes inspect +LEDGER ENTRIES, and several Pulsar features break the one-entry-one-message assumption. They inspect +them differently, which decides who is affected: **Latest-n** and **Approximate position (% of data)** +COUNT entries, so anything that makes an entry not equal one message skews them. **Approximate +position (% of time)** only samples the first and last entry's TIMESTAMPS and then seeks by time, so +it is count-free and immune. Batching (many messages per entry) and chunking (one message across +many entries) are covered by self-asserting fixtures - `produceBatched`, `produceChunked`, and +`BatchingFixtureSpec` / `MessageShapeFixtureSpec` pin the broker facts so a fixture cannot silently +stop producing the shape it claims. + +**Server-side markers have NO e2e coverage, and cannot**: transaction/replication markers occupy +entries but reach no consumer, and `dekaf-e2e-pulsar` runs with transactions disabled, so no marker +entry can exist in this harness. They are covered one tier down by `markerEntriesTest` (server) - +classification pinned with hand-built metadata, walk arithmetic with a pure lambda, and both verified +once against a throwaway transaction-enabled broker. `resolveLatestN` skips markers (a marker's exact +delivered count is zero, so crossing one stays correct); `ApproximateEntryPosition` is documented as +UNDETECTED there rather than fixed, because finding markers means examining entries, which is +O(topic) and defeats a mode whose whole value is a single counter read. Skip-n counts delivered +messages and all time-based positions are count-free, so both are immune. + **Regression coverage - the `KnownBug` lane** (catalogued bugs fixed; see §3). Twelve run green as ordinary regressions in `knownbugs/*Spec` (BUG-1,3,4,5,6,10,12,13,14,15,18 + -BUG-19→NAV-14). Six are fixed app-side but not driveable from Playwright and stay `ignore`d with inline -rationale - covered instead by **jest** component tests (BUG-2/7: `KeyValueEditor` / -`AvailableInContextsButton`) and **server** unit tests (BUG-8/9/17: `LibraryBugRegressionsTest`). +BUG-19→NAV-14). Five are fixed app-side but not driveable from Playwright and have **no e2e test at +all** - not an `ignore`d one; the suite carries none (§3). They are covered in another tier instead: +**jest** component tests (BUG-2/7: `KeyValueEditor` / `AvailableInContextsButton`) and **server** +unit tests (BUG-8/9/17: `LibraryBugRegressionsTest`), with a pointer at the foot of +`MoreKnownBugsSpec` where they would otherwise have lived. **Two were reclassified as intended design** after owner review: **BUG-11** - auto-refresh is deliberately ONE global toggle ("we either want to refresh any table, or not"); a `MoreKnownBugsSpec` test now pins the global-shared semantics. **BUG-16** - per-connection library-storage scoping was diff --git a/e2e/build.sbt b/e2e/build.sbt index 6b4cbcc39..a92446bd1 100644 --- a/e2e/build.sbt +++ b/e2e/build.sbt @@ -51,6 +51,11 @@ lazy val root = project // Test framework "org.scalatest" %% "scalatest" % scalatestVersion % Test, + // Parsing what the app EXPORTS (CS-28 reads the downloaded .zip's JSON back as records rather + // than substring-searching it). Already on the classpath via pulsar-client-admin-original; + // declared here, at that same version, because a test asserting on parsed JSON should not + // depend on which JSON library the Pulsar client happens to pull in. + "com.fasterxml.jackson.core" % "jackson-databind" % "2.14.2" % Test, // Renders ScalaTest's `-h` HTML report (see testOptions above); required on the classpath. "com.vladsch.flexmark" % "flexmark-all" % "0.64.8" % Test, ), diff --git a/e2e/scripts/fresh-data-dir.sh b/e2e/scripts/fresh-data-dir.sh new file mode 100755 index 000000000..91c715649 --- /dev/null +++ b/e2e/scripts/fresh-data-dir.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# The throwaway DEKAF_DATA_DIR used when DEKAF_FRESH_DATA=1 (see run-dekaf.sh). +# +# It lives here, in one script with two verbs, because two different scripts need to agree on it and +# they never run together: run-dekaf.sh creates it and then `exec`s the server, so the process that +# made the directory is replaced by one that has no idea it is temporary. CI later kills that whole +# process tree, so nothing the server could have registered - a trap, an atexit - ever runs either. +# The path was `mktemp -d`, i.e. a fresh unguessable name per run, which made the tree impossible to +# find afterwards and left one behind on the self-hosted runner every single build. +# +# A DETERMINISTIC path fixes both halves: the run before it can be cleared away up front, and the +# teardown step (stack-down.sh, which CI runs with `if: always()`) knows exactly what to remove +# without having been told. +# +# Deterministic is not enough on its own, though, and the first version of this script stopped +# there: ONE path shared by every stack on the box, `rm -rf`'d at startup by run-dekaf.sh and again +# at teardown by stack-down.sh. Two stacks running side by side - a second Dekaf on its own port, +# with or without a Pulsar container of its own, which is how you run two - would therefore delete +# each other's LIVE data dir. So the path also carries a STACK IDENTITY. +# +# That identity is derived, not configured: it is exactly the pair of variables that already +# distinguishes one stack from another and is already in the environment of BOTH ends - the Dekaf +# port run-dekaf.sh serves on, and the Pulsar container name stack-down.sh removes. Nothing has to +# be passed along or remembered; each end computes the same answer from what it was given anyway. +# +# fresh-data-dir.sh path print the directory (no side effects) +# fresh-data-dir.sh clean remove it if it exists +# +# NOTE for teardown: `clean` removes THIS stack's tree only, so it has to run with the same +# DEKAF_PORT / PULSAR_CONTAINER_NAME the stack was brought up with (CI sets DEKAF_PORT at job level, +# so every step of the job agrees). Getting that wrong now leaks a tree instead of destroying a live +# one, and `clean` names the trees it left behind so the leak is visible rather than silent. +# +# $RUNNER_TEMP is the GitHub-runner-scoped temp dir - already per-job and wiped by the runner - so it +# is the right home on CI; locally it falls back to $TMPDIR. +set -euo pipefail + +fresh_data_base() { + local base="${RUNNER_TEMP:-${TMPDIR:-/tmp}}" + echo "${base%/}/dekaf-e2e-fresh-data" +} + +# Which stack this is. Sanitized to [A-Za-z0-9._-] so a container name containing a slash cannot +# push the tree outside the base dir, and so an identity cannot collide with a different one that +# literally lacks the offending character: `tr -c` REPLACES each disallowed character with `_` +# rather than dropping it, so the character still leaves a mark (`a/b` -> `a_b`, not `ab`). +stack_id() { + printf '%s-%s' "${DEKAF_PORT:-8090}" "${PULSAR_CONTAINER_NAME:-dekaf-e2e-pulsar}" | tr -c 'A-Za-z0-9._-' '_' +} + +fresh_data_dir() { + echo "$(fresh_data_base)-$(stack_id)" +} + +case "${1:-path}" in + path) + fresh_data_dir + ;; + clean) + dir="$(fresh_data_dir)" + if [ -d "$dir" ]; then + rm -rf "$dir" + echo "Removed the e2e fresh data dir: $dir" + else + echo "No e2e fresh data dir to remove ($dir)." + fi + # Anything left belongs to a DIFFERENT stack identity and is deliberately not touched. Naming + # it is the whole mitigation for the one hazard per-stack paths introduce: a teardown run + # without the DEKAF_PORT / PULSAR_CONTAINER_NAME its stack used now leaks a tree, which is the + # failure mode the deterministic path was introduced to end. Better loud than invisible. + # The bare `$prefix` (no identity) is matched too: it is where the pre-identity version of this + # script put ITS tree, and a checkout old enough to have created one would otherwise leave it + # orphaned and invisible. Listing is all that happens to it - deleting a path that every stack + # once shared is the bug this identity exists to fix. + prefix="$(fresh_data_base)" + shopt -s nullglob + others=("$prefix"*) + shopt -u nullglob + if [ "${#others[@]}" -gt 0 ]; then + echo "Other stacks' e2e data dirs left untouched (each is removed by its own stack's teardown):" + printf ' %s\n' "${others[@]}" + fi + ;; + *) + echo "usage: $(basename "$0") [path|clean]" >&2 + exit 2 + ;; +esac diff --git a/e2e/scripts/run-dekaf.sh b/e2e/scripts/run-dekaf.sh index 63c99b65b..ea5c0a8b7 100755 --- a/e2e/scripts/run-dekaf.sh +++ b/e2e/scripts/run-dekaf.sh @@ -38,5 +38,32 @@ fi echo "Building UI bundle (ui/)..." (cd "$repo/ui" && npm run build) +# --- Optional per-run data isolation (DEKAF_FRESH_DATA=1) ----------------------------------------- +# Without this the Library writes into $repo/server/data/library and accumulates forever, so +# instance-scoped items (e.g. LIB-16's note) leak between runs and item counts drift. Opt-in +# rather than default because a local dev session usually WANTS its library to persist. +# A bare empty dir will not boot: ConsumerSessionContext reads js/dist/libs.js from the data dir, +# and the schema tooling reads proto/ - so seed both from the repo copy (built just above). +# +# The path is DETERMINISTIC (scripts/fresh-data-dir.sh) rather than `mktemp -d`. This script ends in +# `exec sbt run`, so the process that created the tree is gone by the time anyone could clean it up, +# and CI kills the resulting process tree outright - no trap here would ever fire. A known path is +# what lets the previous run's tree be cleared below and the current one be removed by the teardown +# step (stack-down.sh). With `mktemp -d` every build left one behind on the self-hosted runner. +# +# It is also PER STACK: the `rm -rf` below is why one shared path was dangerous - a second Dekaf on +# its own port would have wiped this one's live data dir on startup, and its teardown would have +# wiped it again. fresh-data-dir.sh derives the identity from DEKAF_PORT (exported above) and +# PULSAR_CONTAINER_NAME, both of which this script's environment already carries. +if [ "${DEKAF_FRESH_DATA:-}" = "1" ]; then + fresh_data="$("$here/fresh-data-dir.sh" path)" + rm -rf "$fresh_data" + mkdir -p "$fresh_data/library" + cp -R "$repo/server/data/js" "$fresh_data/js" + cp -R "$repo/server/data/proto" "$fresh_data/proto" + export DEKAF_DATA_DIR="$fresh_data" + echo "Using a fresh data dir (DEKAF_FRESH_DATA=1): $fresh_data" +fi + echo "Starting Dekaf on :${DEKAF_PORT} → admin :${ADMIN_PORT}, broker :${BROKER_PORT} ..." cd "$repo/server" && exec sbt run diff --git a/e2e/scripts/stack-down.sh b/e2e/scripts/stack-down.sh index 5626b834f..7ef61ff7b 100755 --- a/e2e/scripts/stack-down.sh +++ b/e2e/scripts/stack-down.sh @@ -1,9 +1,20 @@ #!/usr/bin/env bash # Tear down the local Pulsar standalone started by stack-up.sh. set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NAME="${PULSAR_CONTAINER_NAME:-dekaf-e2e-pulsar}" if docker rm -f "$NAME" >/dev/null 2>&1; then echo "Stopped and removed '$NAME'." else echo "No '$NAME' container running." fi + +# The DEKAF_FRESH_DATA tree, if run-dekaf.sh made one. Nothing else can: that script `exec`s the +# server, and CI kills the process tree, so no trap or atexit in the server's own lifetime ever runs. +# This is the teardown hook CI already calls with `if: always()`, which is why the removal lives here. +# +# THIS stack's tree only - the path carries a stack identity derived from DEKAF_PORT and +# PULSAR_CONTAINER_NAME (the same variable that named the container above), so tearing one stack +# down cannot delete a concurrent stack's live data. Run this with the same values the stack was +# brought up with; `clean` prints any tree it deliberately left behind. +"$here/fresh-data-dir.sh" clean diff --git a/e2e/src/main/scala/features/consumersession/ConsumerSessionPage.scala b/e2e/src/main/scala/features/consumersession/ConsumerSessionPage.scala index 159a0a5f8..aa83fd904 100644 --- a/e2e/src/main/scala/features/consumersession/ConsumerSessionPage.scala +++ b/e2e/src/main/scala/features/consumersession/ConsumerSessionPage.scala @@ -11,20 +11,44 @@ import scala.jdk.CollectionConverters.* * Merged surface for both the configuration (CS-2..15) and runtime (CS-16..32) specs. */ final case class ConsumerSessionPage(page: Page): // --- Toolbar / table (pre-existing) --- - val playButton: Locator = page.getByTestId("cs-play") - val stopButton: Locator = page.getByTestId("cs-stop") - val toolsButton: Locator = page.getByTestId("cs-tools") - val searchInput: Locator = page.getByTestId("cs-search") - val startFromSelect: Locator = page.getByTestId("cs-start-from") - val messages: Locator = page.getByTestId("cs-message") - val messageDetails: Locator = page.getByTestId("cs-message-details") + val playButton: Locator = page.getByTestId("cs-play") + val stopButton: Locator = page.getByTestId("cs-stop") + val toolsButton: Locator = page.getByTestId("cs-tools") + val toolsPanel: Locator = page.getByTestId("cs-console") + val toolsResizeHandle: Locator = page.getByTestId("cs-tools-resize-handle") + val toolsCloseButton: Locator = page.getByTestId("cs-tools-close") + val searchInput: Locator = page.getByTestId("cs-search") + val startFromSelect: Locator = page.getByTestId("cs-start-from") + val messages: Locator = page.getByTestId("cs-message") + val messageDetails: Locator = page.getByTestId("cs-message-details") // --- Start From additional inputs (CS-2/3) --- val startFromN: Locator = page.getByTestId("cs-start-from-n") val startFromMessageId: Locator = page.getByTestId("cs-start-from-message-id") - // --- Advanced reveal (CS-15) --- - val advancedToggle: Locator = page.getByTestId("cs-advanced-toggle") + /** The Start-From "additional controls" block - whatever the selected mode reveals below the + * dropdown (the n input, the message-id input, the datetime picker, the relative picker). + * + * Anchored on the CSS-module class PREFIX rather than a `testId`: the datetime and relative + * pickers are third-party/shared components with no instrumentation of their own, and `ui/` is + * out of scope for this change. The `-module__AdditionalControls` prefix is stable across builds; + * only the trailing content hash moves. Scoping is REQUIRED, not cosmetic - the always-mounted + * Producer console renders its own datetime picker, so a page-wide `input[name='year']` matches + * two different controls. */ + val startFromAdditional: Locator = page.locator("[class*='StartFromInput-module__AdditionalControls']") + + // --- The two approximate modes + persistency advice --- + // Both render the SAME percent control, so each carries its own test-id prefix; sharing one would + // let a test drive the entry position while asserting the publish-time position. + val startFromEntryFraction: Locator = page.getByTestId("cs-start-from-entry-fraction") + val startFromEntryFractionSlider: Locator = page.getByTestId("cs-start-from-entry-fraction-slider") + val startFromEntryFractionError: Locator = page.getByTestId("cs-start-from-entry-fraction-error") + val startFromPublishTimeFraction: Locator = page.getByTestId("cs-start-from-publish-time-fraction") + val startFromPublishTimeFractionSlider: Locator = page.getByTestId("cs-start-from-publish-time-fraction-slider") + val startFromPublishTimeFractionError: Locator = page.getByTestId("cs-start-from-publish-time-fraction-error") + val startFromNonPersistentNote: Locator = page.getByTestId("cs-start-from-non-persistent-note") + val startFromMixedPersistencyNote: Locator = page.getByTestId("cs-start-from-mixed-persistency-note") + val startFromProgress: Locator = page.getByTestId("cs-start-from-progress") // --- Session-level editor containers --- val sessionFilters: Locator = page.getByTestId("cs-session-filters") @@ -48,22 +72,152 @@ final case class ConsumerSessionPage(page: Page): val awaitingText: Locator = page.getByText("Awaiting for new messages...") // --- Runtime table / details / export (CS-18..29) --- - val session: Locator = page.getByTestId("cs-session") - val table: Locator = page.getByTestId("cs-table") - val numFound: Locator = page.getByTestId("cs-num-found") - val loaded: Locator = page.getByTestId("cs-loaded") - val messageDetailsClose: Locator = page.getByTestId("cs-message-details-close") - val exportOpen: Locator = page.getByTestId("cs-export-open") + val session: Locator = page.getByTestId("cs-session") + val table: Locator = page.getByTestId("cs-table") + /** react-virtuoso owns the actual vertical scroll offset on this child, not on `table`. */ + val tableScroller: Locator = table.locator("[data-virtuoso-scroller]").first() + val numFound: Locator = page.getByTestId("cs-num-found") + val loaded: Locator = page.getByTestId("cs-loaded") + val messageDetailsClose: Locator = page.getByTestId("cs-message-details-close") + val messageDetailsResizeHandle: Locator = page.getByTestId("cs-message-details-resize-handle") + val exportOpen: Locator = page.getByTestId("cs-export-open") def th(key: String): Locator = page.getByTestId(s"cs-th-$key") def cell(key: String): Locator = page.getByTestId(s"cs-cell-$key") val selectedIndexCell: Locator = page.locator("[data-testid='cs-message'][data-cs-selected='true']") + /** String cells render JSON-quoted (`"k-1"`); a single surrounding quote pair is stripped. */ + private def stripRenderedQuotes(v: String): String = + if v.length >= 2 && v.startsWith("\"") && v.endsWith("\"") then v.substring(1, v.length - 1) else v + /** The visible values of a message column, in rendered row order (e.g. `columnValues("key")`). - * String cells render JSON-quoted (`"k-1"`), so a single surrounding quote pair is stripped. */ + * + * MOUNTED rows only: the table is react-virtuoso, so this sees one viewport-worth of rows and - + * on a running session, which auto-scrolls to the bottom every 200ms - specifically the LAST + * viewport-worth. Fine for sets that fit a viewport and for emptiness checks; any expectation + * that can outgrow a viewport must use [[allColumnValues]] instead (CS-SF-7/9/13/14 failed on + * exactly this, all "missing m-01" - the oldest rows had scrolled off the top). */ def columnValues(key: String): List[String] = - cell(key).allInnerTexts().asScala.toList.map(_.trim).map: v => - if v.length >= 2 && v.startsWith("\"") && v.endsWith("\"") then v.substring(1, v.length - 1) else v + cell(key).allInnerTexts().asScala.toList.map(_.trim).map(stripRenderedQuotes) + + /** Every value of a message column across the WHOLE virtualized list, not just the mounted + * viewport - the scroll-and-collect pattern `CsMergeOrderSpec.deliveredValues` introduced, + * shared so every exact-set assertion benefits. + * + * The entire walk runs inside ONE `page.evaluate`: it steps the virtuoso scroller from the top, + * waits two animation frames per step for React to mount the range, and collects + * (displayIndex, cell text) pairs keyed by the arrival ordinal so a row is never double-counted + * across steps. Unlike `deliveredValues` it does NOT pause the session (callers keep producing + * into live sessions afterwards); instead the running-state auto-scroll is treated as a rival: + * a step whose position was hijacked (the scroller is not where the walk put it) is detected + * and retried, bounded, so the walk terminates either way - at worst the outer polling loop + * calls it again. Returned in arrival (displayIndex) order, with the same one-quote-pair strip + * `columnValues` applies. Parsed with jackson rather than a regex so payloads containing + * brackets, quotes or newlines cannot be silently dropped. */ + def allColumnValues(key: String): List[String] = + val json = page + .evaluate( + s"""async () => { + | const table = document.querySelector("[data-testid='cs-table']"); + | if (!table) return "[]"; + | const byOrdinal = new Map(); + | const harvest = () => { + | for (const r of table.querySelectorAll("tbody tr")) { + | const ord = r.querySelector("[data-testid='cs-message']"); + | const cell = r.querySelector("[data-testid='cs-cell-$key']"); + | if (ord && cell) { + | const n = parseInt(ord.innerText.trim(), 10); + | if (!isNaN(n)) byOrdinal.set(n, cell.innerText.trim()); + | } + | } + | }; + | const result = () => JSON.stringify(Array.from(byOrdinal.entries())); + | harvest(); + | const s = table.querySelector("[data-virtuoso-scroller]"); + | if (!s || s.scrollHeight <= s.clientHeight + 2) return result(); + | const frame = () => new Promise(res => requestAnimationFrame(() => requestAnimationFrame(res))); + | let target = 0; + | let guard = 0; + | while (guard < 300) { + | guard += 1; + | s.scrollTop = target; + | await frame(); + | const atEnd = target >= s.scrollHeight - s.clientHeight - 2; + | if (Math.abs(s.scrollTop - target) > 2 && !atEnd) continue; // auto-scroll hijacked the step - retry it + | harvest(); + | if (atEnd) break; + | target = Math.min(target + s.clientHeight * 0.8, s.scrollHeight - s.clientHeight); + | } + | return result(); + |}""".stripMargin + ) + .toString + ConsumerSessionPage.jsonMapper + .readTree(json) + .elements() + .asScala + .map(pair => pair.get(0).asInt() -> pair.get(1).asText()) + .toList + .sortBy(_._1) + .map((_, value) => stripRenderedQuotes(value.trim)) + + /** EVERY loaded message's value, read out of the app's own bulk export instead of the table. + * + * [[allColumnValues]] walks the virtualized list a viewport at a time and gives up after 300 + * steps, which is a few thousand rows on this 1280x800 context - so the exact-set assertions the + * high-pressure specs need (tens of thousands of messages) are simply out of its reach. The + * export writes the whole retained buffer, not the mounted rows, in one file. + * + * TWO PREMISES, both enforced rather than assumed, because either one silently truncates the + * oracle into something that still looks like a set: + * - the session must NOT be running: while running the app shows (and exports) only the last + * 250 messages, so a running export would quietly compare 250 rows and pass; + * - the session's retention (`data-cs-retained`, the `numDisplayItems` limit) must be wide + * enough to still hold everything the test produced - see [[setNumDisplayItems]]. Callers + * assert that themselves against their own expected total, which this cannot know. + * + * Returned in the table's DISPLAY order (publish time ascending, stable), not arrival order: + * this is a SET oracle. Arrival order has its own reader in `CsMergeOrderSpec`. */ + def exportedValues(): List[String] = + assert( + state != "running", + "exportedValues must be called on a paused/stopped session: a RUNNING session renders and " + + "exports only its last 250 messages, so the exported set would silently be a tail" + ) + exportOpen.click() + val modal = ExportModal(page) + modal.selectFormat("json-value-per-entry") + // Generous: the export serializes, zips and downloads the whole retained buffer, which is tens + // of thousands of messages in the pressure specs. + val download = + page.waitForDownload(new Page.WaitForDownloadOptions().setTimeout(120000), () => modal.runButton.click()) + val zipPath = java.nio.file.Files.createTempFile("cs-exported-values", ".zip") + try + download.saveAs(zipPath) + val zip = new java.util.zip.ZipFile(zipPath.toFile) + try + zip.entries().asScala + .filterNot(_.isDirectory) + .map(e => e.getName -> new String(zip.getInputStream(e).readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)) + .toList + // The exporter chunks by size and names each chunk `-.json`; + // ordering by that first index keeps a multi-chunk export in one sequence. + .sortBy((name, _) => ConsumerSessionPage.chunkFirstIndex(name)) + .flatMap { (name, text) => + val root = ConsumerSessionPage.jsonMapper.readTree(text) + assert(root.isArray, s"the exported chunk '$name' is not a JSON array: ${text.take(200)}") + root.elements().asScala.toList.map { node => + // The premise the whole comparison rests on: the exporter writes each value as it + // reached the browser, and these specs produce STRING payloads, so every entry must + // be a JSON string. A payload that stopped being one would otherwise compare as "". + assert(node.isTextual, s"an exported value is not a JSON string: ${node.toString.take(200)}") + node.asText + } + } + finally zip.close() + finally + java.nio.file.Files.deleteIfExists(zipPath) + modal.close() /** The rendered (DOM) width of a column header in px - proves a persisted width is actually applied. */ def columnDomWidth(key: String): Double = @@ -90,6 +244,13 @@ final case class ConsumerSessionPage(page: Page): def firstRenderedIndex: Int = messages.allInnerTexts().asScala.headOption.map(_.trim).flatMap(_.toIntOption).getOrElse(-1) + /** Pixel position of the virtualized message viewport. This complements `firstRenderedIndex`: + * the rendered range can stay the same after a small but still user-visible scroll jump. */ + def tableScrollTop: Double = + tableScroller.evaluate("element => element.scrollTop") match + case n: Number => n.doubleValue() + case value => throw new AssertionError(s"message viewport scrollTop is not numeric: $value") + // --- Scoped sub-objects --- def sessionFilterPanel: FilterPanel = FilterPanel(page, sessionFilters) /** The per-target filter chain (visible without "advanced"); this is the path that actually filters @@ -98,17 +259,100 @@ final case class ConsumerSessionPage(page: Page): def target(i: Int = 0): TargetSelector = TargetSelector(page, targets.nth(i)) // --- Navigation --- + // + // openForTopic PINS the delivery order to Best effort. The generic feature lanes (filters, + // table, chunking, start-from, library, ...) were written and verified against live-following + // delivery; since 2026-08-11 the product DEFAULT is Guaranteed (an exact replay that + // auto-pauses when caught up), under which their premises - a session that keeps 'running', + // post-Play traffic streaming in - do not hold. Per the standing rule, a scenario must not + // inherit the default it does not test, so the shared entry selects Best effort explicitly. + // Mode-aware specs overwrite it per cell; the DEFAULT itself is pinned by CS-MO-0 through + // openForTopicDefaults, the one navigation that leaves the configuration untouched. def openForTopic(tenant: String, namespace: String, topic: String): Unit = + openForTopic(tenant, namespace, topic, persistency = "persistent") + + /** `persistency` is the route segment: "persistent" or "non-persistent". */ + def openForTopic(tenant: String, namespace: String, topic: String, persistency: String): Unit = + page.navigate(s"/tenants/$tenant/namespaces/$namespace/topics/$persistency/$topic/consumer-session") + setDeliveryOrder("Best effort") + + /** Navigate WITHOUT touching the configuration - what CS-MO-0 pins the product default through. */ + def openForTopicDefaults(tenant: String, namespace: String, topic: String): Unit = page.navigate(s"/tenants/$tenant/namespaces/$namespace/topics/persistent/$topic/consumer-session") - /** Namespace-level mount (no current topic) - drives CS-7. */ + /** Namespace-level mount (no current topic) - drives CS-7 and the no-current-topic lanes; pins + * Best effort for the same reason openForTopic does. */ def openForNamespace(tenant: String, namespace: String): Unit = page.navigate(s"/tenants/$tenant/namespaces/$namespace/consumer-session") + setDeliveryOrder("Best effort") + + /** Switch target 1's topic selector to "Specific Topic(s)" and enter the given FQNs. */ + def setTargetTopicsSpecific(topicFqns: Seq[String]): Unit = + page.getByTestId("cs-target-mode").first().selectOption(new SelectOption().setValue("multi-topic-selector")) + topicFqns.foreach { fqn => + val input = page.getByTestId("cs-target-fqn-input").first() + input.fill(fqn) + input.press("Enter") + } + + /** Toggle target 1's read-compacted consumption mode. */ + def toggleTargetCompacted(): Unit = page.getByTestId("cs-target-compacted").first().click() + + val startFromDegradedBanner: Locator = page.getByTestId("cs-start-from-degraded") def setStartFrom(label: String): Unit = startFromSelect.selectOption(new SelectOption().setLabel(label)) - def revealAdvanced(): Unit = advancedToggle.click() + /** Every Start-From option as (visible label, is it selectable). A mode the selected topics cannot + * honour is rendered `disabled` rather than hidden, so the list is a stable catalog and the flag + * is what varies - which is why both halves are read here rather than just the labels. */ + def startFromOptions: List[(String, Boolean)] = + startFromSelect.locator("option").all().asScala.toList + .map(o => o.innerText().trim -> (o.getAttribute("disabled") == null)) + + def startFromLabels: List[String] = startFromOptions.map(_._1) + def disabledStartFromLabels: List[String] = startFromOptions.filterNot(_._2).map(_._1) + + /** Start From = "Approximate position (% of data)", set to `percent` of the retained broker entries + * (0-100; the model stores the fraction). */ + def setStartFromEntryPercent(percent: String): Unit = startFromEntryFraction.fill(percent) + + /** Start From = "Approximate position (% of time)", set to `percent` between a logical topic's + * retained boundary-entry publish times (0-100; the model stores the fraction). */ + def setStartFromPublishTimePercent(percent: String): Unit = startFromPublishTimeFraction.fill(percent) + + /** Start From = "Specific time", set to `at` in the BROWSER's local zone (same machine as the + * test JVM, so `LocalDateTime.ofInstant(i, ZoneId.systemDefault)` is the right conversion). + * + * The picker is second-granular and rebuilds its Date from ALL six inputs on every change, so + * the fields are filled in coarse-to-fine order and the final `second` write is what commits the + * complete tuple. That last write is nudged through a different value first: React suppresses an + * onChange when the input's value is unchanged, which would otherwise leave the picker holding + * whatever it was seeded with (`new Date()`) whenever the target second happened to match. */ + def setStartFromDateTime(at: java.time.LocalDateTime): Unit = + def part(name: String, value: Int): Unit = + startFromAdditional.locator(s"input[name='$name']").fill(value.toString) + part("year", at.getYear) + part("month", at.getMonthValue) + part("day", at.getDayOfMonth) + part("hour24", at.getHour) + part("minute", at.getMinute) + part("second", if at.getSecond == 0 then 1 else 0) + part("second", at.getSecond) + + /** Start From = "Relative time ago", e.g. `setStartFromRelative(10, "second")`. `unit` is the + * option value: second | minute | hour | day | week | month | year. */ + def setStartFromRelative(value: Int, unit: String): Unit = + startFromAdditional.locator("input[type='number']").fill(value.toString) + startFromAdditional.locator("select").selectOption(new SelectOption().setValue(unit)) + + /** Delivery order: 'Guaranteed', 'Best effort', or 'Fastest'. */ + def setDeliveryOrder(label: String): Unit = + page.getByTestId("cs-delivery-order").selectOption(new SelectOption().setLabel(label)) + + /** Order by: 'Publish time', 'Broker publish time', or 'Event time'. Hidden for Fastest. */ + def setOrderTime(label: String): Unit = + page.getByTestId("cs-delivery-order-key").selectOption(new SelectOption().setLabel(label)) def setDeserializer(label: String): Unit = deserializerSelect.selectOption(new SelectOption().setLabel(label)) def addTarget(): Unit = addTargetButton.click() @@ -133,10 +377,88 @@ final case class ConsumerSessionPage(page: Page): playButton.click() def stop(): Unit = stopButton.click() + + // --- the browser-wide delivery controls (localStorage-backed, in the toolbar) --- + val rateLimitInput: Locator = page.getByTestId("cs-rate-limit") + val pauseAfterInput: Locator = page.getByTestId("cs-pause-after") + + /** Commit a rate limit (msgs/second, 0 clears). The input is draft-committed on Enter/blur. */ + def setRateLimit(n: Int): Unit = + rateLimitInput.fill(if n > 0 then n.toString else "") + rateLimitInput.press("Enter") + + /** Commit an auto-pause threshold (messages loaded, 0 clears). */ + def setPauseAfter(n: Int): Unit = + pauseAfterInput.fill(if n > 0 then n.toString else "") + pauseAfterInput.press("Enter") def clickFirstMessage(): Unit = messages.first().click() def searchInResults(t: String): Unit = searchInput.fill(t) - // Force-click: the button's own "Toggle additional tools" tooltip can overlay it and intercept a normal click. - def openTools(): Unit = toolsButton.click(new Locator.ClickOptions().setForce(true)) + /** Open without accidentally closing an already-open default/persisted panel. Waiting for the + * toolbar first keeps the visibility check from racing the initial React render. */ + def openTools(): Unit = + assertThat(toolsButton).isVisible() + if !toolsResizeHandle.isVisible then toggleTools() + + /** Close through the panel's user-facing cross, but remain idempotent for setup/cleanup paths. */ + def closeTools(): Unit = + assertThat(toolsButton).isVisible() + if toolsResizeHandle.isVisible then toolsCloseButton.click() + + /** Deliberately invert visibility. Force-click because the button's tooltip can overlap it. */ + def toggleTools(): Unit = toolsButton.click(new Locator.ClickOptions().setForce(true)) + + /** Drag either consumer-session pane handle through the real document-level mouse path used by + * the resize hook. Positive x/y moves right/down; negative moves left/up. */ + private def dragPaneResize(handle: Locator, dx: Int, dy: Int): Unit = + val b: BoundingBox = handle.boundingBox() + assert(b != null, s"resize handle '${handle}' has no live bounding box") + val sx = b.x + b.width / 2 + val sy = b.y + b.height / 2 + page.mouse().move(sx, sy) + page.mouse().down() + page.mouse().move(sx + dx, sy + dy, new Mouse.MoveOptions().setSteps(8)) + page.mouse().up() + // The shared resize primitive batches updates through requestAnimationFrame. + page.waitForTimeout(100) + + /** The bottom panel grows when its top edge is dragged up, and shrinks when dragged down. */ + def resizeToolsBy(dy: Int): Unit = dragPaneResize(toolsResizeHandle, dx = 0, dy = dy) + + /** The right-hand inspector grows when its left edge is dragged left, and shrinks when dragged right. */ + def resizeMessageDetailsBy(dx: Int): Unit = dragPaneResize(messageDetailsResizeHandle, dx = dx, dy = 0) + + def toolsPanelDomHeight: Double = + val b = toolsPanel.boundingBox() + if b == null then -1.0 else b.height + + def sessionDomHeight: Double = + val b = session.boundingBox() + if b == null then -1.0 else b.height + + def tableDomWidth: Double = + val b = table.boundingBox() + if b == null then -1.0 else b.width + + def messageDetailsDomWidth: Double = + val b = messageDetails.boundingBox() + if b == null then -1.0 else b.width + + /** Sizes are JSON numbers because the panes use `use-local-storage-state`. */ + def storedPaneSize(paneId: String): Double = + val value = page.evaluate( + s"""() => { + | const raw = localStorage.getItem('pane:$paneId:size'); + | if (raw === null) return -1; + | const parsed = JSON.parse(raw); + | return typeof parsed === 'number' ? parsed : -1; + |}""".stripMargin + ) + value match { case n: Number => n.doubleValue(); case _ => -1.0 } + + def setStoredPaneSize(paneId: String, size: Double): Unit = + page.evaluate( + s"""() => localStorage.setItem('pane:$paneId:size', JSON.stringify($size))""" + ) // --- state (CS-16..19) --- def state: String = session.getAttribute("data-cs-state") @@ -150,9 +472,47 @@ final case class ConsumerSessionPage(page: Page): assertThat(messages).hasCount(n, new LocatorAssertions.HasCountOptions().setTimeout(timeoutMs)) /** Wait until the toolbar reports `n` messages loaded. Use this instead of `waitMessages` when n is - * larger than a viewport - the message table is virtualized, so DOM rows != loaded messages. */ + * larger than a viewport - the message table is virtualized, so DOM rows != loaded messages. + * + * The counter renders through `numeral(n).format('0,0')`, so the expected text must carry the + * same thousands separator - `hasText("2000")` against a counter showing "2,000" waited out its + * whole timeout on a session that had in fact finished. */ def awaitLoaded(n: Int, timeoutMs: Double = 30000): Unit = - assertThat(loaded).hasText(n.toString, new LocatorAssertions.HasTextOptions().setTimeout(timeoutMs)) + val expected = java.text.NumberFormat.getIntegerInstance(java.util.Locale.US).format(n.toLong) + assertThat(loaded).hasText(expected, new LocatorAssertions.HasTextOptions().setTimeout(timeoutMs)) + + /** How many messages the session is actually HOLDING, after display retention - a different + * number from [[loadedCount]] and read from a different source. + * + * `cs-loaded` is the SERVER's `numMessageSent`; this is the length of the browser's own message + * array, which is capped at the session's `numDisplayItems` limit (10,000 unless the config + * raises it). Any exact-set assertion over the whole delivered stream is only meaningful while + * these two agree - past the cap the browser has deliberately dropped its oldest rows, and the + * exported set is a tail rather than the whole story. */ + def retainedCount: Int = + val raw = session.getAttribute("data-cs-retained") + raw.toIntOption.getOrElse(throw new AssertionError(s"data-cs-retained does not read as a number: '$raw'")) + + /** The Session Configuration's "Limit num. display messages" toggle and its number field. */ + val limitDisplayItemsToggle: Locator = page.getByTestId("cs-limit-display-items") + val numDisplayItemsInput: Locator = page.getByTestId("cs-num-display-items") + + /** Keep at most `n` messages on screen - the session's own `numDisplayItems` retention. + * + * A test that wants to assert the EXACT set of a large delivery has to raise this: the default + * is 10,000 and the browser drops its oldest rows past it, so beyond that the delivered set is + * unobservable and only counts remain. The toggle is what reveals the field (it is + * `visibility: hidden` while off, so Playwright cannot fill it), and it is idempotent here so a + * caller need not know the current state. */ + def setNumDisplayItems(n: Int): Unit = + if limitDisplayItemsToggle.getAttribute("data-checked") != "true" then limitDisplayItemsToggle.click() + numDisplayItemsInput.fill(n.toString) + + /** The toolbar's loaded counter as a number, right now. Digits only: it is rendered through + * `numeral(n).format('0,0')`, so a four-figure count carries a thousands separator. */ + def loadedCount: Int = + val digits = loaded.innerText().replaceAll("[^0-9]", "") + digits.toIntOption.getOrElse(throw new AssertionError(s"the loaded counter does not read as a number: '${loaded.innerText()}'")) // --- lifecycle triggers (CS-18) --- def wheelUpOverTable(): Unit = @@ -210,3 +570,13 @@ final case class ConsumerSessionPage(page: Page): def pressKeyOnTable(key: String): Unit = page.keyboard().press(key) page.waitForTimeout(120) + +object ConsumerSessionPage: + /** Parses what the full-table harvest returns. jackson-databind is on the compile classpath via + * the Pulsar admin client (and version-pinned for tests in build.sbt). */ + private val jsonMapper = new com.fasterxml.jackson.databind.ObjectMapper() + + /** The first message index in an exported chunk's file name (`/-.json`), or + * `Int.MaxValue` for the exporter's fallback names, which sort last. */ + private def chunkFirstIndex(entryName: String): Int = + entryName.substring(entryName.lastIndexOf('/') + 1).takeWhile(_.isDigit).toIntOption.getOrElse(Int.MaxValue) diff --git a/e2e/src/main/scala/features/consumersession/DeliveredMessages.scala b/e2e/src/main/scala/features/consumersession/DeliveredMessages.scala new file mode 100644 index 000000000..49a15532c --- /dev/null +++ b/e2e/src/main/scala/features/consumersession/DeliveredMessages.scala @@ -0,0 +1,71 @@ +package features.consumersession + +/** The IDENTITY oracle for what a consumer session delivered. + * + * WHY THIS EXISTS. "Zero loss" used to be concluded from the final loaded COUNT: produce 55,000, + * see 55,000, done. A count cannot distinguish correct from merely plausible - one message lost + * anywhere plus one delivered twice anywhere reaches exactly the same number - and the count the + * consumer-session toolbar shows is not even read from the delivered rows: it is the server's own + * `numMessageSent`, so a send counted but never rendered, or a rendered row the server never + * counted, are both invisible to it. `harness.DeliveredMessagesSpec` pins that difference by + * running both oracles over the same wrong data. + * + * MULTISET, not order. Delivery order across a pause is deliberately NOT a promise: a message + * refused at the closed gate is negative-acknowledged and comes back on the broker's redelivery + * schedule, so it can legitimately arrive behind a newer one (see `ConsumerListener`'s + * "OUTWARD-ONLY BOUNDS" note). Ordering is a separate contract with its own coverage in + * `CsMergeOrderSpec`, which reads arrival ordinals rather than a set. So the comparison here + * counts occurrences: a duplicate is as much a failure as a loss, and neither can hide behind the + * other. + * + * BOUNDED DIAGNOSTICS. These sets run to tens of thousands, so a bare `==` failure would print + * two unreadable walls. The report says how many are missing, how many are unexpected, how many of + * the unexpected are DUPLICATES of expected values, and shows a sample of each. + */ +object DeliveredMessages: + + /** How many samples of each kind a failure report shows. */ + private val SampleSize = 10 + + private def occurrences(values: Seq[String]): Map[String, Int] = + values.groupMapReduce(identity)(_ => 1)(_ + _) + + /** `None` when `delivered` and `expected` are the same MULTISET, otherwise the diagnosis. */ + def difference(delivered: Seq[String], expected: Seq[String]): Option[String] = + val deliveredCounts = occurrences(delivered) + val expectedCounts = occurrences(expected) + + // (value, how many copies are owed / how many are surplus) + def shortfall(want: Map[String, Int], have: Map[String, Int]): List[(String, Int)] = + want.iterator + .map((value, wanted) => value -> (wanted - have.getOrElse(value, 0))) + .filter(_._2 > 0) + .toList + .sortBy(_._1) + + val missing = shortfall(expectedCounts, deliveredCounts) + val unexpected = shortfall(deliveredCounts, expectedCounts) + + if missing.isEmpty && unexpected.isEmpty then None + else + // A surplus copy of a value that WAS expected is a duplicate delivery; a surplus copy of a + // value that was not expected is the wrong message entirely. Naming which one it is turns a + // failure into a diagnosis - they have completely different causes. + val (duplicates, foreign) = unexpected.partition((value, _) => expectedCounts.contains(value)) + def sample(what: String, entries: List[(String, Int)]): String = + if entries.isEmpty then "" + else + val shown = entries.take(SampleSize).map((value, n) => if n == 1 then value else s"$value x$n") + val more = if entries.size > SampleSize then s", ... (${entries.size - SampleSize} more)" else "" + s"\n $what: ${entries.map(_._2).sum} message(s) over ${entries.size} value(s): ${shown.mkString(", ")}$more" + Some( + s"delivered ${delivered.size} message(s), expected ${expected.size}" + + sample("MISSING (never delivered)", missing) + + sample("DUPLICATED (delivered more than once)", duplicates) + + sample("FOREIGN (not in the expected set at all)", foreign) + ) + + /** Fail unless `delivered` is EXACTLY `expected`, counting duplicates. `what` names the set so a + * spec asserting several of them says which one broke. */ + def assertExactly(delivered: Seq[String], expected: Seq[String], what: String): Unit = + difference(delivered, expected).foreach(problem => throw new AssertionError(s"$what: $problem")) diff --git a/e2e/src/main/scala/features/consumersession/ToolsPanel.scala b/e2e/src/main/scala/features/consumersession/ToolsPanel.scala index cf91e5e17..b5252b0b7 100644 --- a/e2e/src/main/scala/features/consumersession/ToolsPanel.scala +++ b/e2e/src/main/scala/features/consumersession/ToolsPanel.scala @@ -2,7 +2,25 @@ package features.consumersession import com.microsoft.playwright.{Locator, Page} -/** The Tools/Console bottom panel (toggled by `cs-tools`). */ +/** Default Topic Positions column indices. Tests use a fresh browser context, so there is no + * persisted user reorder and the table starts in this order. */ +object TopicPositionsColumn: + val Topic = 0 + val RetainedEntryPosition = 1 + val EntriesAfterCursor = 2 + val EntryPositionFraction = 3 + val PublishTimeLag = 4 + val PublishTimeFraction = 5 + val FirstConsumedPublished = 6 + val FirstConsumedId = 7 + val LastConsumedPublished = 8 + val LastConsumedId = 9 + val OldestRetainedPublished = 10 + val OldestRetainedId = 11 + val NewestRetainedPublished = 12 + val NewestRetainedId = 13 + +/** The More tools bottom panel (toggled by `cs-tools`). */ final case class ToolsPanel(page: Page): val produceTab: Locator = page.getByTestId("console-tab-produce") val replTab: Locator = page.getByTestId("console-tab-repl") @@ -15,4 +33,41 @@ final case class ToolsPanel(page: Page): val replClear: Locator = page.getByTestId("cs-repl-clear") val replLogs: Locator = page.getByTestId("cs-repl-logs") + /** Replace the Context REPL expression in Monaco. */ + def writeRepl(code: String): Unit = + replEditor.locator(".monaco-editor").click() + page.keyboard().press("ControlOrMeta+A") + page.keyboard().press("Delete") + page.keyboard().`type`(code) + val logs: Locator = page.getByTestId("cs-logs") + + // --- Topic Positions (the per-topic debug view) --- + val topicPositionsTab: Locator = page.getByTestId("console-tab-topic-positions") + val topicPositionsTable: Locator = page.getByTestId("topic-positions-table") + val topicPositionsNotStarted: Locator = page.getByTestId("topic-positions-not-started") + val topicPositionsError: Locator = page.getByTestId("topic-positions-error") + + /** A row of the (shared-Table-backed) positions table, matched by its topic cell. */ + def topicPositionsRow(topicFqn: String): Locator = + page.locator("[data-testid='topic-positions'] tbody tr").filter(new Locator.FilterOptions().setHasText(topicFqn)) + + /** The row's cells, in DEFAULT header order (progress, processed bounds, stored endpoints): + * 0 topic/partition, 1 stored entry position (x / y), 2 stored entries after position, + * 3 stored entry position %, 4 publish-time gap, 5 publish-time position %, + * 6 earliest processed publish time, 7 earliest processed message id, + * 8 furthest processed publish time, 9 furthest processed message id, + * 10 earliest stored publish time, 11 earliest stored message id, + * 12 latest stored publish time, 13 latest stored message id. + */ + def topicPositionsCells(topicFqn: String): Vector[String] = + import scala.jdk.CollectionConverters.* + topicPositionsRow(topicFqn).locator("td").allTextContents().asScala.toVector + + /** One cell in the default order, kept as a locator so an assertion can wait for a poll to fill it. */ + def topicPositionsCell(topicFqn: String, column: Int): Locator = + topicPositionsRow(topicFqn).locator("td").nth(column) + + /** Click a sortable header of the positions table by its column key. */ + def topicPositionsSortBy(columnKey: String): Unit = + page.locator(s"[data-testid='topic-positions'] [data-testid='table-th'][data-column-key='$columnKey']").click() diff --git a/e2e/src/main/scala/features/library/LibrarySidebar.scala b/e2e/src/main/scala/features/library/LibrarySidebar.scala index d0fdb763a..bc187d9fc 100644 --- a/e2e/src/main/scala/features/library/LibrarySidebar.scala +++ b/e2e/src/main/scala/features/library/LibrarySidebar.scala @@ -24,8 +24,17 @@ final case class LibrarySidebar(page: Page): def openConsumerSessionsSubtab(): Unit = consumerSessionsSubtab.click() def openAllItemsSubtab(): Unit = allItemsSubtab.click() - /** From the Notes tab: create a note (first-note button when empty, else the "+" new-note button). */ + /** From the Notes tab: create a note (first-note button when empty, else the "+" new-note button). + * + * The panel renders a "Loading..." placeholder until its first `ListLibraryItems` resolves, so + * until then NEITHER button exists. `isVisible` does not wait, so branching on it while that + * fetch is still in flight takes the else-branch and then burns the whole timeout on + * `lib-new-note` - a button that can never appear for a topic with no notes. Waiting for either + * button first is the missing readiness precondition: it makes the branch read a settled panel + * rather than whichever render happened to be on screen. */ def createNote(): Unit = + createFirstNoteButton.or(newNoteButton).first() + .waitFor(new Locator.WaitForOptions().setTimeout(15000)) if createFirstNoteButton.isVisible then createFirstNoteButton.click() else newNoteButton.click() diff --git a/e2e/src/main/scala/harness/DekafInstance.scala b/e2e/src/main/scala/harness/DekafInstance.scala new file mode 100644 index 000000000..85936bed5 --- /dev/null +++ b/e2e/src/main/scala/harness/DekafInstance.scala @@ -0,0 +1,361 @@ +package harness + +import java.net.{ServerSocket, URI} +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.nio.file.{Files, Path, Paths} +import java.time.Duration +import java.util.concurrent.TimeUnit +import scala.jdk.CollectionConverters.* +import scala.util.Using + +/** A short-lived Dekaf server started with arbitrary `DEKAF_*` overrides, on a free port, with a + * data dir of its own - the affordance the CONFIGURATION specs need and that the shared stack + * cannot give them. + * + * Why it has to exist: a configuration property is only observable in a process that was STARTED + * with it. `scripts/run-dekaf.sh` brings up exactly one Dekaf with one fixed config, so every spec + * in this suite has so far exercised one point in the config space - and the two cookie-hardening + * bugs that shipped (`Secure` computed but never interpolated; `cookieSameSite` matched only + * lowercase, so `Lax` silently emitted NO attribute) were invisible to every tier precisely + * because nothing ran the env-var -> config-load -> behaviour path end to end. A test that hands + * the cookie writer its parameters directly cannot see either. + * + * The shape mirrors `run-dekaf.sh`: put the per-arch `envoy.bin` on PATH, point the server at the + * e2e Pulsar, seed a throwaway `DEKAF_DATA_DIR` with `js/` + `proto/` (a bare empty dir does not + * boot - `ConsumerSessionContext` reads `js/dist/libs.js` from it at startup), start, wait for + * GENUINE readiness, and always tear down. Two deliberate differences: + * + * - it runs the STAGED launcher (`server/target/universal/stage/bin/dekaf`) rather than + * `sbt run`. `sbt run` forks the app into a grandchild JVM, so the handle a test holds is + * sbt's, and killing it can leave the app alive holding the port - a leak that makes the NEXT + * instance fail for no visible reason. The staged script `exec`s java, so the handle IS the + * server and its Envoy child is a plain descendant. + * - the port is PROBED from a range well clear of the shared stack (:8090) and of :8080, rather + * than hardcoded, so a leftover instance cannot silently take over a later run. + * + * That staged build is produced on demand when it is MISSING, and otherwise REUSED - see + * `launcher` for why staging every run was worse than the staleness it prevents, and for the + * `DEKAF_SERVER_STAGE=force` escape hatch you want after editing `server/`. + */ +object DekafInstance: + + /** A running instance. `baseUrl` is where the app is actually SERVED (origin + basePath), which + * is not always `publicBaseUrl` - a reverse proxy that strips the prefix makes them differ. */ + final case class Instance(port: Int, basePath: String, dataDir: Path, log: Path): + def origin: String = s"http://localhost:$port" + def baseUrl: String = origin + basePath + + /** Absolute URL for an in-app path. Specs MUST use these: `DekafSuite`'s BrowserContext carries + * the shared stack's baseURL, so a relative `page.navigate("/overview")` would leave the + * instance under test entirely. */ + def url(path: String): String = baseUrl + path + + // --------------------------------------------------------------------------------------------- + // Locations + // --------------------------------------------------------------------------------------------- + + /** The `e2e/` project. sbt forks tests with the project base as the working directory; the second + * candidate keeps the harness usable from an IDE that chose the repo root instead. */ + private val e2eRoot: Path = + Seq(Paths.get("."), Paths.get("e2e")) + .map(_.toAbsolutePath.normalize) + .find(p => Files.isRegularFile(p.resolve("build.sbt")) && Files.isDirectory(p.resolve("src/test/scala"))) + .getOrElse(throw new IllegalStateException( + s"cannot locate the e2e project from ${Paths.get("").toAbsolutePath}" + )) + + private val repoRoot: Path = + val root = e2eRoot.getParent + if root == null || !Files.isRegularFile(root.resolve("server/build.sbt")) then + throw new IllegalStateException(s"cannot locate the repo root (server/build.sbt) above $e2eRoot") + root + + /** Everything this helper writes: one log per instance (kept - it is the only diagnosis when a + * server fails to boot) and the throwaway data dirs (removed, see `withInstance`). Under + * `e2e/target/`, which is gitignored, same as `target/traces`. */ + private val workDir: Path = e2eRoot.resolve("target/config-instances") + + /** The directory holding the per-arch `envoy.bin`, i.e. what `bin/get-bin-dir.js` prints. The + * embedded Envoy is what actually listens on `DEKAF_PORT`, so without this nothing serves. */ + private val envoyBinDir: Path = + val os = System.getProperty("os.name", "").toLowerCase + val platform = + if os.contains("mac") || os.contains("darwin") then "darwin" + else if os.contains("win") then "win32" + else "linux" + val arch = System.getProperty("os.arch", "").toLowerCase match + case "aarch64" | "arm64" => "arm64" + case "x86_64" | "amd64" | "x64" => "x64" + case other => other + val dir = repoRoot.resolve(s"bin/$platform/$arch") + if !Files.isDirectory(dir) then + throw new IllegalStateException(s"no binary dependencies for $platform/$arch: $dir does not exist") + dir + + // --------------------------------------------------------------------------------------------- + // The staged launcher + // --------------------------------------------------------------------------------------------- + + private def stagedLauncher: Path = repoRoot.resolve("server/target/universal/stage/bin/dekaf") + + /** Resolved ONCE per test JVM. `lazy val` is the memoization: only a run that actually starts an + * instance pays for it, and a suite that starts twelve pays once. + * + * Staging runs when the launcher is MISSING, or on `DEKAF_SERVER_STAGE=force`. It is not the + * default because the sbt that would do it is a second sbt sharing this one's `~/.ivy2` locks, + * and that contention is real: a `sbt stage` launched from inside a running e2e run intermittently + * exits 1 with no diagnostic at all, right after loading the meta-build. Hence the retries below, + * and hence the fall back to an existing launcher rather than failing the whole lane over it. + * + * The cost of not staging every time is honest and worth naming: after editing `server/`, these + * specs test the LAST STAGED build unless you re-stage. `DEKAF_SERVER_STAGE=force` (or + * `cd server && sbt stage`) is the fix; `e2e/README.md` §6 says so. */ + private lazy val launcher: Path = + val mode = sys.env.get("DEKAF_SERVER_STAGE").map(_.trim.toLowerCase).getOrElse("") + val haveLauncher = Files.isExecutable(stagedLauncher) + if haveLauncher && mode != "force" then + println(s"[DekafInstance] using the staged server at $stagedLauncher (DEKAF_SERVER_STAGE=force to re-stage)") + stagedLauncher + else + Files.createDirectories(workDir) + var failures = List.empty[String] + var staged = false + var attempt = 1 + while !staged && attempt <= 3 do + stageOnce(attempt) match + case None => staged = true + case Some(err) => failures = failures :+ err + attempt += 1 + if staged then stagedLauncher + else if haveLauncher then + System.err.println( + s"[DekafInstance] WARNING: could not re-stage the server; falling back to the EXISTING build at " + + s"$stagedLauncher, which may predate your server/ changes.\n ${failures.mkString("\n ")}" + ) + stagedLauncher + else + throw new IllegalStateException( + s"could not stage the server, and there is no build at $stagedLauncher to fall back to. These " + + "specs start their own Dekaf and need it. Stage it directly - `cd server && sbt stage` - inside " + + s"the nix shell, then re-run.\n ${failures.mkString("\n ")}" + ) + + /** One `sbt stage` attempt: `None` on success, `Some(diagnosis)` on failure. */ + private def stageOnce(attempt: Int): Option[String] = + val log = workDir.resolve(s"sbt-stage-$attempt.log") + println(s"[DekafInstance] staging the server (attempt $attempt: cd server && sbt stage) -> $log") + val pb = new ProcessBuilder("sbt", "-batch", "stage") + pb.directory(repoRoot.resolve("server").toFile) + pb.redirectErrorStream(true) + pb.redirectOutput(log.toFile) + try + val process = pb.start() + if !process.waitFor(20, TimeUnit.MINUTES) then + process.destroyForcibly() + Some(s"attempt $attempt: timed out after 20 minutes; see $log") + else if process.exitValue() != 0 then + Some(s"attempt $attempt: exit ${process.exitValue()}; tail of $log:\n${tail(log, 15)}") + else if !Files.isExecutable(stagedLauncher) then + Some(s"attempt $attempt: exit 0 but $stagedLauncher is missing; see $log") + else None + catch case e: java.io.IOException => Some(s"attempt $attempt: cannot run `sbt` (${e.getMessage})") + + // --------------------------------------------------------------------------------------------- + // Ports + // --------------------------------------------------------------------------------------------- + + /** Deliberately clear of :8080 (an unrelated service is often there) and of :8090 (the shared + * stack this suite otherwise runs against). Probed rather than hardcoded so a leftover instance + * cannot make a later run mysterious. */ + private val portRange: Vector[Int] = (8700 to 8799).toVector + + /** A port is free only if nothing ANSWERS on it and it can still be bound. + * + * The bind check alone is not enough, and quietly handed out an occupied port: Java's + * `ServerSocket` sets `SO_REUSEADDR`, so binding the wildcard address succeeds on BSD/macOS even + * while another process holds the same port on `127.0.0.1`. Envoy then binds the wildcard too, + * loopback traffic goes to the more specific bind - i.e. to the other program - and the test + * fails against a server that is not Dekaf. (Seen for real: an unrelated local service on + * 127.0.0.1:8765 answered the readiness poll with its own 401.) So connect first, on both + * loopback families, and only trust a refusal. */ + private def isFree(port: Int): Boolean = + def answers(host: String): Boolean = + val socket = new java.net.Socket() + try + socket.connect(new java.net.InetSocketAddress(host, port), 250) + true + catch case _: Throwable => false + finally try socket.close() catch case _: Throwable => () + if answers("127.0.0.1") || answers("::1") then false + else + try Using.resource(new ServerSocket(port))(_ => true) + catch case _: Throwable => false + + private def freePort(): Int = + // A random start spreads two runs on one box across the range instead of racing on 8700. + val offset = scala.util.Random.nextInt(portRange.size) + portRange.indices.iterator + .map(i => portRange((offset + i) % portRange.size)) + .find(isFree) + .getOrElse(throw new IllegalStateException( + s"no free port in ${portRange.head}..${portRange.last} - something is holding the whole range" + )) + + // --------------------------------------------------------------------------------------------- + // Data dirs + // --------------------------------------------------------------------------------------------- + + private def safe(label: String): String = label.replaceAll("[^A-Za-z0-9._-]", "-") + + private def copyTree(from: Path, to: Path): Unit = + Using.resource(Files.walk(from)) { walk => + walk.iterator().asScala.foreach { src => + val dst = to.resolve(from.relativize(src).toString) + if Files.isDirectory(src) then Files.createDirectories(dst) + else + Files.createDirectories(dst.getParent) + Files.copy(src, dst) + } + } + + /** A seeded, EMPTY-library data dir. The caller owns it (see `withDataDir`); `withInstance` makes + * and removes one per instance, which is what keeps Library state from bleeding between specs. + * + * A bare empty dir does not boot: the server reads `js/dist/libs.js` and `proto/` out of it at + * startup. Same seeding as `run-dekaf.sh`'s `DEKAF_FRESH_DATA=1` block. */ + def newDataDir(label: String): Path = + Files.createDirectories(workDir) + val dir = Files.createTempDirectory(workDir, s"${safe(label)}-data-") + Files.createDirectories(dir.resolve("library")) + copyTree(repoRoot.resolve("server/data/js"), dir.resolve("js")) + copyTree(repoRoot.resolve("server/data/proto"), dir.resolve("proto")) + dir + + def deleteTree(dir: Path): Unit = + if Files.exists(dir) then + Using.resource(Files.walk(dir)) { walk => + walk.sorted(java.util.Comparator.reverseOrder()).iterator().asScala.foreach(Files.deleteIfExists) + } + + /** A data dir that OUTLIVES a single instance - for asserting that two instances started on the + * same `dataDir` share state while two on different ones do not. Always removed. */ + def withDataDir[T](label: String)(body: Path => T): T = + val dir = newDataDir(label) + try body(dir) + finally deleteTree(dir) + + // --------------------------------------------------------------------------------------------- + // Start / stop + // --------------------------------------------------------------------------------------------- + + /** Pinned to HTTP/1.1 deliberately. The JDK client defaults to HTTP/2, which on cleartext means an + * `Upgrade: h2c` probe on the first request to each new origin - and Jetty behind Envoy answers + * that with `400 Invalid Upgrade header`. Every instance here is a NEW origin (a new port), so + * the readiness poll would intermittently see a 400 that says nothing about readiness and burn + * the whole timeout on a server that was already up. */ + private val http: HttpClient = + HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).connectTimeout(Duration.ofSeconds(2)).build() + + private def get(url: String): Option[(Int, String)] = + try + val request = HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(5)).GET().build() + val response = http.send(request, HttpResponse.BodyHandlers.ofString()) + Some(response.statusCode -> response.body) + catch case _: Throwable => None + + private def tail(file: Path, lines: Int): String = + try Files.readString(file).linesIterator.toList.takeRight(lines).mkString("\n") + catch case _: Throwable => s"(no log at $file)" + + /** Poll a REAL precondition, never a sleep: `/health` answers 200 "ok" only once the gRPC server + * is up (`GrpcServer.isRunning`) AND the request survived the round trip through the embedded + * Envoy that fronts it - which is also the only thing in this suite that exercises Envoy's + * `basePath` routing. A dead process fails immediately rather than burning the timeout. */ + private def awaitReady(instance: Instance, process: Process, timeoutMs: Long): Unit = + val deadline = System.currentTimeMillis() + timeoutMs + var lastSeen = "no response yet" + while System.currentTimeMillis() < deadline do + if !process.isAlive then + throw new IllegalStateException( + s"Dekaf on :${instance.port} exited with code ${process.exitValue()} before becoming ready.\n" + + s"--- ${instance.log} (tail) ---\n${tail(instance.log, 40)}" + ) + get(instance.url("/health")) match + case Some((200, body)) if body.trim == "ok" => return + case Some((code, body)) => lastSeen = s"HTTP $code: ${body.take(200)}" + case None => lastSeen = "no connection" + Thread.sleep(200) + throw new IllegalStateException( + s"Dekaf on :${instance.port} did not become ready within ${timeoutMs}ms (last: $lastSeen).\n" + + s"--- ${instance.log} (tail) ---\n${tail(instance.log, 40)}" + ) + + /** Kill the server AND its Envoy child. Envoy is a plain descendant of the JVM, so it is + * snapshotted before the parent dies (a dead parent has no descendants to enumerate) and swept + * afterwards; a surviving Envoy would hold the port and make the next instance fail for a reason + * that has nothing to do with the test that failed. */ + private def stop(instance: Instance, process: Process): Unit = + val children = process.descendants().iterator().asScala.toList + process.destroy() + if !process.waitFor(20, TimeUnit.SECONDS) then + process.destroyForcibly() + process.waitFor(10, TimeUnit.SECONDS) + children.filter(_.isAlive).foreach(_.destroy()) + val deadline = System.currentTimeMillis() + 10000 + while children.exists(_.isAlive) && System.currentTimeMillis() < deadline do Thread.sleep(100) + children.filter(_.isAlive).foreach(_.destroyForcibly()) + // The port must actually come back, or the next instance inherits a mystery. + val portDeadline = System.currentTimeMillis() + 15000 + while !isFree(instance.port) && System.currentTimeMillis() < portDeadline do Thread.sleep(100) + if !isFree(instance.port) then + System.err.println(s"[DekafInstance] WARNING: :${instance.port} is still held after stopping ${instance.log}") + + /** Start a Dekaf with `overrides` on a free port and a fresh data dir, hand it to `body`, and + * ALWAYS tear it down - the data dir included. */ + def withInstance[T](label: String, overrides: (String, String)*)(body: Instance => T): T = + val dataDir = newDataDir(label) + try withInstanceOn(label, dataDir, overrides*)(body) + finally deleteTree(dataDir) + + /** As `withInstance`, but on a data dir the CALLER owns (and must remove). + * + * The port is only known once it has been probed, so an override that has to REFER to it - a + * `publicBaseUrl` that differs from where the app is served, which is what a prefix-stripping + * reverse proxy looks like - writes `{origin}` and gets `http://localhost:`. */ + def withInstanceOn[T](label: String, dataDir: Path, overrides: (String, String)*)(body: Instance => T): T = + val launcherPath = launcher // stage before claiming a port: staging can take minutes + val port = freePort() + val requested = overrides.toMap.view.mapValues(_.replace("{origin}", s"http://localhost:$port")).toMap + // `normalizeConfig` strips the trailing slash, so the default "/" is served at the origin root. + val basePath = requested.getOrElse("DEKAF_BASE_PATH", "/").stripSuffix("/") + Files.createDirectories(workDir) + val log = workDir.resolve(s"${safe(label)}-$port.log") + val instance = Instance(port = port, basePath = basePath, dataDir = dataDir, log = log) + + val env = Map( + "DEKAF_PORT" -> port.toString, + "DEKAF_PUBLIC_BASE_URL" -> instance.baseUrl, + "DEKAF_PULSAR_WEB_URL" -> Config.pulsarAdminUrl, + "DEKAF_PULSAR_BROKER_URL" -> Config.pulsarServiceUrl, + "DEKAF_DATA_DIR" -> dataDir.toString + ) ++ requested + + val pb = new ProcessBuilder(launcherPath.toString) + // cwd = server/: this is not a binary build, so Javalin serves the UI bundle from the RELATIVE + // `src/main/resources/ui/static`, exactly as `run-dekaf.sh`'s `sbt run` does. + pb.directory(repoRoot.resolve("server").toFile) + pb.redirectErrorStream(true) + pb.redirectOutput(log.toFile) + pb.environment().put("PATH", s"${sys.env.getOrElse("PATH", "")}:${envoyBinDir.toString}") + env.foreach((k, v) => pb.environment().put(k, v)) + + val process = pb.start() + try + awaitReady(instance, process, timeoutMs = 120000) + body(instance) + catch + case t: Throwable => + // The server log is the only place a config-rejection or a boot failure is visible. + System.err.println(s"[DekafInstance] $label on :$port failed: ${t.getMessage}\n${tail(log, 25)}") + throw t + finally stop(instance, process) diff --git a/e2e/src/main/scala/harness/PulsarFixtures.scala b/e2e/src/main/scala/harness/PulsarFixtures.scala index 66661ee32..c691e54c0 100644 --- a/e2e/src/main/scala/harness/PulsarFixtures.scala +++ b/e2e/src/main/scala/harness/PulsarFixtures.scala @@ -2,9 +2,10 @@ package harness import net.datafaker.Faker import org.apache.pulsar.client.admin.PulsarAdmin -import org.apache.pulsar.client.api.{PulsarClient, Schema} +import org.apache.pulsar.client.api.{Message as PulsarMessage, MessageId as PulsarMessageId, PulsarClient, Schema} import org.apache.pulsar.common.policies.data.{ClusterData, ResourceGroup, TenantInfo} +import java.util.concurrent.TimeUnit import scala.collection.mutable import scala.jdk.CollectionConverters.* @@ -99,6 +100,44 @@ class PulsarFixtures: admin.topics().createNonPartitionedTopic(fqn) fqn + /** One quadrant of the topic matrix the consumer session must handle. + * + * NOTE ON NON-PERSISTENT: a non-persistent topic keeps nothing on disk - messages published + * while no consumer is attached are dropped forever. So the "pre-produce, then start from + * Earliest" shape is meaningless there; only produce-AFTER-play can be asserted. `retains` + * encodes that so specs can branch on capability instead of hard-coding topic names. + */ + case class TopicKind(persistent: Boolean, partitions: Int): + def scheme: String = if persistent then "persistent" else "non-persistent" + def isPartitioned: Boolean = partitions > 0 + def retains: Boolean = persistent + def label: String = + s"${if persistent then "persistent" else "non-persistent"}/${if isPartitioned then s"partitioned($partitions)" else "non-partitioned"}" + + object TopicKind: + val PersistentNonPartitioned = TopicKind(persistent = true, partitions = 0) + val PersistentPartitioned = TopicKind(persistent = true, partitions = 3) + val NonPersistentNonPartitioned = TopicKind(persistent = false, partitions = 0) + val NonPersistentPartitioned = TopicKind(persistent = false, partitions = 3) + /** The full matrix the consumer session is expected to support. */ + val all: List[TopicKind] = + List(PersistentNonPartitioned, PersistentPartitioned, NonPersistentNonPartitioned, NonPersistentPartitioned) + + /** Create a topic of the given kind in an existing namespace; returns its FQN. */ + def createTopicOfKind(tenant: String, namespace: String, kind: TopicKind): String = + val topic = unique("topic") + val fqn = s"${kind.scheme}://$tenant/$namespace/$topic" + if kind.isPartitioned then admin.topics().createPartitionedTopic(fqn, kind.partitions) + else admin.topics().createNonPartitionedTopic(fqn) + fqn + + /** Fresh tenant → namespace → topic of the given kind; returns (tenant, namespace, shortTopic, fqn). */ + def freshTopicPartsOfKind(kind: TopicKind): (String, String, String, String) = + val t = createTenant() + val ns = createNamespace(t) + val fqn = createTopicOfKind(t, ns, kind) + (t, ns, fqn.substring(fqn.lastIndexOf('/') + 1), fqn) + /** Convenience: fresh tenant → namespace → topic, returns the topic FQN. */ def freshTopic(): String = val t = createTenant() @@ -113,11 +152,444 @@ class PulsarFixtures: admin.topics().createNonPartitionedTopic(s"persistent://$t/$ns/$topic") (t, ns, topic) - /** Produce `n` simple string messages to a topic FQN. */ - def produceStrings(topicFqn: String, n: Int): Unit = + /** Produce `n` simple string messages to a topic FQN, and return exactly what was produced. + * + * `prefix` is what makes a payload an IDENTITY rather than scenery. The default `msg` repeats + * across every topic and every call, so `msg-7` on topic A, `msg-7` on topic B and a redelivered + * `msg-7` are one indistinguishable string: an exact-set oracle built on them cannot tell a + * message that came from the wrong stream, or arrived twice, from the right one - only a COUNT + * can be asserted, and a count is satisfied by one loss plus one duplicate. Give each topic (and + * each produce phase) its own prefix and the set becomes assertable. */ + def produceStrings(topicFqn: String, n: Int, prefix: String = "msg"): Vector[String] = + val values = (1 to n).map(i => s"$prefix-$i").toVector val producer = client.newProducer(Schema.STRING).topic(topicFqn).create() - try (1 to n).foreach(i => producer.send(s"msg-$i")) + try values.foreach(producer.send) finally producer.close() + values + + /** Produce `values` one at a time, ROUND-ROBIN across `topicFqns`, batching disabled, with a + * clock tick between sends: publish time is millisecond-granular and stamped by the producer, + * so two sends inside one tick would TIE and fall back to the topic-name tie-break - which + * need not match production order. The tick wait is what makes publish times strictly + * increasing across the WHOLE set, which is what the cross-topic ordering oracles rest on. */ + def produceRoundRobin(topicFqns: Seq[String], values: Seq[String]): Unit = + val producers = topicFqns.map(fqn => client.newProducer(Schema.STRING).topic(fqn).enableBatching(false).create()) + try + values.zipWithIndex.foreach { (v, i) => + producers(i % producers.size).send(v) + Thread.sleep(2) + } + finally producers.foreach(p => scala.util.Try(p.close())) + + /** Bulk produce with ASYNC sends (batched entries, ~100x faster than the blocking loop) - for + * tests that need tens of thousands of messages as scenery, not as per-entry fixtures. Returns + * exactly what was produced; see [[produceStrings]] for why `prefix` matters. */ + def produceStringsFast(topicFqn: String, n: Int, prefix: String = "msg"): Vector[String] = + val values = (1 to n).map(i => s"$prefix-$i").toVector + val producer = client.newProducer(Schema.STRING).topic(topicFqn).create() + try + values.foreach(producer.sendAsync) + producer.flush() + finally producer.close() + values + + /** Bulk produce like `produceStringsFast`, but UNBATCHED so the default router rotates PER + * MESSAGE: on a partitioned topic every partition receives an equal share. With batching on, + * the router rotates per BATCH, and a pipelined bulk produce lands in so few batches that most + * partitions of a WIDE topic get nothing at all - useless for tests about width. Unbatched + * async sends fill the producer's pending queue, so this blocks on it rather than throw. */ + def produceStringsFastRoundRobin(topicFqn: String, n: Int): Unit = + val producer = client.newProducer(Schema.STRING).topic(topicFqn).enableBatching(false).blockIfQueueFull(true).create() + try + (1 to n).foreach(i => producer.sendAsync(s"msg-$i")) + producer.flush() + finally producer.close() + + /** Force-delete a topic out from under its consumers: everything it held, recorded ends + * included, silently stops being deliverable - the retention/trim race the start-from give-up + * window exists for, made absolute and reproducible. (Auto-creation may resurrect the NAME as + * an empty topic; the old ledger never comes back, which is the point.) */ + def forceDeleteTopic(topicFqn: String): Unit = + admin.topics.delete(topicFqn, true) + + // --------------------------------------------------------------------------------------------- + // Batching + // + // A Pulsar broker addresses its log by ENTRY, not by message, and the Java producer batches by + // default - so an ordinary application writes many messages per entry. Everything the suite + // produced before this section went out one-message-per-entry (a blocking `send` per message + // closes each batch immediately), which meant no test could ever observe an entry-vs-message + // confusion. `PulsarAdmin.examineMessage` - what the "skip first n" / "latest n" start-from modes + // are built on - counts ENTRIES, so that gap hid a real defect. These helpers make both shapes + // explicit and provable. + // --------------------------------------------------------------------------------------------- + + /** Broker-side entry count of a NON-partitioned topic (a `-partition-K` FQN counts as one). + * This is the number `examineMessage` indexes into, so `numberOfEntries < messages produced` is + * exactly the condition under which entry-addressing and message-addressing diverge. + * Persistent topics only - a non-persistent topic has no managed ledger and 405s. */ + def numberOfEntries(nonPartitionedTopicFqn: String): Long = + admin.topics().getInternalStats(nonPartitionedTopicFqn).numberOfEntries + + /** How many partitions a topic has, or 0 when it is not partitioned. */ + def partitionCount(topicFqn: String): Int = + admin.topics().getPartitionedTopicMetadata(topicFqn).partitions + + /** Total broker entries behind a topic FQN, summed across partitions when it names a partitioned + * topic - the parent of a partitioned topic has no managed ledger of its own, so asking it for + * internal stats 404s. Partitions that are not materialized yet contribute nothing. */ + def totalEntries(topicFqn: String): Long = + partitionCount(topicFqn) match + case 0 => numberOfEntries(topicFqn) + case n => + (0 until n).map { i => + try numberOfEntries(s"$topicFqn-partition-$i") + catch case _: Throwable => 0L + }.sum + + /** The broker entries added to `topicFqn` since `before`, once the admin counter has SETTLED - two + * reads a short quiescence gap apart that agree. + * + * Every producer ack is already held (and flushed) when this is called, so the writes are durable + * and the counter can only LAG, never grow anew: this is not readiness polling, it is measuring + * the admin counter catch up. The gap matters on a PARTITIONED topic, where the entries land + * across partitions and the counter climbs in STEPS - a single non-zero reading can catch it + * mid-climb and UNDERCOUNT, and an undercount sits below `values.size` even for genuinely + * unbatched output, which is exactly the fixture the batching guard exists to reject. */ + private def settledEntryDelta(topicFqn: String, before: Long): Long = + Eventually.eventually(timeoutMs = 15000, intervalMs = 250) { + val d1 = totalEntries(topicFqn) - before + assert(d1 >= 1L, s"no entries visible yet on $topicFqn") + Thread.sleep(400) // quiescence gap, NOT a readiness wait: every ack is held, so the counter only lags + val d2 = totalEntries(topicFqn) - before + assert(d1 == d2, s"the broker entry counter is still rising on $topicFqn: $d1 then $d2 - reading it mid-climb would undercount") + d2 + } + + /** Produce `values` with batching DISABLED: one message per broker entry, guaranteed. + * The baseline half of every batched/unbatched pair. */ + def produceUnbatched(topicFqn: String, values: Seq[String]): Unit = + val producer = client.newProducer(Schema.STRING).topic(topicFqn).enableBatching(false).create() + try values.foreach(producer.send) + finally producer.close() + + /** Produce `values` so that consecutive groups of `messagesPerBatch` genuinely SHARE one broker + * entry - what the Java client does by default in any real application. + * + * The three settings are all load-bearing: `enableBatching` alone changes nothing if each + * message is sent with a blocking `send` (that flushes a one-message batch every time), so the + * sends must be async and only then flushed; and the publish delay has to be long enough that + * `batchingMaxMessages` - not a timer - is what closes a batch, otherwise a slow box silently + * degrades to singletons. + * + * On a persistent topic the outcome is VERIFIED against `numberOfEntries` and an unbatched + * result THROWS. That guard is the point: a "batched" fixture that quietly produced one-message + * entries would recreate the exact blind spot batched coverage exists to close, and every test + * built on it would keep passing while proving nothing. */ + def produceBatched(topicFqn: String, values: Seq[String], messagesPerBatch: Int): Unit = + require(messagesPerBatch >= 2, s"messagesPerBatch must be >= 2 to batch anything, got $messagesPerBatch") + val verifiable = topicFqn.startsWith("persistent://") // a non-persistent topic has no ledger to count + val isPartitioned = verifiable && partitionCount(topicFqn) > 0 + val before = if verifiable then totalEntries(topicFqn) else -1L + val producer = client + .newProducer(Schema.STRING) + .topic(topicFqn) + .enableBatching(true) + .batchingMaxMessages(messagesPerBatch) + .batchingMaxBytes(4 * 1024 * 1024) // never the binding limit for these short payloads + .batchingMaxPublishDelay(60, TimeUnit.SECONDS) // never the binding limit either: size or flush closes a batch + .create() + try + val acks = values.map(v => producer.sendAsync(v)) + producer.flush() // closes the trailing partial batch + acks.foreach(_.get(60, TimeUnit.SECONDS)) // and every send really landed + finally producer.close() + if verifiable then + val expected = math.ceil(values.size.toDouble / messagesPerBatch).toLong + // The managed-ledger counter is read back through the admin API, so allow it a moment to + // reflect the writes we already hold producer acks for. On a single log the final count is + // known (`expected`), so waiting for exactly that and no more is both sufficient and precise. + // On a PARTITIONED topic the final count is router-dependent and unknown, so the counter must + // be read once it has SETTLED - a `>= 1` reading can catch it mid-climb and undercount, and an + // undercount would pass the batching guard below on genuinely unbatched output. + val added = + if isPartitioned then settledEntryDelta(topicFqn, before) + else + Eventually.eventually(timeoutMs = 10000, intervalMs = 200) { + val d = totalEntries(topicFqn) - before + assert(d >= expected, s"only $d entries visible yet for ${values.size} messages on $topicFqn") + d + } + // THE guard: if the messages did not actually share entries there is no point running any + // batched test on top, because it would be indistinguishable from the unbatched one. + assert( + added < values.size, + s"produceBatched did NOT batch: ${values.size} messages at $messagesPerBatch per batch became " + + s"$added broker entries on $topicFqn - one per message. An unbatched 'batched' fixture proves " + + "nothing; fix the producer settings, do not relax this." + ) + // On a single log the split is fully determined. A partitioned topic batches per partition and + // the router decides where each batch lands, so only the inequality above is guaranteed there. + if !isPartitioned then + assert(added == expected, s"expected exactly $expected entries for ${values.size} messages at $messagesPerBatch per batch, got $added") + + /** Produce one broker entry for each explicitly supplied batch, preserving deliberately uneven + * batch sizes. Restricted to persistent non-partitioned topics because only there is the exact + * entry delta deterministic and directly verifiable. + * + * Flushing after every group is load-bearing: it closes a short group before the next one can + * join it. The final entry-count assertion makes a changed client batching policy fail at the + * fixture boundary instead of weakening every entry-position test built on this arrangement. */ + def produceBatches(topicFqn: String, batches: Seq[Seq[String]]): Unit = + require(batches.nonEmpty, "batches must not be empty") + require(batches.forall(_.nonEmpty), "each explicit batch must contain at least one message") + require(topicFqn.startsWith("persistent://"), s"explicit batches require a persistent topic: $topicFqn") + require(partitionCount(topicFqn) == 0, s"explicit batches require a non-partitioned topic: $topicFqn") + + val before = numberOfEntries(topicFqn) + val producer = client + .newProducer(Schema.STRING) + .topic(topicFqn) + .enableBatching(true) + .batchingMaxMessages(batches.map(_.size).max) + .batchingMaxBytes(4 * 1024 * 1024) + .batchingMaxPublishDelay(60, TimeUnit.SECONDS) + .create() + try + batches.foreach { batch => + val acks = batch.map(value => producer.sendAsync(value)) + producer.flush() + acks.foreach(_.get(60, TimeUnit.SECONDS)) + } + finally producer.close() + + val expected = batches.size.toLong + Eventually.eventually(timeoutMs = 10000, intervalMs = 200) { + val added = numberOfEntries(topicFqn) - before + assert(added == expected, s"expected $expected explicit batches to add $expected entries on $topicFqn, got $added") + } + + // --------------------------------------------------------------------------------------------- + // Blind-spot payload shapes: ties, chunking, sized payloads + // + // The batching section above exists because every fixture used to produce the ONE shape in which + // a real defect class was invisible. These three helpers close the remaining shapes the same way, + // and follow the same rule: each ASSERTS its own premise against the broker and throws when the + // data did not land in the shape its tests need - a fixture that silently degrades recreates the + // blind spot it exists to close. `harness.MessageShapeFixtureSpec` pins the broker facts under + // each (pure broker specs - they run without a Dekaf instance). + // --------------------------------------------------------------------------------------------- + + /** Produce batch pairs to TWO topics until at least one attempt's batches genuinely SHARE a + * publish timestamp across the topics, and return everything produced as (valuesA, valuesB). + * + * WHY: the Java producer batches by default and every message in a batch carries the batch + * container's single publish time - so in production, publish-time ties are the norm, both + * within a stream and across streams. Every other ordering fixture here tick-separates its + * sends precisely so ties can NEVER occur, which leaves the merge's tie-break (and the + * Guaranteed barrier's behavior on tied heads) unreachable. + * + * Each attempt interleaves async sends message-by-message so the two batches fill in lockstep + * and close within the same producer-clock millisecond in the common case; whether they REALLY + * tied is then read back from the broker, and more pairs are appended until one did. Premises + * asserted before returning, everything from the broker: + * - each attempt added exactly ONE entry per topic (the batches really batched); + * - every batch's messages share exactly one publish time; + * - at least one attempt's A-batch and B-batch share the SAME publish time (the cross-stream + * tie), or this THROWS after `maxAttempts` rather than let a tie-free arrangement pass for + * a tied one. */ + def produceTiedBatches(fqnA: String, fqnB: String, messagesPerBatch: Int, maxAttempts: Int = 8): (Vector[String], Vector[String]) = + require(messagesPerBatch >= 2, s"messagesPerBatch must be >= 2 to batch anything, got $messagesPerBatch") + Seq(fqnA, fqnB).foreach { fqn => + require(fqn.startsWith("persistent://"), s"tied batches need a verifiable ledger: $fqn") + require(partitionCount(fqn) == 0, s"tied batches need a single log per topic: $fqn") + } + def newBatchingProducer(fqn: String) = client + .newProducer(Schema.STRING) + .topic(fqn) + .enableBatching(true) + .batchingMaxMessages(messagesPerBatch) + .batchingMaxBytes(4 * 1024 * 1024) + .batchingMaxPublishDelay(60, TimeUnit.SECONDS) + .create() + val beforeA = numberOfEntries(fqnA) + val beforeB = numberOfEntries(fqnB) + val sentA = Vector.newBuilder[String] + val sentB = Vector.newBuilder[String] + var attempts = 0 + var tied = false + val producerA = newBatchingProducer(fqnA) + val producerB = newBatchingProducer(fqnB) + try + while !tied && attempts < maxAttempts do + attempts += 1 + val va = (1 to messagesPerBatch).map(i => s"tie-a$attempts-$i") + val vb = (1 to messagesPerBatch).map(i => s"tie-b$attempts-$i") + // Interleaved per MESSAGE: both batches close (size-triggered) on back-to-back calls. + val acks = (0 until messagesPerBatch).flatMap(i => Seq(producerA.sendAsync(va(i)), producerB.sendAsync(vb(i)))) + producerA.flush() + producerB.flush() + acks.foreach(_.get(60, TimeUnit.SECONDS)) + sentA ++= va + sentB ++= vb + // Did THIS attempt's two batches land on one shared millisecond? (An accidental tie + // between different attempts would not put tied heads in front of the merge reliably.) + val timesA = readAllMessages(fqnA).map(_.getPublishTime) + val timesB = readAllMessages(fqnB).map(_.getPublishTime) + tied = timesA.takeRight(messagesPerBatch).toSet.intersect(timesB.takeRight(messagesPerBatch).toSet).nonEmpty + finally + scala.util.Try(producerA.close()) + scala.util.Try(producerB.close()) + assert( + tied, + s"produceTiedBatches could not manufacture a cross-topic publish-time tie in $attempts attempt(s) - " + + "a ties test running on tie-free data proves nothing; raise maxAttempts, do not relax this." + ) + // The batching guard, exact: one entry per attempt per topic. + Eventually.eventually(timeoutMs = 15000, intervalMs = 250) { + val addedA = numberOfEntries(fqnA) - beforeA + val addedB = numberOfEntries(fqnB) - beforeB + assert( + addedA == attempts && addedB == attempts, + s"tied batches did NOT batch: $attempts attempt(s) became $addedA entries on $fqnA and $addedB on $fqnB - expected exactly one each" + ) + } + // Within-batch premise: a batch is ONE publish time, for every batch produced. + Seq(fqnA -> sentA.result(), fqnB -> sentB.result()).foreach { case (fqn, sent) => + val read = readAllMessages(fqn) + assert(read.map(_.getValue) == sent, s"read-back of $fqn does not match produce order: ${read.map(_.getValue)}") + read.grouped(messagesPerBatch).foreach { batch => + assert( + batch.map(_.getPublishTime).distinct.size == 1, + s"a batch on $fqn does not share one publish time: ${batch.map(m => m.getValue -> m.getPublishTime)}" + ) + } + } + (sentA.result(), sentB.result()) + + /** Produce ONE message whose payload the client splits across several broker entries - Pulsar + * CHUNKING, the exact dual of batching (batching: many messages, one entry; chunking: one + * message, many entries). Returns the entry (chunk) count. + * + * The chunk size is pinned SMALL so the payload stays test-sized instead of having to exceed + * the broker's 5 MB frame; chunking requires batching off. Persistent non-partitioned only, + * and the premise is ASSERTED: the send must have added exactly ceil(bytes / chunkBytes) > 1 + * entries. A fixture that quietly sent one entry would recreate the exact blind spot chunked + * coverage exists to close - an entry-addressed walk that miscounts chunked messages. */ + def produceChunked(nonPartitionedTopicFqn: String, value: String, chunkBytes: Int = 64 * 1024): Int = + require(nonPartitionedTopicFqn.startsWith("persistent://"), s"chunk verification needs a ledger: $nonPartitionedTopicFqn") + require(partitionCount(nonPartitionedTopicFqn) == 0, s"chunk verification needs a single log: $nonPartitionedTopicFqn") + val payloadBytes = value.getBytes(java.nio.charset.StandardCharsets.UTF_8).length + val expectedChunks = math.ceil(payloadBytes.toDouble / chunkBytes).toInt + require(expectedChunks >= 2, s"a $payloadBytes-byte value fits one $chunkBytes-byte chunk - nothing would be chunked") + val before = numberOfEntries(nonPartitionedTopicFqn) + val producer = client + .newProducer(Schema.STRING) + .topic(nonPartitionedTopicFqn) + .enableBatching(false) // chunking and batching are mutually exclusive on the producer + .enableChunking(true) + .chunkMaxMessageSize(chunkBytes) + .create() + try producer.send(value) + finally producer.close() + Eventually.eventually(timeoutMs = 15000, intervalMs = 250) { + val added = numberOfEntries(nonPartitionedTopicFqn) - before + assert( + added == expectedChunks, + s"produceChunked did NOT chunk as arranged: a $payloadBytes-byte message at $chunkBytes bytes/chunk " + + s"became $added entries on $nonPartitionedTopicFqn, expected $expectedChunks. An unchunked 'chunked' " + + "fixture proves nothing; fix the producer settings, do not relax this." + ) + } + expectedChunks + + /** Produce `count` messages of exactly `payloadBytes` ASCII bytes each (a short readable prefix, + * x-padded), UNBATCHED so each is its own entry, with async sends so hundreds of MiB do not pay + * a blocking round trip per message. Persistent non-partitioned only. Premises ASSERTED: + * exactly `count` entries were added, and the topic's storage grew by at least + * count x payloadBytes - a fixture that silently produced small (or batched) messages would + * leave the merge's held-BYTES watermark unreachable and any test built on it vacuous. */ + def produceSized(nonPartitionedTopicFqn: String, count: Int, payloadBytes: Int): Vector[String] = + require(nonPartitionedTopicFqn.startsWith("persistent://"), s"sized-payload verification needs a ledger: $nonPartitionedTopicFqn") + require(partitionCount(nonPartitionedTopicFqn) == 0, s"sized-payload verification needs a single log: $nonPartitionedTopicFqn") + require(payloadBytes >= 16, s"payloadBytes must fit the value prefix, got $payloadBytes") + val before = numberOfEntries(nonPartitionedTopicFqn) + val beforeStorage = admin.topics().getStats(nonPartitionedTopicFqn).getStorageSize + val values = (1 to count).map { i => + val prefix = f"big-$i%05d-" + prefix + "x" * (payloadBytes - prefix.length) + }.toVector + val producer = client + .newProducer(Schema.STRING) + .topic(nonPartitionedTopicFqn) + .enableBatching(false) + .blockIfQueueFull(true) // async sends throttle against the client memory limit instead of failing + .create() + try + val acks = values.map(producer.sendAsync) + producer.flush() + acks.foreach(_.get(120, TimeUnit.SECONDS)) + finally producer.close() + Eventually.eventually(timeoutMs = 30000, intervalMs = 500) { + val added = numberOfEntries(nonPartitionedTopicFqn) - before + assert(added == count, s"produceSized: expected $count unbatched entries on $nonPartitionedTopicFqn, got $added") + } + val storageAdded = admin.topics().getStats(nonPartitionedTopicFqn).getStorageSize - beforeStorage + assert( + storageAdded >= count.toLong * payloadBytes, + s"produceSized: the broker holds only $storageAdded bytes for $count x $payloadBytes-byte messages on " + + s"$nonPartitionedTopicFqn - the payloads did not land at the size the byte-watermark tests need" + ) + values + + /** Messages the broker has DISPATCHED to the single subscription of a non-partitioned topic. + * The topic must carry exactly ONE subscription (a fresh topic consumed only by the session + * under test), so the number is attributable without knowing Dekaf's internal subscription + * name. This is the oracle a flow-control pause leaves behind: a paused consumer stops asking + * for more, so the counter PLATEAUS while backlog remains. */ + def subscriptionDispatchCount(nonPartitionedTopicFqn: String): Long = + val subs = admin.topics().getStats(nonPartitionedTopicFqn).getSubscriptions.asScala + assert(subs.size == 1, s"expected exactly one subscription on $nonPartitionedTopicFqn, got ${subs.keys.toList}") + subs.values.head.getMsgOutCounter + + /** Register a JSON schema (an Avro-record JSON definition) on a topic through the admin API, + * and assert the registry really holds it - the premise any schema-decode test rests on. */ + def registerJsonSchema(topicFqn: String, avroRecordJson: String): Unit = + val payload = new org.apache.pulsar.common.protocol.schema.PostSchemaPayload( + "JSON", + avroRecordJson, + java.util.Collections.emptyMap[String, String]() + ) + admin.schemas().createSchema(topicFqn, payload) + Eventually.eventually(timeoutMs = 10000, intervalMs = 200) { + val stored = admin.schemas().getSchemaInfo(topicFqn) + assert( + stored != null && stored.getType == org.apache.pulsar.common.schema.SchemaType.JSON, + s"the registered JSON schema did not land on $topicFqn: $stored" + ) + } + + /** Every message currently retained on a NON-partitioned topic, oldest first, read with a + * non-durable Reader (no cursor left behind). The oracle for the id- and time-addressed + * start-from modes: only the broker knows the real message ids and publish times. */ + def readAllMessages(nonPartitionedTopicFqn: String): Vector[PulsarMessage[String]] = + val reader = client + .newReader(Schema.STRING) + .topic(nonPartitionedTopicFqn) + .startMessageId(PulsarMessageId.earliest) + .create() + try + val buf = Vector.newBuilder[PulsarMessage[String]] + while reader.hasMessageAvailable do + val m = reader.readNext(10, TimeUnit.SECONDS) + if m != null then buf += m + buf.result() + finally reader.close() + + /** A message id in the space-separated hex the Start-From "Message with specific ID" input takes + * (`hexStringToByteArray` in ui/components/conversions), e.g. "08 c3 03 10 cd 04 20 00 30 01". */ + def messageIdHex(messageId: PulsarMessageId): String = + messageId.toByteArray.map(b => f"${b & 0xff}%02x").mkString(" ") /** Best-effort teardown of everything this fixture created. Runs after each test. * Failures are logged (not thrown - teardown must not fail a test) so leaks aren't silent. diff --git a/e2e/src/test/scala/configuration/CfgBasePathSpec.scala b/e2e/src/test/scala/configuration/CfgBasePathSpec.scala new file mode 100644 index 000000000..5d63cb9b5 --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgBasePathSpec.scala @@ -0,0 +1,93 @@ +package configuration + +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.Response +import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import java.util.function.Consumer +import scala.jdk.CollectionConverters.* + +/** CFG-BP - `basePath`, the reverse-proxy setting that decides where the whole application lives. + * + * It is the highest-value property after the cookie attributes because it breaks SUBTLY: the + * server starts, `/health` answers, and the failure only shows up as an asset or a gRPC-web call + * resolved against the wrong root. It is also the only thing in this suite that exercises the + * embedded Envoy proxy's routing at all - `basePath` is interpolated straight into the Envoy + * config's two route prefixes and their regex rewrites, and no other test can vary it. + * + * So this test does not stop at "the page loaded": it drives a deep link, a gRPC-web round trip, + * an in-app route change, and the cookie scope, and it checks that the origin ROOT serves nothing - + * which is what distinguishes "the app moved to the sub-path" from "the app answers everywhere". + */ +class CfgBasePathSpec extends DekafSuite: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + + test("CFG-BP-1: under a sub-path basePath the app works there, and the origin root serves nothing") { + val subPath = "/dekaf-e2e" + DekafInstance.withInstance("basepath", "DEKAF_BASE_PATH" -> subPath) { instance => + val tenant = fixtures.createTenant() + val namespace = fixtures.createNamespace(tenant) + val topic = fixtures.unique("topic") + admin.topics().createNonPartitionedTopic(s"persistent://$tenant/$namespace/$topic") + + // 1) The app MOVED, it was not aliased: the same route at the origin root is not served. + val atRoot = page.navigate(s"${instance.origin}/tenants/$tenant/overview") + assert( + atRoot != null && atRoot.status() == 404, + s"the origin root answered ${Option(atRoot).map(_.status())} for a real app route; expected 404 with basePath=$subPath" + ) + + // Everything the page fetches from here on, so the assertions below are about what the browser + // REALLY requested rather than about what the DOM happens to say. + val responses = java.util.Collections.synchronizedList(new java.util.ArrayList[(Int, String)]()) + val record: Consumer[Response] = (response: Response) => responses.add(response.status() -> response.url()) + page.onResponse(record) + + // 2) A deep link under the sub-path loads ... + val deepLink = instance.url(s"/tenants/$tenant/namespaces/$namespace/topics/persistent/$topic/overview") + val loaded = page.navigate(deepLink) + assert(loaded != null && loaded.status() == 200, s"$deepLink answered ${Option(loaded).map(_.status())}") + + // 3) ... and the app actually came up on it: breadcrumbs are rendered from resource data that + // only exists on the other side of a gRPC-web call. + assertThat(page.getByTestId("breadcrumbs")).isVisible(vis(30000)) + assertThat(page.locator("[data-testid=breadcrumb][data-crumb-type=tenant]")).containsText(tenant) + assertThat(page.locator("[data-testid=breadcrumb][data-crumb-type=namespace]")).containsText(namespace) + + // 4) In-app navigation stays under the sub-path (the router's basename is the publicBaseUrl + // path). A basename left at "/" would push the browser to the origin root and 404 on reload. + page.locator("[data-testid=breadcrumb][data-crumb-type=tenant]").click() + val expectedUrl = instance.url(s"/tenants/$tenant/overview") + harness.Eventually.eventually(20000) { + assert( + page.url().startsWith(expectedUrl), + s"in-app navigation went to '${page.url()}', expected it to stay under the sub-path at '$expectedUrl'" + ) + } + + page.offResponse(record) + val seen = responses.asScala.toList + + // 5) gRPC-web resolved UNDER the sub-path. (gRPC-web reports failures in trailers, so the + // transport status is exactly the "not a 404" this is about.) + val api = seen.filter(_._2.startsWith(instance.url("/api/"))) + assert(api.nonEmpty, s"no gRPC-web call reached ${instance.url("/api/")}; requested: ${seen.map(_._2).distinct.sorted}") + assert(api.forall(_._1 == 200), s"gRPC-web calls did not resolve: ${api.filter(_._1 != 200)}") + + // 6) Static assets are resolved against `` - i.e. the sub-path - and nothing the + // page asked for was missing. A root-rooted URL anywhere in the app 404s here and only here. + assert( + seen.exists((status, url) => status == 200 && url.startsWith(instance.url("/ui/static/"))), + s"nothing was served from ${instance.url("/ui/static/")}; requested: ${seen.map(_._2).distinct.sorted}" + ) + val broken = seen.filter((status, url) => status >= 400 && url.startsWith(instance.origin)) + assert(broken.isEmpty, s"requests under ${instance.origin} failed: ${broken.distinct}") + + // 7) The credential cookie is scoped to the sub-path, so a different app on the same host + // cannot be handed Dekaf's Pulsar credentials. + val cookie = context.cookies().asScala.find(_.name == "pulsar_auth") + .getOrElse(fail(s"no pulsar_auth cookie; cookies: ${context.cookies()}")) + assert(cookie.path == subPath, s"cookie Path is '${cookie.path}', expected '$subPath'") + } + } diff --git a/e2e/src/test/scala/configuration/CfgCookieSpec.scala b/e2e/src/test/scala/configuration/CfgCookieSpec.scala new file mode 100644 index 000000000..e66020881 --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgCookieSpec.scala @@ -0,0 +1,137 @@ +package configuration + +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.Response +import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import java.net.URLDecoder +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.function.Predicate +import scala.jdk.CollectionConverters.* + +/** CFG-COOKIE - `cookieSecure` / `cookieSameSite`, driven the way an operator sets them: as + * `DEKAF_*` environment variables on a server process, observed as the `Set-Cookie` header the + * browser really receives when a credential is added through the UI. + * + * This is the property these specs exist for. Two bugs shipped here, and BOTH were config-to- + * behaviour wiring failures rather than logic errors: + * + * - `Secure` was computed into a local and never interpolated into the header string, so + * `cookieSecure: true` silently produced a non-Secure cookie; + * - `cookieSameSite` was matched against lowercase literals only, so the perfectly ordinary + * `Lax` (or `STRICT`, or a value with a trailing space) fell through to "" and emitted NO + * SameSite attribute at all - a cookie that looked hardened and was not. + * + * Neither was visible to any test, because every test called the cookie writer with hand-passed + * parameters. The path under test here is the whole one: env var -> `readConfig` -> the package + * `config` val -> `pulsarAuthToCookie` -> Javalin -> the embedded Envoy proxy -> Chromium. The + * mixed-case `Lax` in CFG-COOKIE-1 is deliberate: it is the exact value that shipped broken. + * + * The attribute list is asserted EXACTLY (as a sorted list, so order is free but duplicates and + * extras are not) rather than by `contains`: a `contains("SameSite=Lax")` cannot notice a smuggled + * `Domain=` or a `Secure` that went missing, which is half of what these two settings are for. + */ +class CfgCookieSpec extends DekafSuite: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + + /** Attributes every `Set-Cookie` from this server carries regardless of the two settings. */ + private val alwaysOn = List("Path=/", "HttpOnly", "Max-Age=31536000") + + /** The `Set-Cookie` written in response to ADDING A CREDENTIAL THROUGH THE UI, split into its + * cookie-name=value pair and its attribute list. + * + * Driving the real flow matters: `/pulsar-auth/add` is the route an operator's browser hits, it + * goes through Envoy, and it is one of the four call sites that share `setCookieAndSuccess`. */ + private def addCredentialSetCookie(instance: DekafInstance.Instance): (String, String, List[String]) = + val credential = fixtures.unique("cred").replaceAll("[^A-Za-z0-9_-]", "-") + + page.navigate(instance.url("/overview")) + page.getByTestId("credentials-button").click() + assertThat(page.getByTestId("modal")).isVisible(vis(20000)) + page.getByTestId("credentials-add").click() + page.getByTestId("credentials-name").fill(credential) + + val isAdd: Predicate[Response] = + (response: Response) => response.url().contains("/pulsar-auth/add/") && response.request().method() == "POST" + val response = page.waitForResponse(isAdd, () => page.getByTestId("credentials-save").click()) + + assert(response.status() == 200, s"adding a credential answered ${response.status()}: ${response.text()}") + // `headersArray` keeps the RAW response headers (unlike `headers()`, which lower-cases and joins + // repeated ones), so a second Set-Cookie would show up as a second entry rather than merge. + val setCookies = response.headersArray().asScala.toList + .filter(_.name.equalsIgnoreCase("set-cookie")) + .map(_.value) + assert(setCookies.size == 1, s"expected exactly one Set-Cookie header, got ${setCookies.size}: $setCookies") + + val header = setCookies.head + val parts = header.split(";", -1).toList.map(_.trim).filter(_.nonEmpty) + val pair = parts.head + + // Prove we intercepted the right response before asserting anything about its attributes: the + // cookie must be the credential cookie and must carry the credential this test just added. + assert(pair.startsWith("pulsar_auth="), s"Set-Cookie is not the pulsar_auth cookie: $header") + val decoded = URLDecoder.decode(pair.stripPrefix("pulsar_auth="), UTF_8) + assert(decoded.contains(s"\"$credential\""), s"the cookie does not carry the added credential '$credential': $decoded") + + (header, credential, parts.tail) + + private def assertAttributes(header: String, actual: List[String], expected: List[String]): Unit = + assert( + actual.sorted == expected.sorted, + s"Set-Cookie attributes were ${actual.mkString("[", ", ", "]")}, expected ${expected.mkString("[", ", ", "]")}\n raw: $header" + ) + + test("CFG-COOKIE-1: cookieSecure=true + a MIXED-CASE cookieSameSite=Lax reach Set-Cookie") { + DekafInstance.withInstance( + "cookie-lax", + "DEKAF_COOKIE_SECURE" -> "true", + "DEKAF_COOKIE_SAME_SITE" -> "Lax" // capitalised ON PURPOSE - the value that shipped broken + ) { instance => + val (header, _, attributes) = addCredentialSetCookie(instance) + assertAttributes(header, attributes, alwaysOn ++ List("Secure", "SameSite=Lax")) + + // ... and the browser ACCEPTED it with those attributes. The header being right is the bug + // that shipped; the cookie actually being stored hardened is the thing the operator wanted. + // (http://localhost is a trustworthy origin, so Chromium keeps a `Secure` cookie set there.) + val cookie = context.cookies().asScala.find(_.name == "pulsar_auth") + .getOrElse(fail(s"no pulsar_auth cookie in the browser context; header was: $header")) + assert(cookie.secure, s"the stored cookie is not Secure: $cookie") + assert(cookie.httpOnly, s"the stored cookie is not HttpOnly: $cookie") + assert(cookie.path == "/", s"the stored cookie's path is ${cookie.path}") + assert( + cookie.sameSite == com.microsoft.playwright.options.SameSiteAttribute.LAX, + s"the stored cookie's SameSite is ${cookie.sameSite}, expected LAX" + ) + } + } + + test("CFG-COOKIE-2: cookieSameSite=strict emits SameSite=Strict, and no Secure when it is unset") { + DekafInstance.withInstance("cookie-strict", "DEKAF_COOKIE_SAME_SITE" -> "strict") { instance => + val (header, _, attributes) = addCredentialSetCookie(instance) + // The absence of `Secure` is asserted by the list being exact - a `contains` check could not + // tell an unset cookieSecure from one that leaked in. + assertAttributes(header, attributes, alwaysOn :+ "SameSite=Strict") + } + } + + test("CFG-COOKIE-3: cookieSameSite=NONE with cookieSecure=true emits SameSite=None; Secure") { + DekafInstance.withInstance( + "cookie-none", + "DEKAF_COOKIE_SECURE" -> "true", + "DEKAF_COOKIE_SAME_SITE" -> "NONE" // upper case, the third spelling an operator may write + ) { instance => + val (header, _, attributes) = addCredentialSetCookie(instance) + assertAttributes(header, attributes, alwaysOn ++ List("Secure", "SameSite=None")) + } + } + + test("CFG-COOKIE-4: cookieSameSite=none WITHOUT cookieSecure emits no SameSite at all") { + // Browsers reject `SameSite=None` on a non-Secure cookie outright - the whole cookie is dropped + // and authentication breaks. The server withholds the attribute (and logs a warning) rather than + // emit a cookie the browser will refuse, so the observable contract is "no SameSite, no Secure". + DekafInstance.withInstance("cookie-none-insecure", "DEKAF_COOKIE_SAME_SITE" -> "none") { instance => + val (header, _, attributes) = addCredentialSetCookie(instance) + assertAttributes(header, attributes, alwaysOn) + } + } diff --git a/e2e/src/test/scala/configuration/CfgDataDirSpec.scala b/e2e/src/test/scala/configuration/CfgDataDirSpec.scala new file mode 100644 index 000000000..84ba070ed --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgDataDirSpec.scala @@ -0,0 +1,84 @@ +package configuration + +import features.library.LibrarySidebar +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters.* +import scala.util.Using + +/** CFG-DD - `dataDir`, the one place Dekaf keeps user state. There is no database: the Library + * writes managed items under `dataDir/library`, so this setting is the whole isolation boundary + * between two Dekafs on one host. The desktop app relies on it (a data dir per saved connection), + * and the e2e stack scripts rely on it (`DEKAF_FRESH_DATA=1`). + * + * Isolation is asserted in BOTH directions on purpose. "A second instance cannot see the item" is + * satisfied just as well by a query that can never find anything, so the same assertion is run + * against a third instance started on the FIRST data dir, where it must find it. + */ +class CfgDataDirSpec extends DekafSuite: + private def cnt(ms: Int) = new LocatorAssertions.HasCountOptions().setTimeout(ms.toDouble) + private def text(ms: Int) = new LocatorAssertions.ContainsTextOptions().setTimeout(ms.toDouble) + + private val itemType = "message-filter" + + /** Open the topic overview on `instance` and read the Library's count for `itemType`, waiting for + * the count to have genuinely LOADED (the UI collapses an unresolved count to 0). */ + private def libraryCount(url: String): LibrarySidebar = + page.navigate(url) + val lib = LibrarySidebar(page) + lib.openLibraryTab() + lib.openAllItemsSubtab() + lib.awaitTypeLoaded(itemType, 30000) + lib + + test("CFG-DD-1: a Library item saved under one dataDir is absent from an instance on another, and present on the same one") { + val tenant = fixtures.createTenant() + val namespace = fixtures.createNamespace(tenant) + val topic = fixtures.unique("topic") + admin.topics().createNonPartitionedTopic(s"persistent://$tenant/$namespace/$topic") + val topicPath = s"/tenants/$tenant/namespaces/$namespace/topics/persistent/$topic/overview" + val itemName = fixtures.unique("cfg-dd-item").replaceAll("[^A-Za-z0-9_-]", "-") + + DekafInstance.withDataDir("datadir-a") { dirA => + // --- instance A, on dirA: create the item ------------------------------------------------ + DekafInstance.withInstanceOn("datadir-a", dirA) { a => + page.navigate(a.url(topicPath)) + val lib = LibrarySidebar(page) + lib.openLibraryTab() + lib.createItemNamed(itemType, itemName) + + // It survived a reload, so it came off disk rather than out of the page's own state ... + assertThat(libraryCount(a.url(topicPath)).typeFound(itemType)).containsText("1", text(20000)) + + // ... and the disk it came off is the configured dataDir. This is an oracle INDEPENDENT of + // the UI that wrote it, which the Library otherwise has none of (README §6). + // (Items are stored as protobuf, so the bytes are decoded LENIENTLY - Latin-1 never throws - + // and searched for the name rather than parsed.) + assert( + filesUnder(dirA.resolve("library")).exists(f => latin1(f).contains(itemName)), + s"no file under $dirA/library mentions '$itemName': ${filesUnder(dirA.resolve("library"))}" + ) + + // --- instance B, on a data dir of its own: cannot see it ------------------------------ + DekafInstance.withInstance("datadir-b") { b => + assertThat(libraryCount(b.url(topicPath)).typeFound(itemType)).hasCount(0, cnt(20000)) + } + } + + // --- instance C, a NEW process on dirA: sees it again ------------------------------------ + // Without this, "B cannot see it" would be satisfied by a lookup that never finds anything. + DekafInstance.withInstanceOn("datadir-a-again", dirA) { c => + assertThat(libraryCount(c.url(topicPath)).typeFound(itemType)).containsText("1", text(20000)) + } + } + } + + private def filesUnder(dir: Path): List[Path] = + if !Files.isDirectory(dir) then Nil + else Using.resource(Files.walk(dir))(_.iterator().asScala.filter(Files.isRegularFile(_)).toList) + + private def latin1(file: Path): String = + new String(Files.readAllBytes(file), java.nio.charset.StandardCharsets.ISO_8859_1) diff --git a/e2e/src/test/scala/configuration/CfgDefaultAuthSpec.scala b/e2e/src/test/scala/configuration/CfgDefaultAuthSpec.scala new file mode 100644 index 000000000..7cf29888b --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgDefaultAuthSpec.scala @@ -0,0 +1,61 @@ +package configuration + +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.Locator +import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import java.net.URLDecoder +import java.nio.charset.StandardCharsets.UTF_8 +import scala.jdk.CollectionConverters.* + +/** CFG-AUTH - `defaultPulsarAuth`: credentials the operator configures once and every browser then + * gets as its "Default" credential, without anyone typing them in. The value is a JSON blob parsed + * at startup, injected into the `pulsar_auth` cookie on EVERY response that writes one + * (`setCookieAndSuccess` re-seeds `Default` each time, so a changed configuration reaches users + * that already have a cookie), and used to build the Pulsar clients for every request. + * + * The configured value here is a JWT-shaped credential whose token is the literal words + * `e2e.dummy.token` - it satisfies the app's JWT format check without being, or resembling, a + * secret. `empty` would have been benign too, but it is also the DEFAULT default, so it could not + * tell a working configuration from an ignored one. + */ +class CfgDefaultAuthSpec extends DekafSuite: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + + test("CFG-AUTH-1: the configured defaultPulsarAuth is the Default credential in the UI, in the cookie, and in use") { + val configured = """{"type":"jwt","token":"e2e.dummy.token"}""" + DekafInstance.withInstance("default-auth", "DEKAF_DEFAULT_PULSAR_AUTH" -> configured) { instance => + page.navigate(instance.url("/overview")) + + // 1) It is actually IN USE, not merely stored: the navigation tree's tenant list is a gRPC + // call whose Pulsar clients are built from the current credentials, and the interceptor + // answers UNAUTHENTICATED for the whole call when they cannot build one. + assertThat(page.locator("[data-testid=nav-tree-node][data-node-type=tenant][data-node-name='public']")) + .isVisible(vis(30000)) + + // 2) The credentials manager shows it as the Default credential, with its configured TYPE - + // "Empty" here would mean the JSON was ignored and the built-in default used instead. + page.getByTestId("credentials-button").click() + assertThat(page.getByTestId("modal")).isVisible(vis(20000)) + val defaultRow = page.locator("[data-testid=credentials-row][data-cred-name='Default']") + assertThat(defaultRow).isVisible(vis(20000)) + assertThat(defaultRow).containsText("JWT") + + // 3) ... and it is the one selected for this browser, which is what "default credentials for + // all users" means. + assertThat(defaultRow.getByText("Current", new Locator.GetByTextOptions().setExact(true))).isVisible(vis(20000)) + + // 4) The credential itself reached the browser, not just a label rendered from a gRPC reply: + // the HttpOnly cookie that every Pulsar call is authenticated with carries the configured + // type. (The token value is deliberately NOT asserted - the manager masks credentials, and + // a test that pinned a secret out of a cookie would be a bad pattern to copy.) + val cookie = context.cookies().asScala.find(_.name == "pulsar_auth") + .getOrElse(fail(s"no pulsar_auth cookie; cookies: ${context.cookies()}")) + val decoded = URLDecoder.decode(cookie.value, UTF_8) + assert( + decoded.contains("\"Default\":{\"type\":\"jwt\""), + s"the Default credential in the cookie is not the configured one: $decoded" + ) + } + } diff --git a/e2e/src/test/scala/configuration/CfgInstanceIdentitySpec.scala b/e2e/src/test/scala/configuration/CfgInstanceIdentitySpec.scala new file mode 100644 index 000000000..89c012c63 --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgInstanceIdentitySpec.scala @@ -0,0 +1,64 @@ +package configuration + +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.Page +import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import scala.jdk.CollectionConverters.* + +/** CFG-ID - `pulsarName` and `pulsarColor`: the two settings whose entire purpose is to be SEEN, so + * that an operator with a staging and a production Dekaf open side by side can tell which window + * is which. Both travel the same path (config -> the Freemarker `index.ftl` model -> the app's + * bootstrap config object -> React), and neither has any other effect, so if they do not render + * they do nothing at all. + * + * One property per test, each on an instance that overrides only that property, so a failure names + * one setting. + */ +class CfgInstanceIdentitySpec extends DekafSuite: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + // The tree node exists before its label does, so `isVisible` is not the readiness this needs and + // the assertion default (5s) is too short for a cold instance. + private def text(ms: Int) = new LocatorAssertions.ContainsTextOptions().setTimeout(ms.toDouble) + + test("CFG-ID-1: the configured pulsarName is rendered in the navigation tree and on the instance overview") { + // Unique, so no default, placeholder or leftover string can accidentally satisfy the assertion. + val instanceName = fixtures.unique("cfg-instance") + DekafInstance.withInstance("pulsar-name", "DEKAF_PULSAR_NAME" -> instanceName) { instance => + page.navigate(instance.url("/overview")) + + // The root node of the navigation tree - present on every page, and the app's own fallback + // when the name is unset is the literal "Pulsar Instance", which this cannot match. + val root = page.locator("[data-testid=nav-tree-node][data-node-type=instance]") + assertThat(root).isVisible(vis(30000)) + assertThat(root).containsText(instanceName, text(30000)) + + // ... and the "Instance Name" row of the instance overview. + val row = page.locator("tr", new Page.LocatorOptions().setHasText("Instance Name")) + assertThat(row).containsText(instanceName, text(30000)) + } + } + + test("CFG-ID-2: the configured pulsarColor reaches the DOM as the accent border") { + // A colour with no chance of being a default anywhere in the stylesheet. + DekafInstance.withInstance("pulsar-color", "DEKAF_PULSAR_COLOR" -> "#ff0099") { instance => + page.navigate(instance.url("/overview")) + assertThat(page.locator("[data-testid=nav-tree-node][data-node-type=instance]")).isVisible(vis(30000)) + + // The accent is a portal appended to whose inline style is built from the config value; + // it carries no testId, so it is selected by that inline style and asserted on the COMPUTED + // one - which is what the browser really paints, and what normalises the hex to rgb(). + val painted = page.evaluate( + """() => Array.from(document.body.children) + | .filter(el => el.style && el.style.boxShadow) + | .map(el => getComputedStyle(el).boxShadow)""".stripMargin + ).asInstanceOf[java.util.List[String]].asScala.toList + + assert(painted.size == 1, s"expected exactly one accent element under , got: $painted") + val boxShadow = painted.head + assert(boxShadow.contains("rgb(255, 0, 153)"), s"the accent is not the configured colour: '$boxShadow'") + assert(boxShadow.contains("inset"), s"the accent is not drawn inset: '$boxShadow'") + assert(boxShadow.contains("2px"), s"the accent has no width: '$boxShadow'") + } + } diff --git a/e2e/src/test/scala/configuration/CfgPublicBaseUrlSpec.scala b/e2e/src/test/scala/configuration/CfgPublicBaseUrlSpec.scala new file mode 100644 index 000000000..b56fc19a1 --- /dev/null +++ b/e2e/src/test/scala/configuration/CfgPublicBaseUrlSpec.scala @@ -0,0 +1,61 @@ +package configuration + +import harness.{DekafInstance, DekafSuite} +import com.microsoft.playwright.Response + +import java.util.function.Consumer +import scala.jdk.CollectionConverters.* + +/** CFG-PBU - `publicBaseUrl`, isolated from `basePath`. + * + * The two settings are usually turned on together, and CFG-BP already covers that. What is left + * unproven there is which of them owns which effect, so this test uses the deployment shape where + * they genuinely differ: a reverse proxy that STRIPS the prefix before forwarding. Dekaf is then + * served at the internal root (`basePath` untouched) while `publicBaseUrl` carries the public + * mount point, so that links and cookies are written for the URL the user's browser actually has. + * The server tier already assumes this shape - `pulsarAuthRoutesHttpTest` uses + * `publicBaseUrl = http://gateway.example/dekaf` with no basePath at all. + * + * Without the proxy in front, the sub-path requests the browser then makes have nothing to answer + * them, and that is exactly the point: this test asserts that the browser REQUESTS them, not that + * they succeed. Full app behaviour under a public prefix is CFG-BP's job. + */ +class CfgPublicBaseUrlSpec extends DekafSuite: + + test("CFG-PBU-1: publicBaseUrl alone moves link resolution and the cookie Path, with basePath untouched") { + val publicPath = "/gw" + DekafInstance.withInstance("public-base-url", "DEKAF_PUBLIC_BASE_URL" -> s"{origin}$publicPath") { instance => + val responses = java.util.Collections.synchronizedList(new java.util.ArrayList[String]()) + val record: Consumer[Response] = (response: Response) => responses.add(response.url()) + page.onResponse(record) + + // basePath really is untouched: the app is still served at the internal root, which is what + // the proxy forwards to. (If this 404'd, the effects below would be basePath's, not + // publicBaseUrl's, and the test would prove nothing about the property it names.) + val served = page.navigate(s"${instance.origin}/") + assert(served != null && served.status() == 200, s"the internal root answered ${Option(served).map(_.status())}") + + // 1) Link/asset resolution follows publicBaseUrl. `document.baseURI` is what every relative + // URL in the page resolves against - the app renders its links, its script and its + // stylesheet relatively for exactly this reason. + val baseUri = page.evaluate("() => document.baseURI").asInstanceOf[String] + assert(baseUri == s"${instance.origin}$publicPath/", s"document.baseURI is '$baseUri'") + + // 2) ... and that resolution reaches the NETWORK: the browser fetched the app's own assets + // under the public prefix. (They 404 here - there is no proxy to strip it back off - so + // what is asserted is the URL that was requested.) + page.offResponse(record) + val requested = responses.asScala.toList + assert( + requested.exists(_.startsWith(s"${instance.origin}$publicPath/ui/static/")), + s"no asset was requested under ${instance.origin}$publicPath/ui/static/; requested: ${requested.distinct.sorted}" + ) + + // 3) The credential cookie is scoped to the PUBLIC mount point, not to the internal root it + // was served from - otherwise every other app behind the same proxy host would be sent + // Dekaf's Pulsar credentials. + val cookie = context.cookies().asScala.find(_.name == "pulsar_auth") + .getOrElse(fail(s"no pulsar_auth cookie; cookies: ${context.cookies()}")) + assert(cookie.path == publicPath, s"cookie Path is '${cookie.path}', expected '$publicPath'") + } + } diff --git a/e2e/src/test/scala/features/consumersession/ConsumerSessionConfigSpec.scala b/e2e/src/test/scala/features/consumersession/ConsumerSessionConfigSpec.scala index 2ee1c70c3..32fcaea8a 100644 --- a/e2e/src/test/scala/features/consumersession/ConsumerSessionConfigSpec.scala +++ b/e2e/src/test/scala/features/consumersession/ConsumerSessionConfigSpec.scala @@ -8,12 +8,32 @@ import scala.jdk.CollectionConverters.* class ConsumerSessionConfigSpec extends DekafSuite: private def count(n: Int) = new LocatorAssertions.HasCountOptions().setTimeout(20000) - test("CS-1: Start-From offers all 7 options") { + /** The whole Start-From catalog. Named rather than counted: a count is satisfied by a renamed or + * swapped mode just as happily as by the right one, which is how this assertion previously + * survived a new mode being added without saying anything about it. */ + private val allStartFromModes = List( + "Earliest message", + "Latest message", + "Message with specific ID", + "Specific time", + "Relative time ago", + "Skip first n messages", + "Latest n messages", + // Two approximate modes, not one: "about half way in" is either half the ENTRIES or half the + // PUBLISH-TIME RANGE, and a single control could not say which. Named in full here because a count would be + // satisfied by one mode renamed to the other's label. + "Approximate position (% of data)", + "Approximate position (% of time)" + ) + + test("CS-1: Start-From offers every mode, all of them selectable on a persistent topic") { val (t, ns, topic) = fixtures.freshTopicParts() - ConsumerSessionPage(page).openForTopic(t, ns, topic) - val opts = page.getByTestId("cs-start-from").locator("option").allTextContents().asScala.toList - assert(opts.size == 7, s"got: $opts") - assert(opts.contains("Earliest message") && opts.contains("Latest message") && opts.contains("Message with specific ID")) + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + assert(cs.startFromLabels == allStartFromModes, s"got: ${cs.startFromLabels}") + // A persistent topic keeps history, so nothing is greyed out - the counterpart on a + // non-persistent topic is CsStartFromMatrixSpec CS-SFM-2. + assert(cs.disabledStartFromLabels.isEmpty, s"unexpectedly disabled on a persistent topic: ${cs.disabledStartFromLabels}") } test("CS-17: Stop clears the loaded messages") { diff --git a/e2e/src/test/scala/features/consumersession/CsApproximatePartitionedSpec.scala b/e2e/src/test/scala/features/consumersession/CsApproximatePartitionedSpec.scala new file mode 100644 index 000000000..3724acd3f --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsApproximatePartitionedSpec.scala @@ -0,0 +1,248 @@ +package features.consumersession + +import org.apache.pulsar.client.api.Message as PulsarMessage + +/** The two APPROXIMATE Start-From modes on a PARTITIONED topic - the shape in which their + * definitions actually differ from one another. + * + * `CsStartFromOutcomesSpec` covers both modes on a persistent NON-partitioned topic, where each + * mode sees exactly one physical log; on that shape "resolve every log independently" and "pool one + * answer across the logs" are the same sentence, so neither rule is under test. The production + * wiring (`handleStartFrom`, the two `Approximate*` branches) differs precisely here: + * + * - **Approximate position (% of data)** resolves EVERY PHYSICAL TOPIC INDEPENDENTLY: each partition leaves + * `floor(fraction x its own entry count)` entries behind, so a partition holding two entries + * and one holding eight start at different depths. + * - **Approximate position (% of time)** groups the physical topics by their LOGICAL topic, pools + * `min(first publish time)` .. `max(last publish time)` across the group, and seeks EVERY + * partition to that ONE instant - so a partition that went idle early cannot drag the cutoff + * backwards, and a partition that only started late is not given a range of its own. + * + * ONE arrangement drives both, and it is deliberately skewed on BOTH axes at once - the partitions + * hold 8 / 4 / 2 / 0 messages and cover three different stretches of the range: + * + * {{{ + * t = 0s t = 4s t = 8s t = 12s + * p0 a-01..a-04 a-05..a-08 8 messages, the middle of the range + * p1 b-01..b-04 4 messages, then idle for 12s + * p2 c-01 c-02 2 messages, only the late half + * p3 EMPTY - materialized, never written + * }}} + * + * p3 IS THE ONE PARTITION THAT HOLDS NOTHING, and it is here rather than in a unit test because the + * "empty" answer is a fact about the BROKER, not about the arithmetic. `PulsarAdmin.examineMessage` + * does not report an empty topic as an empty range - it FAILS (pinned at the broker level by + * `harness.BatchingFixtureSpec` BATCH-4), so an empty partition reaches the server as an error that + * it has to tell apart from an operational one. Both modes then have to SKIP it: counting it as + * time zero would drag the pooled range's start back to 1970 and put every interior fraction before + * the real data, i.e. turn 50% into Earliest. The pure arithmetic for that rule lives in + * `server/.../consumer/session_runner/approximatePublishTimePositionTest`; what only a real broker can + * show is that the failure is classified as "this partition holds nothing" and not as "the lookup + * broke", and that the session still starts. + * + * An empty partition is arrangeable here for the same reason the skew is: the schedule addresses + * the PHYSICAL partitions directly and simply never names p3. (This spec's doc used to say the + * opposite - that it could not be arranged through the parent's router - which was true of the + * router and irrelevant, since nothing here goes through it.) + * + * The pooled endpoints are each owned by exactly ONE partition, which is what makes the pooling + * rule falsifiable rather than merely satisfied: `min(first)` is p1's t=0 and `max(last)` is p2's + * t=12, so taking the max of the firsts, or the min of the lasts, or skipping an idle partition, + * every one of them moves the cutoff somewhere else. At 50% it lands at t=6s, in the middle of the + * only four-second gap in the arrangement, two seconds clear of the nearest message on either side. + * + * The two modes therefore answer differently, and each differs from the mistake it is exposed to. + * Both facts are asserted BEFORE the UI is driven, so a run can never pass by the answers having + * quietly converged: + * + * - data 50% -> per partition: 4 of 8, 2 of 4 and 1 of 2 entries left behind, which keeps p1's + * later half. A single cut through the POOLED stream would have kept a-04 and c-01 instead. + * - time 50% -> one cutoff at t=6s for EVERY partition: none of p1 (idle since t=0, six seconds + * before the cutoff) and BOTH of p2 (on its own 8s..12s range, 50% would be t=10s and would + * drop c-01). p3 contributes no endpoint at all; counted as time zero it would drag the pooled + * start to 1970 and hand back the whole topic. + * + * Every expectation is derived from the broker - real entry counts and real publish times - rather + * than written down, for the same reason `CsStartFromOutcomesSpec` does it: a hard-coded set would + * encode one particular arrangement as "the" answer instead of the rule. + */ +class CsApproximatePartitionedSpec extends StartFromSupport: + + /** FOUR partitions, spelled out rather than taken from `TopicKind.PersistentPartitioned` (three), + * because the fourth is the empty one and is part of the arrangement. */ + private val kind = fixtures.TopicKind(persistent = true, partitions = 4) + + /** The partition deliberately left holding nothing. */ + private val EmptyPartition = 3 + + /** Real wall clock between the arrangement's groups. Both modes are asserted at 50%, whose cutoff + * lands in the middle of the second gap, so this is also the safety margin: 4s puts the nearest + * message two seconds away and no produce round trip can move one across it. */ + private val GroupGapMs = 4000L + + /** What each partition is given, group by group: the index is which 4s slot it is published in, + * the map is partition -> values. */ + private val schedule: Seq[Map[Int, Seq[String]]] = Seq( + Map(1 -> Seq("b-01", "b-02", "b-03", "b-04")), + Map(0 -> Seq("a-01", "a-02", "a-03", "a-04")), + Map(0 -> Seq("a-05", "a-06", "a-07", "a-08"), 2 -> Seq("c-01")), + Map(2 -> Seq("c-02")) + ) + + /** Publish the schedule straight to the partitions, unbatched (one message per broker ENTRY, so + * entry-position arithmetic and a message count coincide and the expectation stays + * readable), with a real gap between groups. + * + * Addressing the partitions directly is the only way to control the skew: the parent topic's + * router decides where a message lands, and neither "p1 gets four messages and then goes idle" + * nor "p2 only exists in the late half" can be arranged through it. The gaps are ARRANGEMENT, not + * readiness waits - the only way to give a topic a time range is to publish across one. */ + private def arrange(fqn: String): Unit = + var groupStartedAt = System.currentTimeMillis() + schedule.zipWithIndex.foreach { (group, index) => + if index > 0 then + awaitClockGap(groupStartedAt, GroupGapMs) + groupStartedAt = System.currentTimeMillis() + group.toSeq.sortBy(_._1).foreach((partition, values) => fixtures.produceUnbatched(s"$fqn-partition-$partition", values)) + } + + /** Assert the arrangement landed as the class doc describes it, and hand back what each partition + * really holds, oldest first - the oracle both expectations are derived from. A drifted + * arrangement must fail HERE, naming the arrangement, rather than surfacing later as an + * unexplained set mismatch. */ + private def arrangedContents(fqn: String): Vector[Vector[PulsarMessage[String]]] = + // Reading every partition also MATERIALIZES the empty one: `createPartitionedTopic` writes + // metadata, and a partition nothing ever touched may not exist as a topic at all. The reader + // creates it, so the session meets the case this spec is about - a partition the broker knows + // and reports as holding nothing - rather than a missing topic, which is a different failure. + val perPartition = (0 until kind.partitions).toVector.map(p => fixtures.readAllMessages(s"$fqn-partition-$p")) + val arranged = schedule.flatMap(_.toSeq).groupBy(_._1).view.mapValues(_.flatMap(_._2)).toMap + (0 until kind.partitions).foreach { p => + val expected = arranged.getOrElse(p, Seq.empty) + assert(perPartition(p).map(_.getValue) == expected, s"partition $p holds ${perPartition(p).map(_.getValue)}, arranged $expected") + assert( + perPartition(p).map(_.getPublishTime) == perPartition(p).map(_.getPublishTime).sorted, + s"partition $p is not in publish-time order: ${perPartition(p).map(m => m.getValue -> m.getPublishTime)}" + ) + } + // EXACTLY ONE empty partition, and it is the one the arrangement names. Asserted through the + // broker's own entry count as well: "materialized and holding nothing" is the arranged state, + // and a partition that does not exist would fail here, naming itself. + val empties = perPartition.zipWithIndex.filter(_._1.isEmpty).map(_._2) + assert(empties == Vector(EmptyPartition), s"expected partition $EmptyPartition and only it to be empty, empty were: $empties") + val emptyEntries = fixtures.numberOfEntries(s"$fqn-partition-$EmptyPartition") + assert(emptyEntries == 0L, s"partition $EmptyPartition was supposed to hold nothing, the broker reports $emptyEntries entries") + + // The two skews the whole spec rests on, over the partitions that hold something: they must hold + // DIFFERENT numbers of messages, and must cover DIFFERENT stretches of the time range. + val nonEmpty = perPartition.filter(_.nonEmpty) + assert(nonEmpty.map(_.size).distinct.size == nonEmpty.size, s"the counts are not skewed: ${perPartition.map(_.size)}") + val spans = nonEmpty.map(msgs => msgs.head.getPublishTime -> msgs.last.getPublishTime) + assert(spans.distinct.size == nonEmpty.size, s"the time spans are not skewed: $spans") + // ... and exactly one partition owns each pooled endpoint, or the pooling rule is not falsifiable. + assert(spans.count(_._1 == spans.map(_._1).min) == 1, s"the pooled EARLIEST is not owned by one partition: $spans") + assert(spans.count(_._2 == spans.map(_._2).max) == 1, s"the pooled LATEST is not owned by one partition: $spans") + perPartition + + /** Where entry position starts on ONE partition: leave `floor(fraction x entries)` entries behind. */ + private def entryAnswer(perPartition: Vector[Vector[PulsarMessage[String]]], entriesOf: Int => Long): Vector[String] = + perPartition.zipWithIndex.flatMap { case (msgs, p) => + val entries = entriesOf(p) + assert(entries == msgs.size, s"partition $p: $entries entries for ${msgs.size} unbatched messages") + msgs.drop(math.floor(0.5 * entries).toInt).map(_.getValue) + } + + // ------------------------------------------------------------------------------------------- + + test("CS-SF-20: data position resolves EACH PARTITION on its own entry count") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + arrange(fqn) + val perPartition = arrangedContents(fqn) + + // The contract, applied per physical topic. The entry count comes from the broker even though + // the fixture is unbatched - the mode is defined over ENTRIES, and reading it back is what keeps + // that visible (and asserts the fixture really did write one message per entry). + val expected = entryAnswer(perPartition, p => fixtures.numberOfEntries(s"$fqn-partition-$p")) + assert( + expected.toSet == Set("a-05", "a-06", "a-07", "a-08", "b-03", "b-04", "c-02"), + s"the arrangement no longer produces the documented per-partition answer: $expected" + ) + + // The mistake this cell exists to catch: one cut through the pooled stream instead of one cut + // per partition. Same number of messages, different messages - so only an exact SET separates + // them, and they must genuinely differ or this test proves nothing. + val pooled = perPartition.flatten.sortBy(_.getPublishTime).map(_.getValue).takeRight(expected.size) + assert(pooled.toSet != expected.toSet, s"the pooled answer coincides with the per-partition one: $pooled") + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("50") + cs.play() + cs.assertState("running") + // EVERY partition, the empty one included, is really part of this session. Without this the + // empty partition would be decorative: a server that dropped it from the target list, or never + // subscribed to it because it could not resolve a position for it, would deliver exactly the + // same messages and pass. + awaitConsumersFlowing(fqn, kind) + assertLoadedExactlyWithCounter(cs, expected) + } + + test("CS-SF-21: publish-time position pools ONE cutoff across a logical topic's partitions") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + arrange(fqn) + val perPartition = arrangedContents(fqn) + + // The contract: min over the partitions' FIRST publish times .. max over their LAST, one + // interpolated instant, every partition seeked to it. Over the partitions that HOLD something - + // an empty one has no first and no last, and is skipped rather than counted as time zero. + val nonEmpty = perPartition.filter(_.nonEmpty) + val earliest = nonEmpty.map(_.head.getPublishTime).min + val latest = nonEmpty.map(_.last.getPublishTime).max + val cutoff = earliest + math.floor(0.5 * (latest - earliest)).toLong + val all = perPartition.flatten + // The cutoff has to fall in a GAP, or a produce round trip decides the outcome instead of the rule. + assert( + all.forall(m => math.abs(m.getPublishTime - cutoff) > 1000), + s"the cutoff $cutoff is within a second of a message: ${all.map(m => m.getValue -> m.getPublishTime)}" + ) + val expected = all.filter(_.getPublishTime >= cutoff).map(_.getValue) + assert( + expected.toSet == Set("a-05", "a-06", "a-07", "a-08", "c-01", "c-02"), + s"the arrangement no longer produces the documented pooled answer: $expected" + ) + + // The mistake this cell exists to catch: a range per PARTITION instead of one per logical topic. + // p1 went idle at t=0 and the pooled cutoff leaves all of it behind; on a range of its own its + // later half comes back. p2 started at t=8 and the pooled cutoff keeps both of its messages; on + // a range of its own 50% is t=10 and c-01 is dropped. + val perPartitionAnswer = nonEmpty.flatMap { msgs => + val first = msgs.head.getPublishTime + val last = msgs.last.getPublishTime + if last <= first then msgs.map(_.getValue) // a partition occupying one instant resolves to Earliest + else msgs.filter(_.getPublishTime >= first + math.floor(0.5 * (last - first)).toLong).map(_.getValue) + } + assert(perPartitionAnswer.toSet != expected.toSet, s"a per-partition range gives the same answer here: $perPartitionAnswer") + // And the sibling mode must not coincide either, or a session that ran the wrong one would pass. + val byEntry = entryAnswer(perPartition, p => perPartition(p).size.toLong) + assert(byEntry.toSet != expected.toSet, s"the entry mode gives the same answer here: $byEntry") + // The mistake the EMPTY partition exposes: counting it as time zero rather than skipping it. + // Its "first publish time" would be the epoch, so the pooled range would start in 1970 and 50% + // of it lands decades before this topic existed - i.e. the whole topic comes back. Derived, not + // asserted as a hunch: this is the position such an implementation would seek to. + val emptyAsTimeZeroCutoff = math.floor(0.5 * latest).toLong + val emptyAsTimeZeroAnswer = all.filter(_.getPublishTime >= emptyAsTimeZeroCutoff).map(_.getValue) + assert( + emptyAsTimeZeroAnswer.toSet != expected.toSet, + s"counting the empty partition as time zero gives the same answer here, so this arrangement cannot catch it: $emptyAsTimeZeroAnswer" + ) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("50") + cs.play() + cs.assertState("running") + awaitConsumersFlowing(fqn, kind) // including the empty partition - see CS-SF-20 + assertLoadedExactlyWithCounter(cs, expected) + } diff --git a/e2e/src/test/scala/features/consumersession/CsChunkingSpec.scala b/e2e/src/test/scala/features/consumersession/CsChunkingSpec.scala new file mode 100644 index 000000000..f1af16f8d --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsChunkingSpec.scala @@ -0,0 +1,165 @@ +package features.consumersession + +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions + +/** CHUNKING through the consumer session - the exact dual of the batching blind spot this suite + * was rebuilt around. Batching packs MANY messages into ONE broker entry; chunking spreads ONE + * message across MANY entries. `PulsarAdmin.examineMessage` counts entries either way, the + * counting Start-From modes promise MESSAGES ("Skip first n messages", "Latest n messages"), and + * until this spec no fixture ever produced a chunked message - so a chunk-unaware entry walk was + * structurally unobservable, exactly as the entry-vs-message defect was before `produceBatched`. + * + * The broker facts underneath (the message really is several entries; a chunk-capable consumer + * reassembles it) are pinned in `harness.MessageShapeFixtureSpec` SHAPE-1. Here the whole + * pipeline is under test: the session's consumer, the start-from resolution, the gRPC stream and + * the table must all treat the reassembled message as ONE message - in rendering, in counting, + * and where counting is IMPOSSIBLE, in refusing. + * + * THE TWO COUNTED MODES END UP IN DIFFERENT PLACES, and that asymmetry is the point of the spec: + * - "Skip first n messages" counts what the consumer DELIVERS, and a chunk-capable consumer + * delivers a reassembled message once - so it stays exact on chunked data (CS-CHK-2); + * - "Latest n messages" counts stored ENTRIES walking backwards, and on chunked data an entry + * count is not a message count in either direction, so the server REFUSES rather than + * answering with a set nobody asked for (CS-CHK-3). The refusal is + * `handleStartFrom.isChunkPiece` / `logEntryOf` and it is unit-pinned in + * `markerEntriesTest`; what only a browser can show is that it reaches the USER instead of + * becoming a silently wrong session. + * + * All three tests are single-topic and single-stream, so no delivery-order layer is involved and + * none is pinned. + */ +class CsChunkingSpec extends StartFromSupport: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + private def count(ms: Int) = new LocatorAssertions.HasCountOptions().setTimeout(ms.toDouble) + + /** SMALL ON PURPOSE, in both dimensions. + * + * 90 payload bytes at 32 bytes per chunk is 3 broker entries - the same shape a megabyte at the + * default chunk size has, and the shape the assertions need. It is small because the value + * column renders `limitString(value, 100)`: a 40 KiB payload (what this spec used to produce) + * reaches the table as a 100-character prefix plus an ellipsis, and so does a TORN fragment of + * it - the exact defect CS-CHK-1 exists to catch would have rendered identically to the correct + * result and passed. At 90 characters the whole value renders (92 with the JSON quotes, inside + * the 100 limit), so the assertion really does discriminate an intact message from a 32-byte + * fragment. + * + * VERIFIED against Pulsar 3.2.1 on a throwaway topic: a 90-byte value with + * `chunkMaxMessageSize = 32` becomes exactly 3 entries carrying `num_chunks_from_msg = 3` and + * chunk ids 0, 1, 2, and a reader reassembles them into ONE message equal to the value, with + * nothing following it. `produceChunked` asserts the entry count itself and throws if the data + * did not chunk, so a fixture that silently degraded could not make these tests vacuous. */ + private val ChunkBytes = 32 + private val chunkedValue = "chunk-" + "x" * 84 + + /** m-01, m-02, CHUNKED, m-04, m-05 - five messages in 2 + 3 + 2 = 7 broker entries, so message + * positions and entry positions cannot coincide anywhere past m-02. Returns (t, ns, topic). */ + private def arrangeAroundChunk(): (String, String, String) = + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, Seq("m-01", "m-02")) + val chunks = fixtures.produceChunked(fqn, chunkedValue, chunkBytes = ChunkBytes) + fixtures.produceUnbatched(fqn, Seq("m-04", "m-05")) + // The arrangement premise, restated against the broker: 4 plain entries + the chunk entries. + val entries = fixtures.numberOfEntries(fqn) + assert(entries == 4 + chunks, s"expected ${4 + chunks} entries around the chunked message, got $entries") + assert(chunks == 3, s"the arrangement assumes 3 chunk entries, got $chunks") + (t, ns, topic) + + test("CS-CHK-1: a chunked message arrives INTACT and renders as ONE row, not one per chunk") { + // What fails here [phase-2 bite-check - needs the running app]: a session consumer that + // surfaced chunks as rows shows 7 rows and a counter of 7 (fails the counter leg); a decode + // that tears or truncates the payload delivers a 32-byte fragment where the exact set expects + // all 90 characters, which is visible now that the value fits inside the cell's 100-character + // render limit. + val (t, ns, topic) = arrangeAroundChunk() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, Seq("m-01", "m-02", chunkedValue, "m-04", "m-05")) + } + + test("CS-CHK-2: Skip first n counts a chunked message as ONE message, not as its entries") { + // Skip 3 MESSAGES of [m-01, m-02, CHUNKED, m-04, m-05] leaves exactly {m-04, m-05}. An + // entry-addressed skip of 3 cuts INSIDE the chunked message (the first three entries are + // m-01, m-02 and chunk 1 of 3) and either delivers the chunked message too or a torn + // fragment of it - both fail the exact set. This mode stays EXACT on chunked data because it + // counts what the consumer delivers, which is the reassembled message, once. + // [phase-2 bite-check - needs the running app] + val (t, ns, topic) = arrangeAroundChunk() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("3") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, Seq("m-04", "m-05")) + } + + test("CS-CHK-3: Latest n REFUSES when its backward walk reaches a chunk, and shows nothing instead") { + // THE CORRECTED EXPECTATION. This test used to assert that Latest-3 delivered exactly + // {CHUNKED, m-04, m-05} - written on the assumption that the start-from code is chunk-UNAWARE + // and ought to be taught to count chunked messages as one. It is not unaware, and it cannot + // be taught that: the walk reads ENTRY metadata, and on a chunked topic entries and messages + // have no fixed relationship in either direction, so there is no count to substitute. It + // therefore REFUSES, and refusing is the correct behavior - the wrong answer it declines to + // give is exactly the one the old assertion demanded be accepted. + // + // The walk is what triggers it, so the refusal is precise rather than topic-wide: from the end + // the entries are m-05, m-04, chunk 3, chunk 2, chunk 1, m-02, m-01, so a request for 3 + // messages MUST step onto chunk 3 (the second leg below pins the other side of that boundary). + // + // [phase-2 bite-check - needs the running app. A walk that silently counted chunk entries + // would leave the session `running` with rows on screen and no error, failing all three + // assertions here; a refusal that never reached the browser would fail the first.] + val (t, ns, topic) = arrangeAroundChunk() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("3") + cs.play() + + // The server's own words, surfaced by `notifyError` - the same path CS-FC-3/4's creation-time + // refusals take. Matched on the phrase that names the cause, not on the whole sentence. + val refusal = page.getByText( + java.util.regex.Pattern.compile("stores CHUNKED messages", java.util.regex.Pattern.CASE_INSENSITIVE) + ).first() + assertThat(refusal).isVisible(vis(15000)) + + // AND NOT A WRONG SET. A create that was refused leaves nothing running, so the session falls + // back to `new` and no row is ever rendered - the observable difference between "declined" and + // "answered with the last three entries". + cs.assertState("new") + assertThat(cs.messages).hasCount(0, count(5000)) + + // The remedy the refusal names works on the very same topic: 'Skip first n messages' counts + // what is delivered, so it stays exact where the entry walk could not be. + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("3") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, Seq("m-04", "m-05")) + } + + test("CS-CHK-4: Latest n still answers when its walk stops SHORT of the chunk") { + // The other side of CS-CHK-3's boundary, and what keeps that refusal honest: the walk refuses + // when it actually reaches a chunk entry, not because the topic once stored one. The last 2 + // messages are m-04 and m-05, the last 2 entries, and the walk stops before chunk 3 - so this + // is a correct answer and must be delivered, not refused. Without this leg, a walk that + // refused every topic containing any chunked message would still pass CS-CHK-3. + // [phase-2 bite-check - needs the running app] + val (t, ns, topic) = arrangeAroundChunk() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("2") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, Seq("m-04", "m-05")) + } diff --git a/e2e/src/test/scala/features/consumersession/CsConsoleSpec.scala b/e2e/src/test/scala/features/consumersession/CsConsoleSpec.scala index f147b4d10..d14427fe2 100644 --- a/e2e/src/test/scala/features/consumersession/CsConsoleSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsConsoleSpec.scala @@ -3,26 +3,91 @@ package features.consumersession import harness.DekafSuite import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.assertions.LocatorAssertions +import java.util.regex.Pattern +import scala.collection.mutable class CsConsoleSpec extends DekafSuite: - test("CS-30: Tools panel exposes Produce / REPL / Logs tabs (Produce present on a topic)") { + test("CS-30: More tools is open by default and exposes Produce / REPL / Logs tabs") { val (t, ns, topic) = fixtures.freshTopicParts() val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) cs.openTools() + assertThat(cs.toolsButton).containsText("More tools") + assertThat(cs.toolsCloseButton).isVisible() val tools = ToolsPanel(page) assertThat(tools.produceTab).isVisible() assertThat(tools.replTab).isVisible() assertThat(tools.logsTab).isVisible() - assertThat(tools.produceSend).isVisible() // Produce is the default active tab + // Topic Positions is the default active tab - the "where am I?" view leads. + assertThat(tools.topicPositionsNotStarted).isVisible() + tools.produceTab.click(); assertThat(tools.produceSend).isVisible() tools.replTab.click(); assertThat(tools.replRun).isVisible() tools.logsTab.click(); assertThat(page.getByText("logDebug")).isVisible() } - test("CS-31: REPL is disabled until a run, then executes and clears") { + test("CS-30R: Tools panel height is resizable and persists across toggling and reload") { + val (t, ns, topic) = fixtures.freshTopicParts() + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.openTools() + assertThat(cs.toolsResizeHandle).isVisible() + + // 340 CSS px by owner instruction (2026-08-11; was 400rem = 360px under the 0.9 root scale). + // A plain pixel value, deliberately not rem-scaled - the persisted drag sizes are pixels too. + val defaultHeight = cs.toolsPanelDomHeight + assert(math.abs(defaultHeight - 340) <= 3, + s"the resizable Tools panel changed its 340px default: $defaultHeight") + + // A large preference restored in a shorter viewport is capped without rewriting it. More + // importantly, the FIRST drag starts at the VISIBLE cap, not the hidden 900px preference - + // otherwise this 90px drag would appear to do nothing. + cs.setStoredPaneSize("consumer-session-tools", 900) + page.reload() + cs.openTools() + val before = cs.toolsPanelDomHeight + assert(before >= cs.sessionDomHeight * 0.80 - 3, + s"Tools should be draggable well beyond the old 65% cap: $before of ${cs.sessionDomHeight}") + assert(before <= cs.sessionDomHeight * 0.85 + 3, + s"oversized Tools preference was not capped: $before of ${cs.sessionDomHeight}") + assert(cs.storedPaneSize("consumer-session-tools") == 900, + s"responsive cap must not overwrite the preference: ${cs.storedPaneSize("consumer-session-tools")}") + + // Drive the actual top edge down. For a bottom pane that must SHRINK its rendered height. + cs.resizeToolsBy(90) + val resized = cs.toolsPanelDomHeight + assert(resized < before - 40, s"dragging the Tools top edge down did not shrink it: $before -> $resized") + val stored = cs.storedPaneSize("consumer-session-tools") + assert(stored > 0, s"the resized Tools height was not persisted: $stored") + + // Closing through the panel cross and reopening must not reset the pane to its default height. + cs.closeTools() + assertThat(cs.toolsResizeHandle).not().isVisible() + cs.openTools() + assertThat(cs.toolsResizeHandle).isVisible() + val afterReopen = cs.toolsPanelDomHeight + assert(math.abs(afterReopen - resized) <= 3, + s"Tools height changed after hide/reopen: $resized -> $afterReopen") + + // The shared pane contract is browser-persistent, like navtree/split sizes. + page.reload() + cs.openTools() + assertThat(cs.toolsResizeHandle).isVisible() + assert(cs.storedPaneSize("consumer-session-tools") == stored, + s"stored Tools height changed across reload: $stored -> ${cs.storedPaneSize("consumer-session-tools")}") + val afterReload = cs.toolsPanelDomHeight + assert(math.abs(afterReload - resized) <= 3, + s"persisted Tools height was not re-applied after reload: $resized -> $afterReload") + + // The lower bound is intentionally compact too: users can reclaim nearly all vertical space. + cs.resizeToolsBy(2000) + assert(cs.toolsPanelDomHeight <= 100, + s"Tools could not be dragged down to its expanded minimum range: ${cs.toolsPanelDomHeight}") + } + + test("CS-31: REPL is disabled until a run, then exposes lastMessage and clears") { val (t, ns, topic) = fixtures.freshTopicParts() fixtures.produceStrings(s"persistent://$t/$ns/$topic", 1) val cs = ConsumerSessionPage(page) @@ -35,29 +100,69 @@ class CsConsoleSpec extends DekafSuite: cs.setStartFrom("Earliest message") cs.play() cs.assertState("running") + cs.awaitLoaded(1) assertThat(tools.replRun).isEnabled() - // Click the Monaco editor surface (not just the wrapper) so the hidden textarea receives focus. - tools.replEditor.locator(".monaco-editor").click() - page.waitForTimeout(300) - page.keyboard().`type`("2 + 2") + // The help advertises this public binding. Exercise the complete browser -> gRPC -> shared + // Graal context path so a private-only implementation cannot make that promise regress again. + tools.writeRepl("lastMessage.value") page.waitForTimeout(300) tools.replRun.click() - assertThat(tools.replLogs).containsText("4", new LocatorAssertions.ContainsTextOptions().setTimeout(15000)) + assertThat(tools.replLogs).containsText("msg-1", new LocatorAssertions.ContainsTextOptions().setTimeout(15000)) tools.replClear.click() assertThat(tools.replLogs).hasText("") assertThat(tools.replClear).isDisabled() } - test("CS-32: Context Logs render (empty-state placeholder for logDebug output)") { + // Monaco is NOT part of the JS bundle: `@monaco-editor/react` fetches it at runtime through its + // own AMD loader, whose default base is `cdn.jsdelivr.net`. Every code editor in Dekaf therefore + // used to depend on the public internet - seconds of third-party network before the first editor + // appeared, and no editor at all offline, air-gapped, or behind a proxy that blocks the CDN. + // `ui/build.js` now copies `monaco-editor/min/vs` next to the bundle and `CodeEditor.tsx` points + // the loader at that same-origin path. + // + // Nothing pinned it, because a runner with working internet cannot tell "served by Dekaf" from + // "downloaded from jsdelivr" - both mount an editor. Blocking the CDN outright is what makes the + // difference observable, and makes this a permanent regression rather than a property of the + // network the run happened to have. + test("CS-33: a code editor mounts with the public CDN blocked, from Dekaf's own origin") { + val requested = mutable.ListBuffer.empty[String] + page.onRequest(r => requested += r.url()) + // Not merely observed - REFUSED. On a runner with internet a regression would silently succeed + // through the CDN, and the assertion below would pass while the offline case stayed broken. + page.route(Pattern.compile(".*jsdelivr.*"), _.abort()) + + val (t, ns, topic) = fixtures.freshTopicParts() + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.openTools() + ToolsPanel(page).produceTab.click() // Topic Positions is the default, so reveal Produce explicitly. + + val editor = page.getByTestId("produce-value").locator(".monaco-editor").first() + assertThat(editor).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(30000)) + + // The editor rendering is necessary but not sufficient: assert WHERE Monaco came from. + val fromDekaf = requested.toList.filter(_.contains("/ui/static/dist/vs/")) + assert(fromDekaf.nonEmpty, s"Monaco was not fetched from Dekaf's own origin; requests: ${requested.toList}") + val fromCdn = requested.toList.filter(_.contains("jsdelivr")) + assert(fromCdn.isEmpty, s"the editor still reaches for the public CDN: $fromCdn") + } + + test("CS-32: Context Logs omit ordinary messages and keep the explicit-output empty state") { val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 1) val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) cs.openTools() val tools = ToolsPanel(page) + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(1) tools.logsTab.click() assertThat(tools.logs).isVisible() assertThat(page.getByText("logDebug")).isVisible() - // NOTE: no text/level filter control exists; generating real debugStdout needs a JS filter/projection (see NOTES). + // Message ingestion itself used to console.log the full JSON object, making this tab a noisy + // duplicate of the message table even when the user wrote no logging code. + assertThat(tools.logs).not().containsText("msg-1") } diff --git a/e2e/src/test/scala/features/consumersession/CsDeliveryControlsSpec.scala b/e2e/src/test/scala/features/consumersession/CsDeliveryControlsSpec.scala new file mode 100644 index 000000000..9c71ef163 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsDeliveryControlsSpec.scala @@ -0,0 +1,102 @@ +package features.consumersession + +import harness.DekafSuite + +/** The two browser-wide delivery controls in the toolbar: "msg/s limit" and "pause after". + * + * Both live in localStorage and ride the session's requests - the rate on each Resume, the + * auto-pause purely client-side - so neither is part of the session's saved definition. What only + * an end-to-end test can see: that the number typed into the toolbar actually SLOWS a real + * session against a real broker, and that the auto-pause lands the session in `paused` near the + * threshold rather than merely somewhere. + * + * THE RATE IS A BAND, THE BUDGET IS EXACT - deliberately different assertions. A rate of 100/s + * starts with a full one-second burst (by design: the first screenful paints at once) and the UI + * flushes every 250ms, so a fixed observation window can only assert a corridor. "Pause after n" + * carries a server-side delivery budget, so its count is asserted with EQUALITY: exactly n, then + * exactly n more. The offline arithmetic lives in deliveryRateLimiterTest and deliveryBudgetTest; + * what this spec pins is that the whole path is wired against a real broker. + */ +class CsDeliveryControlsSpec extends DekafSuite: + + /** Every test leaves the browser-wide settings OFF, so no later spec inherits a throttle. */ + private def clearControls(cs: ConsumerSessionPage): Unit = + cs.setRateLimit(0) + cs.setPauseAfter(0) + + test("DC-1: the rate limit slows a real session, and clearing it restores full speed") { + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 2000) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + try + cs.setRateLimit(100) + cs.setStartFrom("Earliest message") + cs.play() + + // ~4s at 100/s: the burst (100) plus ~400 paced, with generous slack for the stack under + // load. The line that matters is the ceiling: WELL under the 2000 an unlimited session + // loads in this window (DC-1's second half proves that below, on the same topic). + page.waitForTimeout(4000) + val limited = cs.loadedCount + assert(cs.state == "running", s"expected a throttled session to still be running, got '${cs.state}'") + assert(limited >= 100, s"the first second's burst should have painted at least the rate, got $limited") + // The ideal is ~500 (the 100 burst + ~4s at 100/s). 800 tolerates stack jitter but convicts + // a limiter running 2x fast or worse; the old 1500 ceiling waved a 3x-broken limiter through. + assert(limited <= 800, s"a 100/s limit should not have loaded $limited messages in ~4s") + + // Clear the limit and restart: the same topic must now load completely FASTER than the + // throttled run's theoretical minimum ever could - a still-active 100/s limiter needs ~19s + // for the remaining 1900, so finishing inside 12s proves the limit is genuinely off. + cs.stop() + cs.setRateLimit(0) + cs.play() + cs.awaitLoaded(2000, timeoutMs = 12000) + finally clearControls(cs) + } + + test("DC-2: pause-after lands the session in `paused` near the threshold, and Play re-arms it") { + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 2000) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + try + // The rate limit keeps the overshoot small: at 200/s, the chunks that land while the pause + // RPC is in flight are tens of messages, not the rest of the topic. + cs.setRateLimit(200) + cs.setPauseAfter(300) + cs.setStartFrom("Earliest message") + cs.play() + + // EXACTLY 300, not a band: the server-side delivery budget stops the stream at the + // message that spends the last unit, so no chunk latency and no rate-limit burst can + // overshoot it. The client's pause lands after; the count is already settled. + cs.assertState("paused", timeoutMs = 30000) + val firstStop = cs.loadedCount + assert(firstStop == 300, s"'pause after 300' must load exactly 300, got $firstStop") + + // Play again: both halves re-arm - the client threshold at "current + n", the server budget + // at n more - so Play behaves as "give me exactly 300 more". + cs.play() + cs.assertState("paused", timeoutMs = 30000) + val secondStop = cs.loadedCount + assert(secondStop == 600, s"the re-armed budget must land at exactly 600, got $secondStop") + finally clearControls(cs) + } + + test("DC-3: both controls are REMEMBERED across a reload - they belong to the browser") { + val (t, ns, topic) = fixtures.freshTopicParts() + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + try + cs.setRateLimit(123) + cs.setPauseAfter(456) + + page.reload() + val reloaded = ConsumerSessionPage(page) + assert(reloaded.rateLimitInput.inputValue() == "123", s"rate limit lost on reload: '${reloaded.rateLimitInput.inputValue()}'") + assert(reloaded.pauseAfterInput.inputValue() == "456", s"pause-after lost on reload: '${reloaded.pauseAfterInput.inputValue()}'") + finally clearControls(ConsumerSessionPage(page)) + } diff --git a/e2e/src/test/scala/features/consumersession/CsDeliveryModesSpec.scala b/e2e/src/test/scala/features/consumersession/CsDeliveryModesSpec.scala new file mode 100644 index 000000000..4152ab566 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsDeliveryModesSpec.scala @@ -0,0 +1,1004 @@ +package features.consumersession + +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions +import harness.Eventually.eventually + +/** THE DELIVERY-ORDER x TOPIC-LIVENESS MATRIX - now aligned with the GUARANTEED EXACT-REPLAY + * contract (owner decision 2026-08-09; the hold-forever behavior these cells used to pin is + * gone, and with it the PINS markers that guarded it). + * + * Every cell crosses one topic-liveness state with one delivery-order mode, and EVERY test pins + * its mode EXPLICITLY through the UI control (`cs-delivery-order`, driven by label). No test in + * this file depends on the session default: the matrix must keep meaning the same thing whichever + * mode the owner makes the default. (`CsMergeOrderSpec` CS-MO-0 is the one test that pins what + * the default IS.) + * + * The liveness states: + * - A ALL LIVE: every stream keeps receiving while the session runs. + * - B ALL STATIC: backlog exists, nothing new arrives after Play. + * - C MIXED: one stream keeps receiving, the other stopped before Play. + * - D EDGES: empty-from-birth partitions (D1), a static topic that becomes live again + * (D2), a single stream (D3), Guaranteed refusing a non-persistent stream + * (D4), the caught-up pause's one-click switch to Best effort (D5), and + * broker ownership churn via topic unload (D6). + * - R REPLAY: the replay contract's own cells - the banner's content (R1), the + * resume-extends-the-chunk mechanism (R2), and the Latest x Guaranteed gate + * (R3). + * + * The mode contracts asserted, per cell: + * - FASTEST: every produced message exactly once (the identity oracle below); NO order + * assertion anywhere - Fastest is the absence of the merge, and arrival interleaving is the + * broker's business, not a promise. + * - BEST EFFORT: every message exactly once, in global publish-time order wherever timestamps + * are distinct (every arrangement here ticks its sends apart); STATIC TAILS MUST COMPLETE - + * the bounded reorder grace (mergeTopicsGraceMs + one sweep, ~0.75s) releases them. + * - GUARANTEED: an EXACT REPLAY. Play captures every stream's recorded end; the session + * delivers everything recorded up to that boundary - of what retention still holds - in + * strict key order, then AUTO-PAUSES with the caught-up banner (`cs-replay-caught-up`) on + * the ordinary paused state. A message published AFTER Play belongs to the NEXT chunk: it is + * handed back and its consumer held, never delivered into the current chunk. "Load new messages" extends + * the boundary to now and replays the delta exactly; "Switch to Best effort and follow live" is the + * designed way OUT of the replay into live following. No live phase, no stall, no held tail: + * a drained or empty stream holds nothing back, because history needs no proof from the + * future. + * + * THE IDENTITY ORACLE is the exact MULTISET, never a count: `ConsumerSessionPage.exportedValues` + * (the app's own bulk export of every retained row, on a paused session) compared through + * `DeliveredMessages.assertExactly`, so one loss plus one duplicate cannot cancel out. Where a + * mode promises order, arrival order is read separately via `allColumnValues` (ordinal-keyed + * walk of the whole virtualized table). + * + * SEAM VIOLATIONS (a producer-clock reversal across a resume seam) need controlled clock skew + * and are pinned at the unit tier. What IS e2e-checkable, and asserted in every replay/resume + * cell that pauses at a boundary: ordinary chunked flows raise ZERO seam flags - the session + * warning mark (`cs-order-warning`, which carries the seam count under Guaranteed since the + * 2026-08-11 toolbar rework) stays absent and no row carries the marker. + * + * WAITS are deterministic: sentinel/count gating (`awaitLoaded` on totals whose last message is + * the sentinel), the caught-up banner itself as the app's own "the replay reached its boundary" + * disclosure, and `settledLoaded`'s bounded quiet window (~1.2s, justified in StartFromSupport) + * ONLY for asserting "nothing more arrives". + */ +class CsDeliveryModesSpec extends StartFromSupport: + + private def vis(timeoutMs: Double) = new LocatorAssertions.IsVisibleOptions().setTimeout(timeoutMs) + + private val fastestLabel = "Fastest" + private val bestEffortLabel = "Best effort" + private val guaranteedLabel = "Guaranteed" + + private def waitingChip = page.getByTestId("cs-order-waiting") + + // The replay surface (the landed redesign's own test-ids). + private def caughtUpBanner = page.getByTestId("cs-replay-caught-up") + private def bannerExcludedLine = page.getByTestId("cs-replay-caught-up-excluded") + // Labelled "Load new messages up to now"; the testid keeps the older `resume` wording because + // that is what the action still IS - a resume that extends the boundary. + private def loadNewMessages = page.getByTestId("cs-replay-resume") + private def bannerContinueBestEffort = page.getByTestId("cs-replay-continue-best-effort") + // Since 2026-08-11 the session-level count rides the warning mark beside the loaded counter. + private def seamChip = page.getByTestId("cs-order-warning") + private def seamMarkers = page.getByTestId("cs-out-of-order-marker") + + /** Two fresh single-log topics; every multi-TOPIC cell arranges on this shape because it gives + * the test an independent producer handle per stream - liveness is then a per-stream choice. */ + private def twoFreshTopics(): (String, String, (String, String, String)) = + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val (tB, nsB, topicB) = fixtures.freshTopicParts() + (s"persistent://$tA/$nsA/$topicA", s"persistent://$tB/$nsB/$topicB", (tA, nsA, topicA)) + + private def partitionFqns(parentFqn: String, partitions: Int): Seq[String] = + (0 until partitions).map(i => s"$parentFqn-partition-$i") + + /** Open a two-topic session with the mode under test pinned through the control. */ + private def openTwoTopicSession(parts: (String, String, String), fqns: Seq[String], mode: String): ConsumerSessionPage = + val cs = ConsumerSessionPage(page) + cs.openForTopic(parts._1, parts._2, parts._3) + cs.setTargetTopicsSpecific(fqns) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(mode) + cs + + /** Pause (a running session cannot export its whole retained set) and return the identity + * oracle: the exact multiset of every value the session delivered, from the app's own export. + * A session the replay boundary already auto-paused is exported as-is. */ + private def pausedExportedSet(cs: ConsumerSessionPage): List[String] = + if cs.state == "running" then + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + cs.exportedValues() + + /** Arrival-order oracle: (ordinal, value) pairs off the whole virtualized table. Read on a + * paused session so the walk does not race the auto-scroll. */ + private def assertArrivalOrder(cs: ConsumerSessionPage, expected: Seq[String], what: String): Unit = + val delivered = cs.allColumnValues("value") + assert(delivered == expected.toList, s"$what: delivered order was $delivered\n expected ${expected.toList}") + + /** "Nothing more arrives": the loaded counter, once quiet (settledLoaded's bounded window), + * must equal `expected` - with the observed value in the failure so a leak or a shortfall is + * named, not just detected. */ + private def assertSettledLoaded(cs: ConsumerSessionPage, expected: Int, why: String): Unit = + val settled = settledLoaded(cs) + assert(settled == expected, s"$why: the counter settled at $settled, expected $expected") + + /** The replay boundary, observed through the app's own disclosure: exactly `expectedLoaded` + * messages are on the counter, the caught-up banner is up, and the session sits on the + * ORDINARY paused state (the auto-pause is the same state the pause button reaches). */ + private def awaitCaughtUp(cs: ConsumerSessionPage, expectedLoaded: Int, timeoutMs: Double = 30000): Unit = + cs.awaitLoaded(expectedLoaded, timeoutMs) + assertThat(caughtUpBanner).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + + /** Ordinary replay/resume flows must raise NO seam flags: the session-level counter chip is the + * authoritative signal (it rides the stats channel, not the virtualized rows); the mounted-row + * markers are checked on top. */ + private def assertNoSeamViolations(): Unit = + assertThat(seamChip).not().isVisible() + assert(seamMarkers.count() == 0, s"expected no seam-violation row markers, found ${seamMarkers.count()}") + + /** Values of the rows carrying the switch-point divider badge, across the WHOLE virtualized + * table (the badge sits on the row's sticky publish-time cell, so a viewport read could miss + * it). Same walk as `allColumnValues`; call on a paused session. */ + private def switchPointRowValues(): List[String] = + val json = page + .evaluate( + """async () => { + | const table = document.querySelector("[data-testid='cs-table']"); + | if (!table) return "[]"; + | const found = new Map(); + | const harvest = () => { + | for (const r of table.querySelectorAll("tbody tr")) { + | const badge = r.querySelector("[data-testid='cs-order-switch-point']"); + | const ord = r.querySelector("[data-testid='cs-message']"); + | const cell = r.querySelector("[data-testid='cs-cell-value']"); + | if (badge && ord && cell) found.set(parseInt(ord.innerText.trim(), 10), cell.innerText.trim()); + | } + | }; + | harvest(); + | const s = table.querySelector("[data-virtuoso-scroller]"); + | const frame = () => new Promise(res => requestAnimationFrame(() => requestAnimationFrame(res))); + | if (s && s.scrollHeight > s.clientHeight + 2) { + | let target = 0; + | let guard = 0; + | while (guard < 300) { + | guard += 1; + | s.scrollTop = target; + | await frame(); + | harvest(); + | if (target >= s.scrollHeight - s.clientHeight - 2) break; + | target = Math.min(target + s.clientHeight * 0.8, s.scrollHeight - s.clientHeight); + | } + | } + | return JSON.stringify(Array.from(found.entries()).sort((a, b) => a[0] - b[0]).map(e => e[1])); + |}""".stripMargin + ) + .toString + val mapper = new com.fasterxml.jackson.databind.ObjectMapper() + val root = mapper.readTree(json) + (0 until root.size()).toList + .map(root.get(_).asText().trim) + .map(v => if v.length >= 2 && v.startsWith("\"") && v.endsWith("\"") then v.substring(1, v.length - 1) else v) + + // ============================================================================================= + // CELL A - ALL LIVE. Two topics (a live producer handle per stream); both receive during the + // session. Distinct publish times come from produceRoundRobin's inter-send clock tick, so the + // global publish-time order IS the production order and the ordered cells can assert a full + // sequence rather than a set. + // ============================================================================================= + + private val liveSeeds = (0 until 6).map(i => f"m-$i%02d") + private val liveTail = (0 until 8).map(i => f"l-$i%02d") + + test("CS-DM-AF [ALL LIVE x Fastest]: every message from two live topics arrives exactly once - interleaving unasserted") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveSeeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), fastestLabel) + cs.play() + cs.assertState("running") + + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveTail) // both streams live DURING the session + cs.awaitLoaded(liveSeeds.size + liveTail.size) + assertSettledLoaded(cs, liveSeeds.size + liveTail.size, "Fastest must deliver exactly once - the counter may not keep climbing") + // NO order assertion, deliberately: Fastest builds no merge, so arrival interleaving is + // whatever the two racing consumers produced - asserting any order would pin luck. + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() // no ordering layer, so nothing to confess + DeliveredMessages.assertExactly(pausedExportedSet(cs), liveSeeds ++ liveTail, "A-F delivered multiset") + } + + test("CS-DM-AB [ALL LIVE x Best effort]: every message exactly once, in global publish-time order") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveSeeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), bestEffortLabel) + cs.play() + cs.assertState("running") + + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveTail) + cs.awaitLoaded(liveSeeds.size + liveTail.size) + assertSettledLoaded(cs, liveSeeds.size + liveTail.size, "Best effort must deliver exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() // an ordered live run confesses no out-of-order deliveries + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, liveSeeds ++ liveTail, "A-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), liveSeeds ++ liveTail, "A-B delivered multiset") + } + + test("CS-DM-AG [ALL LIVE x Guaranteed]: live traffic arrives in CHUNKS - Play's recorded range replays, later traffic waits for the next boundary") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveSeeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), guaranteedLabel) + cs.play() + // Chunk 1 is everything recorded up to Play: all six seeds - including the globally-last one, + // which the old contract held forever - then the boundary auto-pause with the banner. + awaitCaughtUp(cs, liveSeeds.size) + // (The chip label that used to name the replay here was removed 2026-08-11 with the always-on + // mode text; the caught-up banner awaitCaughtUp asserts IS the replay's announcement.) + + // Live traffic published after Play is recorded PAST the boundary: none of it may deliver + // into this chunk, however long it sits on the broker. + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveTail) + assertSettledLoaded(cs, liveSeeds.size, "post-Play traffic belongs to the NEXT chunk - nothing may leak past the boundary") + DeliveredMessages.assertExactly(cs.exportedValues(), liveSeeds, "A-G first-chunk multiset") + + // "Load new messages" extends the boundary to now: the whole tail replays as chunk 2, exactly once, and + // the session catches up again. + loadNewMessages.click() + awaitCaughtUp(cs, liveSeeds.size + liveTail.size) + assertSettledLoaded(cs, liveSeeds.size + liveTail.size, "the delta must replay exactly once") + assertNoSeamViolations() + assertArrivalOrder(cs, liveSeeds ++ liveTail, "A-G arrival order across both chunks") + DeliveredMessages.assertExactly(cs.exportedValues(), liveSeeds ++ liveTail, "A-G delivered multiset") + } + + // ============================================================================================= + // CELL B - ALL STATIC. One 3-partition topic: the one-click "browse this topic" shape where the + // delivery-mode decision bites hardest, and a partitioned topic is several delivery streams + // without any multi-topic setup. Backlog is spread round-robin across the partition FQNs with + // ticked sends, so every partition holds part of it (asserted) and publish times are globally + // distinct. + // ============================================================================================= + + private val staticBacklog = (0 until 12).map(i => f"m-$i%02d") + + /** Fresh 3-partition topic with `staticBacklog` spread p0,p1,p2,p0,... - deterministic slices: + * p0 = indexes 0,3,6,9; p1 = 1,4,7,10; p2 = 2,5,8,11. The premise (every partition non-empty) + * is asserted against the broker rather than trusted. */ + private def staticPartitionedTopic(): (String, String, String, String) = + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.PersistentPartitioned) + fixtures.produceRoundRobin(partitionFqns(fqn, 3), staticBacklog) + partitionFqns(fqn, 3).foreach { p => + val held = fixtures.readAllMessages(p).map(_.getValue) + assert(held.size == staticBacklog.size / 3, s"the arrangement did not land on $p: $held") + } + (t, ns, topic, fqn) + + private def openPartitionedSession(t: String, ns: String, topic: String, mode: String): ConsumerSessionPage = + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(mode) + cs + + test("CS-DM-BF [ALL STATIC x Fastest]: the whole backlog arrives exactly once") { + val (t, ns, topic, _) = staticPartitionedTopic() + val cs = openPartitionedSession(t, ns, topic, fastestLabel) + cs.play() + cs.awaitLoaded(staticBacklog.size) + assertSettledLoaded(cs, staticBacklog.size, "Fastest must deliver the backlog exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + // NO order assertion: Fastest promises none (see cell A-F). + DeliveredMessages.assertExactly(pausedExportedSet(cs), staticBacklog, "B-F delivered multiset") + } + + test("CS-DM-BB [ALL STATIC x Best effort]: the static backlog COMPLETES, in global publish-time order - the grace releases every tail") { + val (t, ns, topic, _) = staticPartitionedTopic() + val cs = openPartitionedSession(t, ns, topic, bestEffortLabel) + cs.play() + // THE static-tail pin: nothing new will ever arrive, and Best effort must still deliver + // everything - each partition's tail is released by the bounded reorder grace, not by traffic. + cs.awaitLoaded(staticBacklog.size) + assertSettledLoaded(cs, staticBacklog.size, "Best effort must deliver the backlog exactly once") + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, staticBacklog, "B-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), staticBacklog, "B-B delivered multiset") + } + + test("CS-DM-BG [ALL STATIC x Guaranteed]: the WHOLE backlog replays - no held tail - and the session auto-pauses caught up") { + val (t, ns, topic, _) = staticPartitionedTopic() + val cs = openPartitionedSession(t, ns, topic, guaranteedLabel) + cs.play() + // The replay contract on the shape the old contract punished hardest: p0 draining after m-09 + // holds NOTHING back - m-10 and m-11 are recorded history and need no proof from p0's future. + // All 12 deliver, then the boundary auto-pause with the banner on the ordinary paused state. + awaitCaughtUp(cs, staticBacklog.size) + // No stall was ever disclosed, because no stall exists to disclose. + assertThat(waitingChip).not().isVisible() + assertSettledLoaded(cs, staticBacklog.size, "the replay must deliver the backlog exactly once and then stop at the boundary") + // Banner content on a topic with NOTHING past the boundary: the newer-entries line and the + // excluded-topics line say nothing, because a zero is silence, not a claim (R1 pins the + // positive halves). + assertThat(bannerExcludedLine).not().isVisible() + assertNoSeamViolations() + assertArrivalOrder(cs, staticBacklog, "B-G arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), staticBacklog, "B-G delivered multiset") + } + + // ============================================================================================= + // CELL C - MIXED. Two topics: S stopped before Play (its share of the backlog is its last + // word), L keeps receiving during the session. Backlog round-robin S,L,S,L... so S = even + // indexes (b-00..b-06), L = odd (b-01..b-07). + // ============================================================================================= + + private val mixedBacklog = (0 until 8).map(i => f"b-$i%02d") + private val mixedLive = (0 until 6).map(i => f"l-$i%02d") + + test("CS-DM-CF [MIXED x Fastest]: backlog of both plus the live tail, exactly once") { + val (fqnS, fqnL, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnS, fqnL), mixedBacklog) + val cs = openTwoTopicSession(parts, Seq(fqnS, fqnL), fastestLabel) + cs.play() + cs.assertState("running") + fixtures.produceRoundRobin(Seq(fqnL), mixedLive) // only L is live + cs.awaitLoaded(mixedBacklog.size + mixedLive.size) + assertSettledLoaded(cs, mixedBacklog.size + mixedLive.size, "Fastest must deliver exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + // NO order assertion: Fastest promises none. + DeliveredMessages.assertExactly(pausedExportedSet(cs), mixedBacklog ++ mixedLive, "C-F delivered multiset") + } + + test("CS-DM-CB [MIXED x Best effort]: the stopped topic's tail completes and the live topic keeps flowing, all in order") { + val (fqnS, fqnL, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnS, fqnL), mixedBacklog) + val cs = openTwoTopicSession(parts, Seq(fqnS, fqnL), bestEffortLabel) + cs.play() + // The static half's tail (b-07 has no S successor) must complete on the grace alone. + cs.awaitLoaded(mixedBacklog.size) + fixtures.produceRoundRobin(Seq(fqnL), mixedLive) + cs.awaitLoaded(mixedBacklog.size + mixedLive.size) + assertSettledLoaded(cs, mixedBacklog.size + mixedLive.size, "Best effort must deliver exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, mixedBacklog ++ mixedLive, "C-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), mixedBacklog ++ mixedLive, "C-B delivered multiset") + } + + test("CS-DM-CG [MIXED x Guaranteed]: the stopped topic holds nothing back - the recorded range replays whole; the live tail is the next chunk") { + val (fqnS, fqnL, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnS, fqnL), mixedBacklog) + val cs = openTwoTopicSession(parts, Seq(fqnS, fqnL), guaranteedLabel) + cs.play() + // The whole recorded range delivers - b-07 included, which the old contract held behind S's + // silence - then the boundary auto-pause. + awaitCaughtUp(cs, mixedBacklog.size) + assertThat(waitingChip).not().isVisible() + + // L keeps speaking - PAST the boundary, so none of it may appear in this chunk. + fixtures.produceRoundRobin(Seq(fqnL), mixedLive) + assertSettledLoaded(cs, mixedBacklog.size, "post-Play traffic on L belongs to the next chunk") + DeliveredMessages.assertExactly(cs.exportedValues(), mixedBacklog, "C-G first-chunk multiset") + + // "Load new messages": the boundary extends to now and the accumulated delta replays exactly once. + loadNewMessages.click() + awaitCaughtUp(cs, mixedBacklog.size + mixedLive.size) + assertSettledLoaded(cs, mixedBacklog.size + mixedLive.size, "the delta must replay exactly once") + assertNoSeamViolations() + assertArrivalOrder(cs, mixedBacklog ++ mixedLive, "C-G arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), mixedBacklog ++ mixedLive, "C-G delivered multiset") + } + + // ============================================================================================= + // CELL D1 - EDGE: partitions EMPTY FROM BIRTH. One 3-partition topic, backlog on p0 only; + // p1/p2 have never held a message (asserted). Under the old contract this was the + // out-of-the-box stall; under the replay contract an empty recorded range is TRIVIALLY + // FINISHED, so the empty partitions cost nothing at all. + // ============================================================================================= + + private val d1Backlog = (0 until 8).map(i => f"b-$i%02d") + + private def emptyPartitionTopic(): (String, String, String, String) = + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.PersistentPartitioned) + fixtures.produceRoundRobin(Seq(s"$fqn-partition-0"), d1Backlog) + Seq(1, 2).foreach { i => + val held = fixtures.readAllMessages(s"$fqn-partition-$i") + assert(held.isEmpty, s"partition $i must be empty from birth, holds ${held.map(_.getValue)}") + } + (t, ns, topic, fqn) + + test("CS-DM-D1F [EMPTY PARTITIONS x Fastest]: empty partitions cost nothing - the backlog arrives exactly once") { + val (t, ns, topic, _) = emptyPartitionTopic() + val cs = openPartitionedSession(t, ns, topic, fastestLabel) + cs.play() + cs.awaitLoaded(d1Backlog.size) + assertSettledLoaded(cs, d1Backlog.size, "Fastest must deliver the backlog exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + // NO order assertion: Fastest promises none. + DeliveredMessages.assertExactly(pausedExportedSet(cs), d1Backlog, "D1-F delivered multiset") + } + + test("CS-DM-D1B [EMPTY PARTITIONS x Best effort]: an empty partition cannot hold delivery - everything arrives in order") { + val (t, ns, topic, _) = emptyPartitionTopic() + val cs = openPartitionedSession(t, ns, topic, bestEffortLabel) + cs.play() + cs.awaitLoaded(d1Backlog.size) + assertSettledLoaded(cs, d1Backlog.size, "Best effort must deliver the backlog exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, d1Backlog, "D1-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d1Backlog, "D1-B delivered multiset") + } + + test("CS-DM-D1G [EMPTY PARTITIONS x Guaranteed]: empty partitions are trivially finished - the backlog replays in full, no stall, caught up") { + val (t, ns, topic, fqn) = emptyPartitionTopic() + val cs = openPartitionedSession(t, ns, topic, guaranteedLabel) + cs.play() + // The old contract stalled here forever, showing nothing; the replay contract's answer: a + // stream that recorded nothing has nothing to replay and is never waited for. The whole + // backlog delivers, and the session catches up promptly. + awaitCaughtUp(cs, d1Backlog.size) + assertThat(waitingChip).not().isVisible() + assertSettledLoaded(cs, d1Backlog.size, "the empty partitions must not hold or duplicate anything") + + // One ticked word per partition INTO the pause (r-00 lands on p0, r-01/r-02 give the + // never-spoken partitions their first words) - all three are past the boundary, so all three + // wait for the next chunk together. + val release = (0 until 3).map(i => f"r-$i%02d") + fixtures.produceRoundRobin(partitionFqns(fqn, 3), release) + assertSettledLoaded(cs, d1Backlog.size, "pause-window words belong to the next chunk") + + // "Load new messages": all three deliver - the formerly-empty partitions join the replay like any other + // stream, in key order. + loadNewMessages.click() + awaitCaughtUp(cs, d1Backlog.size + release.size) + assertSettledLoaded(cs, d1Backlog.size + release.size, "the delta must replay exactly once") + assertNoSeamViolations() + assertArrivalOrder(cs, d1Backlog ++ release, "D1-G arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d1Backlog ++ release, "D1-G delivered multiset") + } + + // ============================================================================================= + // CELL D2 - EDGE: a static topic BECOMES LIVE again. Two topics, four seeds round-robin + // (A = m-00/m-02, B = m-01/m-03), then ONE more message to the topic that had finished. For + // the replay contract this is the chunk rule at its sharpest: the late word is recorded PAST + // the boundary, so it lands in the NEXT chunk - banner first, then "Load new messages" delivers it. + // ============================================================================================= + + private val d2Seeds = (0 until 4).map(i => f"m-$i%02d") + + test("CS-DM-D2F [STATIC-BECOMES-LIVE x Fastest]: the late word arrives like any other - everything exactly once") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d2Seeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), fastestLabel) + cs.play() + cs.awaitLoaded(d2Seeds.size) + fixtures.produceRoundRobin(Seq(fqnA), Seq("x-00")) + cs.awaitLoaded(d2Seeds.size + 1) + assertSettledLoaded(cs, d2Seeds.size + 1, "Fastest must deliver exactly once") + // NO order assertion: Fastest promises none. + DeliveredMessages.assertExactly(pausedExportedSet(cs), d2Seeds :+ "x-00", "D2-F delivered multiset") + } + + test("CS-DM-D2B [STATIC-BECOMES-LIVE x Best effort]: the finished topic's next word arrives promptly and in order") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d2Seeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), bestEffortLabel) + cs.play() + cs.awaitLoaded(d2Seeds.size) // the static tail completed on the grace alone + fixtures.produceRoundRobin(Seq(fqnA), Seq("x-00")) + cs.awaitLoaded(d2Seeds.size + 1) // and the new word itself is released by the grace, not by traffic + assertSettledLoaded(cs, d2Seeds.size + 1, "Best effort must deliver exactly once") + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, d2Seeds :+ "x-00", "D2-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d2Seeds :+ "x-00", "D2-B delivered multiset") + } + + test("CS-DM-D2G [STATIC-BECOMES-LIVE x Guaranteed]: the late word lands in the NEXT chunk - banner first, then 'Load new messages' delivers it, order intact") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d2Seeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), guaranteedLabel) + cs.play() + // All four seeds replay - m-03 included, which the old contract held against A's silence - + // then the boundary auto-pause. + awaitCaughtUp(cs, d2Seeds.size) + assertThat(waitingChip).not().isVisible() + + // The finished topic speaks again - PAST the boundary. The word is recorded, safe, and NOT + // delivered into this chunk. + fixtures.produceRoundRobin(Seq(fqnA), Seq("x-00")) + assertSettledLoaded(cs, d2Seeds.size, "the late word belongs to the next chunk") + + loadNewMessages.click() + awaitCaughtUp(cs, d2Seeds.size + 1) + assertSettledLoaded(cs, d2Seeds.size + 1, "the late word must deliver exactly once on 'Load new messages'") + assertNoSeamViolations() + assertArrivalOrder(cs, d2Seeds :+ "x-00", "D2-G arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d2Seeds :+ "x-00", "D2-G delivered multiset") + } + + // ============================================================================================= + // CELL D3 - EDGE: a SINGLE non-partitioned topic. One delivery stream needs no MERGE - but the + // replay boundary is not a merge property: Guaranteed builds its layer at ANY stream count, + // because the boundary, the auto-pause and the caught-up signal live in it. So Fastest and + // Best effort stay layer-free (no chip), while a static single topic under Guaranteed replays + // and pauses caught-up like any other shape. + // ============================================================================================= + + private val d3Backlog = (0 until 6).map(i => f"m-$i%02d") + + private def singleStaticTopic(): (String, String, String) = + val parts = fixtures.freshTopicParts() + fixtures.produceRoundRobin(Seq(s"persistent://${parts._1}/${parts._2}/${parts._3}"), d3Backlog) + parts + + private def openSingleTopicSession(parts: (String, String, String), mode: String): ConsumerSessionPage = + val cs = ConsumerSessionPage(page) + cs.openForTopic(parts._1, parts._2, parts._3) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(mode) + cs + + test("CS-DM-D3F [SINGLE TOPIC x Fastest]: the static backlog arrives exactly once, no layer, no chip") { + val cs = openSingleTopicSession(singleStaticTopic(), fastestLabel) + cs.play() + cs.awaitLoaded(d3Backlog.size) + assertSettledLoaded(cs, d3Backlog.size, "the backlog must deliver exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + // NO order assertion: Fastest promises none, even where a single log would happen to provide it. + DeliveredMessages.assertExactly(pausedExportedSet(cs), d3Backlog, "D3-F delivered multiset") + } + + test("CS-DM-D3B [SINGLE TOPIC x Best effort]: one stream builds no layer - delivery is append order, nothing to confess") { + val cs = openSingleTopicSession(singleStaticTopic(), bestEffortLabel) + cs.play() + cs.awaitLoaded(d3Backlog.size) + assertSettledLoaded(cs, d3Backlog.size, "the backlog must deliver exactly once") + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() // no layer exists, nothing may claim one (CS-MO-8's contract, per mode) + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, d3Backlog, "D3-B arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d3Backlog, "D3-B delivered multiset") + } + + test("CS-DM-D3G [SINGLE TOPIC x Guaranteed]: one stream still gets the replay boundary - the backlog replays, then the caught-up pause") { + val cs = openSingleTopicSession(singleStaticTopic(), guaranteedLabel) + cs.play() + // The boundary is per-stream and positional, so it exists at ANY stream count: the single + // static log replays completely and the session auto-pauses at its recorded end. + awaitCaughtUp(cs, d3Backlog.size) + // The boundary layer is real even for one stream: the caught-up banner asserted by + // awaitCaughtUp is its announcement (the always-on chip label was removed 2026-08-11). + assertSettledLoaded(cs, d3Backlog.size, "the backlog must deliver exactly once") + assertNoSeamViolations() + assertArrivalOrder(cs, d3Backlog, "D3-G arrival order") + DeliveredMessages.assertExactly(cs.exportedValues(), d3Backlog, "D3-G delivered multiset") + } + + // ============================================================================================= + // CELL D4 - EDGE: Guaranteed with a NON-PERSISTENT stream in a multi-stream session. A + // non-persistent topic records nothing, so a multi-stream replay over it cannot mean anything: + // the server refuses to build it, up front, with the remediation in the message. (A SINGLE + // non-persistent stream is legal - instant caught-up, pinned by CS-MO-0B; Best effort on the + // same mixed shape works - pinned by CS-MO-6.) + // ============================================================================================= + + test("CS-DM-D4G [NON-PERSISTENT x Guaranteed]: a mixed-persistency multi-stream session is REFUSED by name, and nothing is delivered") { + val (tP, nsP, topicP) = fixtures.freshTopicParts() + val fqnP = s"persistent://$tP/$nsP/$topicP" + fixtures.produceRoundRobin(Seq(fqnP), (0 until 4).map(i => f"m-$i%02d")) + val (_, _, _, fqnN) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.NonPersistentNonPartitioned) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tP, nsP, topicP) + cs.setTargetTopicsSpecific(Seq(fqnP, fqnN)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.play() + + // How the refusal surfaces: the server rejects the session CREATE (before any consumer is + // built), and the client shows the server's reason - which names the offending topics and + // both remediations. + assertThat(page.getByText("Guaranteed ordering is not supported when merging non-persistent topics").first()) + .isVisible(vis(15000)) + assert(cs.loadedCount == 0, s"a refused session must deliver nothing, got ${cs.loadedCount}") + assertSettledLoaded(cs, 0, "a refused session must stay silent") + // Observed 2026-08-09 and pinned: the refusal leaves the session in 'new' - still on the + // configuration view, so the user corrects the mode (or the targets) and presses Play again; + // no consumer was ever built, nothing to stop or clean up. + cs.assertState("new") + } + + // ============================================================================================= + // CELL D5 - EDGE: the ONE-CLICK SWITCH, from the caught-up pause. The caught-up banner offers + // "Switch to Best effort and follow live" - the designed transition out of the replay into live + // following (SetDeliveryOrder on the live session; the only broker-level exercise of that + // RPC). The pause-window traffic and everything after must arrive exactly once, and the FIRST + // row delivered after the switch carries the divider badge (`cs-order-switch-point`) that + // marks where the exact replay ends and the bounded-reorder live following begins. + // ============================================================================================= + + test("CS-DM-D5G [SWITCH]: 'Switch to Best effort and follow live' resumes live delivery from the caught-up pause - marked at the switch point, nothing lost or doubled") { + val (fqnA, fqnB, parts) = twoFreshTopics() + val recorded = (0 until 12).map(i => f"m-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), recorded) + + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), guaranteedLabel) + cs.play() + // Chunk 1: the whole recorded range, then the boundary pause. + awaitCaughtUp(cs, recorded.size) + + // Traffic into the pause window: recorded past the boundary, waiting. + val pauseWindow = (0 until 4).map(i => f"w-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), pauseWindow) + assertSettledLoaded(cs, recorded.size, "pause-window traffic waits behind the boundary") + + // THE SWITCH. The granted SetDeliveryOrder releases the boundary holds and the session + // follows live traffic under Best effort from here on. + bannerContinueBestEffort.click() + cs.awaitLoaded(recorded.size + pauseWindow.size) + assertThat(caughtUpBanner).not().isVisible() // the replay state died with the barrier + + // And it IS live following now: a fresh round arrives promptly, nothing to click. + val liveRound = (0 until 6).map(i => f"p-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), liveRound) + val all = recorded ++ pauseWindow ++ liveRound + cs.awaitLoaded(all.size) + assertSettledLoaded(cs, all.size, "exactly once across the switch - no re-read, no duplicate") + // The proof the session now RUNS Best effort is the switch-point divider on the first + // post-switch row (asserted below) - the chip label that used to read the mode back was + // removed 2026-08-11 with the always-on mode text. + // NOT asserted here: the switch also writes Best effort back into the session CONFIG (so the + // next Play keeps it). The config select only exists on the pre-Play configuration view + // (`currentView` is 'configuration' only while the session is 'new'), so a running session + // cannot show it; that write-through is pinned at the jest tier + // (ConsumerSession.test.ts, the applyDeliveryOrder suite). + + // The toolbar rerenders every second (the live gauges), and one observed run lost this click + // to a node swap (state stayed 'running', no 'pausing' ever recorded) - so the pause request + // is re-issued while the state machine has provably not moved. Never clicked in 'pausing': + // that click would be a resume. + eventually(timeoutMs = 10000, intervalMs = 1500) { + if cs.state == "running" then cs.pauseFromToolbar() + assert(cs.state != "running", "the pause request has not been accepted yet") + } + cs.assertState("paused", timeoutMs = 10000) + // THE SWITCH POINT: exactly one divider badge, attached to the FIRST row delivered after the + // switch - the first pause-window message. Rows above it are the exact replay; rows below + // follow live traffic. + val badgeRows = switchPointRowValues() + assert( + badgeRows == List(pauseWindow.head), + s"expected exactly one switch-point badge, on '${pauseWindow.head}' (the first row delivered after the switch), got $badgeRows" + ) + assertNoSeamViolations() + assertArrivalOrder(cs, all, "D5 arrival order (the release preserves publish-time order)") + DeliveredMessages.assertExactly(cs.exportedValues(), all, "D5 delivered multiset") + } + + // ============================================================================================= + // CELL D6 - EDGE: OWNERSHIP CHURN mid-session. `admin.topics().unload` on ONE stream's topic + // makes the broker drop and re-own it: the session's consumer for that stream silently + // disconnects, reconnects, and everything received-but-unacknowledged is REDELIVERED - on a + // multi-broker cluster this is what a topic ownership move does, and the standalone reproduces + // the same client-visible physics. The pin under the replay contract: a redelivered message + // must count against the SAME replay range - never re-shown (deduped), never lost, and never + // able to re-open a finished stream or double-extend the chunk. Fastest is skipped - it makes + // no ordering claim for churn to break; its exactly-once under pressure is covered by the + // flow-control specs. + // ============================================================================================= + + private val d6Seeds = (0 until 12).map(i => f"m-$i%02d") + private val d6Round = (0 until 8).map(i => f"r-$i%02d") + + /** Consumer connection stamp for the session's subscription on `fqn` - the observable a broker + * unload changes when the consumer really reconnects. Empty while no consumer is attached. */ + private def consumerConnectedSince(fqn: String): List[String] = + import scala.jdk.CollectionConverters.* + admin.topics().getStats(fqn).getSubscriptions.values().asScala.toList + .flatMap(_.getConsumers.asScala.toList.map(_.getConnectedSince)) + + /** Poll the loaded counter until it crosses `n` - mid-replay there is no exact value to wait + * for, only a threshold proving the drain is in flight. */ + private def awaitLoadedAtLeast(cs: ConsumerSessionPage, n: Int, timeoutMs: Long): Unit = + eventually(timeoutMs = timeoutMs, intervalMs = 100) { + val current = cs.loadedCount + assert(current >= n, s"loaded $current, expected at least $n") + } + + test("CS-DM-D6G [CHURN x Guaranteed]: an unload mid-replay redelivers - every copy is deduped against the SAME replay range, and the chunk stays exact") { + // The unload must land while the replay is still draining. Block size alone cannot hold that + // window open: 6,000 messages drained in under a second on a warm standalone (observed + // 2026-08-12 - the auto-pause beat the first assertState poll), so the window is held the way + // R1 holds its announce off: through the product's own delivery rate limit. 6,000 recorded + // messages at 500/s is a ~12 s drain wherever the pipeline's raw speed goes next; x's + // dispatch (below) lands ~8 s in, and the unload strikes a provably LIVE replay - with B's + // receiver queue full of dispatched-but-unemitted in-range messages, which is what gives the + // dedup assertion real redelivered copies to reject. Fast-produced: order is not asserted in + // this cell (ties across the two producers), identity is. + val (fqnD6A, fqnD6B, parts) = twoFreshTopics() + val seedsA = fixtures.produceStringsFast(fqnD6A, 3000, prefix = "a") + val seedsB = fixtures.produceStringsFast(fqnD6B, 3000, prefix = "b") + + val cs = openTwoTopicSession(parts, Seq(fqnD6A, fqnD6B), guaranteedLabel) + cs.setRateLimit(500) + cs.play() + cs.assertState("running") + + // The deterministic un-acked message: a word PAST the boundary, produced while the replay + // drains (the 6,000-message drain outlasts one send by seconds). Pulsar dispatches in order, + // so by the time x reaches the client every in-range B message is already in the session's + // hands - the boundary nack then hands x back UN-ACKED and holds B's consumer, stranding + // nothing. In-range messages cannot play this role: the merge acks them as it emits, and + // emission runs far ahead of the browser-visible counter (observed 2026-08-09, drafts of + // this cell: all 3,000 of B's seeds were dispatched AND acked within the first second, so a + // mid-drain unload found nothing un-acked to redeliver). + fixtures.produceRoundRobin(Seq(fqnD6B), Seq("x-next-chunk")) + // x's first dispatch and refusal, observed on the broker before the churn: the counter + // crossing seedsB + 1 proves the word reached the session once and was nacked (it may not + // appear on screen), which is what makes its eventual arrival below a REdelivery. + eventually(timeoutMs = 60000, intervalMs = 200) { + val dispatched = fixtures.subscriptionDispatchCount(fqnD6B) + assert(dispatched >= seedsB.size + 1, s"x has not been dispatched yet: B dispatched $dispatched of ${seedsB.size + 1}") + } + + // CHURN: unload B. The broker closes the consumer, re-owns the topic, and the consumer + // reconnects - with its boundary hold intact (a held consumer issues no permits), so the + // un-acked x stays parked on the broker for the next chunk. + val before = consumerConnectedSince(fqnD6B) + admin.topics().unload(fqnD6B) + // The reconnect, observed rather than assumed: a consumer is attached again with a NEW + // connection stamp. (If this ever fails, the standalone stopped churning on unload and the + // cell is environment-limited - say so, do not delete the assertion.) + eventually(timeoutMs = 30000, intervalMs = 300) { + val now = consumerConnectedSince(fqnD6B) + assert(now.nonEmpty && now != before, s"no reconnected consumer on $fqnD6B yet: before=$before now=$now") + } + + // The replay completes EXACTLY across the churn: 6,000 - any copy the churn re-sent is + // deduped against the same range (6,001 would name a double-delivery; a timeout names a + // loss or a re-opened stream) - then the boundary pause, with x still outside. + awaitCaughtUp(cs, seedsA.size + seedsB.size, timeoutMs = 90000) + assertSettledLoaded(cs, seedsA.size + seedsB.size, "exactly once across the reconnect - and the nacked word stays out of the closed chunk") + + // The redelivery bracket: nothing new is produced to B from here, so every dispatch B's + // subscription records after this instant is a REdelivery of the un-acked x. (The unload + // RESETS the broker's per-subscription msgOutCounter - observed 2026-08-09 - which is why + // the baseline is read here, post-reload, rather than differenced across the unload.) + val dispatchedBeforeResume = fixtures.subscriptionDispatchCount(fqnD6B) + + // 'Load new messages' extends the boundary: the held consumer releases, the broker redelivers x across + // the churn, and it arrives exactly once. + loadNewMessages.click() + awaitCaughtUp(cs, seedsA.size + seedsB.size + 1, timeoutMs = 45000) + assertSettledLoaded(cs, seedsA.size + seedsB.size + 1, "the next-chunk word must survive the churn and deliver exactly once") + val redelivered = fixtures.subscriptionDispatchCount(fqnD6B) - dispatchedBeforeResume + assert( + redelivered >= 1, + s"x arrived without a post-resume dispatch on B - the redelivery path was not exercised " + + s"(B dispatched $dispatchedBeforeResume before the resume and ${fixtures.subscriptionDispatchCount(fqnD6B)} after)" + ) + assertNoSeamViolations() + DeliveredMessages.assertExactly( + cs.exportedValues(), + seedsA ++ seedsB ++ Seq("x-next-chunk"), + "D6-G delivered multiset (every message exactly once across the churn)" + ) + } + + test("CS-DM-D6B [CHURN x Best effort]: the stream survives an unload - everything before and after arrives exactly once, in order") { + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d6Seeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), bestEffortLabel) + cs.play() + cs.awaitLoaded(d6Seeds.size) + + // Under Best effort nothing is deterministically unacknowledged at steady state (the grace + // has released everything), so the cell pins churn's cost on the FLOW: what passes through a + // reconnecting stream must arrive exactly once and stay ordered. Unload A - either works. + val before = consumerConnectedSince(fqnA) + admin.topics().unload(fqnA) + eventually(timeoutMs = 30000, intervalMs = 300) { + val now = consumerConnectedSince(fqnA) + assert(now.nonEmpty && now != before, s"no reconnected consumer on $fqnA yet: before=$before now=$now") + } + + // The probe proves the reconnected stream DELIVERS again before the ordered round rides it - + // gating on the oracle, not on reconnect timing. + fixtures.produceRoundRobin(Seq(fqnA), Seq("p-00")) + cs.awaitLoaded(d6Seeds.size + 1, timeoutMs = 45000) + + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d6Round) + val all = d6Seeds ++ Seq("p-00") ++ d6Round + cs.awaitLoaded(all.size, timeoutMs = 45000) + assertSettledLoaded(cs, all.size, "exactly once across the reconnect") + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 10000) + assertArrivalOrder(cs, all, "D6-B arrival order across the reconnect") + DeliveredMessages.assertExactly(cs.exportedValues(), all, "D6-B delivered multiset") + } + + test("CS-DM-D6S [CHURN x Best effort x skip-n]: an unload mid-skip does not corrupt the counted budget - the survivors are exact") { + // Skip 14 of 12: the discard spends the whole seeded backlog and then WAITS for two more - + // which parks the session provably mid-skip (dispatched >= 12, loaded == 0) so the unload + // deterministically lands while the budget is live. If the churn redelivers anything the + // discard already decided (acks racing the unload), the budget must not spend a second unit + // on the copy: the survivors would then be wrong by name, not just by count. + val (fqnA, fqnB, parts) = twoFreshTopics() + fixtures.produceRoundRobin(Seq(fqnA, fqnB), d6Seeds) + + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), bestEffortLabel) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("14") + cs.play() + cs.assertState("running") + // Provably mid-skip: every seed has been dispatched to the session, nothing has been shown. + eventually(timeoutMs = 30000, intervalMs = 300) { + val dispatched = fixtures.subscriptionDispatchCount(fqnA) + fixtures.subscriptionDispatchCount(fqnB) + assert(dispatched >= d6Seeds.size, s"the skip has not consumed the seeds yet: dispatched=$dispatched") + } + assert(cs.loadedCount == 0, s"the skip must still be counting, got ${cs.loadedCount} loaded") + + val before = consumerConnectedSince(fqnA) + admin.topics().unload(fqnA) + eventually(timeoutMs = 30000, intervalMs = 300) { + val now = consumerConnectedSince(fqnA) + assert(now.nonEmpty && now != before, s"no reconnected consumer on $fqnA yet: before=$before now=$now") + } + + // Four more ticked words: r-00/r-01 spend the budget's last two units, r-02/r-03 survive. + val tail = (0 until 4).map(i => f"r-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), tail) + cs.awaitLoaded(2, timeoutMs = 45000) + assertSettledLoaded(cs, 2, "exactly two survivors - a double-spent or under-spent budget shows here") + DeliveredMessages.assertExactly(pausedExportedSet(cs), tail.drop(2), "D6-S surviving multiset") + } + + // ============================================================================================= + // CELL R - THE REPLAY CONTRACT'S OWN CELLS: what the redesign added beyond the matrix's + // liveness axes - the banner's content, the repeatable chunk mechanism, and the one + // combination the replay makes meaningless. + // ============================================================================================= + + test("CS-DM-R1 [REPLAY BANNER]: the caught-up banner names the boundary instant, and discloses newer data - approximately - only because there is some") { + // The newer-data indicator is captured AT the caught-up announce (the boundary nacks it has + // seen, refined once by a broker poll right after), so the post-boundary words must land + // BEFORE the announce. Raw emission can finish a small backlog in well under a second - one + // run lost that race - so the drain is throttled through the product's own delivery rate + // limit: 3,000 recorded messages at 500/s hold the announce off for ~6 seconds, and the + // words (one producer connect + five ticked sends, well under a second after Play) land + // mid-drain with seconds to spare. The boundary itself was captured at the seek, before + // play() returned, so the words are past it wherever they land in the drain. + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val backlog = fixtures.produceStringsFast(fqn, 3000, prefix = "bk") + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.setRateLimit(500) + cs.play() + // Post-boundary words, landed while the replay drains: the boundary nack (and the one-shot + // broker poll at the announce) is what feeds the banner's newer-data line. + val newer = (0 until 5).map(i => f"w-$i%02d") + fixtures.produceRoundRobin(Seq(fqn), newer) + + awaitCaughtUp(cs, backlog.size, timeoutMs = 60000) + // The boundary instant, formatted: "Caught up to Aug 09, 2026 14:03:22". + val bannerText = caughtUpBanner.textContent() + assert( + bannerText.matches("(?s).*Caught up to [A-Z][a-z]{2} \\d{2}, \\d{4} \\d{2}:\\d{2}:\\d{2}.*"), + s"the banner must name the boundary instant, got: '$bannerText'" + ) + // The panel no longer reports how much waits past the boundary (the "~N entries" line was + // removed 2026-08-11 - it counted broker entries, not messages). What the cell still pins is + // the CONTRACT that line described: the replay stops exactly at the boundary, and the words + // recorded past it wait for the next chunk. + // No regex matched a new topic, so the excluded-topics line says nothing. + assertThat(bannerExcludedLine).not().isVisible() + assertNoSeamViolations() + assertSettledLoaded(cs, backlog.size, "the replay stops at the boundary, whatever waits past it") + DeliveredMessages.assertExactly(cs.exportedValues(), backlog, "R1 delivered multiset (the boundary excludes the newer words)") + } + + test("CS-DM-R2 [LOAD NEW MESSAGES EXTENDS]: every 'Load new messages' extends the boundary and replays the delta exactly - the chunk mechanism is repeatable, not one-shot") { + val (fqnA, fqnB, parts) = twoFreshTopics() + val seeds = (0 until 6).map(i => f"m-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), seeds) + val cs = openTwoTopicSession(parts, Seq(fqnA, fqnB), guaranteedLabel) + cs.play() + awaitCaughtUp(cs, seeds.size) + + // Chunk 2: a delta produced INTO the pause, spread over both streams; 'Load new messages' replays it + // exactly once and the banner returns at the new boundary. + val delta1 = (0 until 6).map(i => f"d1-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), delta1) + assertSettledLoaded(cs, seeds.size, "the delta waits behind the boundary") + loadNewMessages.click() + awaitCaughtUp(cs, seeds.size + delta1.size) + assertSettledLoaded(cs, seeds.size + delta1.size, "chunk 2 must deliver exactly once") + + // Chunk 3: the same verbs again - the mechanism must not be a one-shot. + val delta2 = (0 until 4).map(i => f"d2-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), delta2) + assertSettledLoaded(cs, seeds.size + delta1.size, "chunk 3 waits behind the extended boundary") + loadNewMessages.click() + awaitCaughtUp(cs, seeds.size + delta1.size + delta2.size) + assertSettledLoaded(cs, seeds.size + delta1.size + delta2.size, "chunk 3 must deliver exactly once") + + assertNoSeamViolations() + assertArrivalOrder(cs, seeds ++ delta1 ++ delta2, "R2 arrival order across three chunks") + DeliveredMessages.assertExactly(cs.exportedValues(), seeds ++ delta1 ++ delta2, "R2 delivered multiset") + } + + test("CS-DM-R3 [LATEST x GUARANTEED, ungated]: 'Latest message' stays selectable under Guaranteed in BOTH directions, with no note, and nothing is rewritten") { + // Owner decision 2026-08-11, reversing the 2026-08-09 disabled-option gate: the combination + // is legal, and what the user gets is the instant caught-up with its panel (R3B pins that + // flow) - the decision belongs there, not to a greyed-out option. This cell pins the un-gate: + // nothing disabled, nothing noted, nothing rewritten. + val (t, ns, topic) = fixtures.freshTopicParts() + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + + // Direction 1: Guaranteed first - "Latest message" stays selectable. + cs.setDeliveryOrder(guaranteedLabel) + assert( + !cs.disabledStartFromLabels.contains("Latest message"), + s"'Latest message' must stay selectable under Guaranteed, disabled set: ${cs.disabledStartFromLabels}" + ) + assertThat(page.getByTestId("cs-start-from-latest-guaranteed-note")).not().isVisible() + + // Direction 2: Latest first, then Guaranteed - the stored value is the user's (M4 lesson). + cs.setDeliveryOrder(bestEffortLabel) + cs.setStartFrom("Latest message") + cs.setDeliveryOrder(guaranteedLabel) + assertThat(cs.startFromSelect).hasValue("latestMessage") + assert( + !cs.disabledStartFromLabels.contains("Latest message"), + s"'Latest message' must stay selectable in the saved-combo direction too, disabled set: ${cs.disabledStartFromLabels}" + ) + } + + test("CS-DM-R3B [LATEST x GUARANTEED, the combination]: playing a saved Latest x Guaranteed combo is an INSTANT caught-up that reads nothing") { + // Untagged 2026-08-09, same day it was found: the first Play now CONSUMES the boundaries the + // create decided (a live-edge seek = an empty boundary) instead of re-reading the broker, so + // this regression joined the green lane. What it pins, permanently: the create is the first + // Play - the boundaries decided at create ARE the first chunk's - so a saved Latest x + // Guaranteed combination on a topic that HOLDS a backlog answers Play with the instant + // caught-up pause and reads nothing. (The bug it caught: the first-play boundary re-capture + // read every persistent topic's real last message ids with no knowledge of the create's + // `seekedToLiveEdge` set, re-arming the barrier over a range the cursor was seeked past - + // the session sat 'running' showing nothing, forever. A NON-EMPTY topic is therefore the + // load-bearing arrangement here; an empty one could not tell the fix from the bug.) + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + // A backlog that must stay UNREAD: Latest means the replay boundary is empty, so these four + // never reach the session in any chunk. + val unread = (0 until 4).map(i => f"old-$i%02d") + fixtures.produceRoundRobin(Seq(fqn), unread) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + // The saved-combo path: Latest first, then Guaranteed - the value is kept, not rewritten. + cs.setStartFrom("Latest message") + cs.setDeliveryOrder(guaranteedLabel) + assertThat(cs.startFromSelect).hasValue("latestMessage") + + cs.play() + // Instant caught-up: the boundary is empty, nothing replays - the backlog stays unread. + assertThat(caughtUpBanner).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + assert(cs.loadedCount == 0, s"Latest x Guaranteed replays nothing, got ${cs.loadedCount}") + + // The chunk verbs still work from here: words recorded during the pause become the next + // chunk's delta, and the pre-play backlog STAYS unread - Latest anchored the boundary past it. + val delta = (0 until 4).map(i => f"d-$i%02d") + fixtures.produceRoundRobin(Seq(fqn), delta) + assertSettledLoaded(cs, 0, "pause-window words wait for the next chunk") + loadNewMessages.click() + awaitCaughtUp(cs, delta.size) + assertSettledLoaded(cs, delta.size, "the delta must deliver exactly once - and the old backlog must stay out") + DeliveredMessages.assertExactly(cs.exportedValues(), delta, "R3B delivered multiset (the unread backlog is not in it)") + } diff --git a/e2e/src/test/scala/features/consumersession/CsDetailsSpec.scala b/e2e/src/test/scala/features/consumersession/CsDetailsSpec.scala index 06794228e..5fbf62d03 100644 --- a/e2e/src/test/scala/features/consumersession/CsDetailsSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsDetailsSpec.scala @@ -2,9 +2,53 @@ package features.consumersession import harness.DekafSuite import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions +import org.apache.pulsar.client.api.{Schema, SubscriptionInitialPosition} +import java.util.concurrent.TimeUnit import java.util.regex.Pattern +import scala.jdk.CollectionConverters.* class CsDetailsSpec extends DekafSuite: + private def hasText = new LocatorAssertions.HasTextOptions().setTimeout(20000) + private def containsText = new LocatorAssertions.ContainsTextOptions().setTimeout(20000) + private def count(n: Int) = new LocatorAssertions.HasCountOptions().setTimeout(20000) + + private final case class MessageViewport( + firstRenderedIndex: Int, + scrollTop: Double, + x: Double, + y: Double, + width: Double, + height: Double + ) + + private def messageViewport(cs: ConsumerSessionPage): MessageViewport = + val box = cs.tableScroller.boundingBox() + assert(box != null, "the virtualized message viewport has no live bounding box") + MessageViewport( + firstRenderedIndex = cs.firstRenderedIndex, + scrollTop = cs.tableScrollTop, + x = box.x, + y = box.y, + width = box.width, + height = box.height + ) + + private def assertViewportUnchanged(expected: MessageViewport, actual: MessageViewport, after: String): Unit = + assert(actual.firstRenderedIndex == expected.firstRenderedIndex, + s"the first rendered message jumped $after: ${expected.firstRenderedIndex} -> ${actual.firstRenderedIndex}") + assert(math.abs(actual.scrollTop - expected.scrollTop) <= 1, + s"the message list scroll offset jumped $after: ${expected.scrollTop} -> ${actual.scrollTop}") + + val geometryDelta = List( + "x" -> math.abs(actual.x - expected.x), + "y" -> math.abs(actual.y - expected.y), + "width" -> math.abs(actual.width - expected.width), + "height" -> math.abs(actual.height - expected.height) + ) + assert(geometryDelta.forall(_._2 <= 1), + s"the message viewport geometry changed $after: $expected -> $actual; deltas: $geometryDelta") + private def loadedPaused(n: Int): ConsumerSessionPage = val (t, ns, topic) = fixtures.freshTopicParts() fixtures.produceStrings(s"persistent://$t/$ns/$topic", n) @@ -17,6 +61,85 @@ class CsDetailsSpec extends DekafSuite: cs.assertState("paused") cs + // CS-23 used to assert only that the panel OPENS - a panel rendering an empty shell, or another + // message entirely, passed. It now pins the panel's CONTENT, tab by tab, against what the broker + // actually stored (read back with the Pulsar client) rather than against the UI's own table view. + test("CS-23: MessageDetails shows the published key, value, metadata and properties") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + + val key = "cs23-key" + val value = "cs23-value-payload" // no spaces: survives Monaco's tokenized rendering verbatim + val producerName = fixtures.unique("cs23-producer") + val properties = Map("cs23-prop-a" -> "alpha", "cs23-prop-b" -> "beta") + + val producer = client.newProducer(Schema.STRING).producerName(producerName).topic(fqn).create() + try + val builder = producer.newMessage().key(key).value(value) + properties.foreach { case (k, v) => builder.property(k, v) } + builder.send() + finally producer.close() + + // GROUND TRUTH: read the message back off the broker. Everything below is asserted against + // THIS, so a UI that renders its own stale copy (or the wrong message) can't pass. + val oracleConsumer = client.newConsumer(Schema.STRING) + .topic(fqn) + .subscriptionName(fixtures.unique("cs23-oracle")) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe() + val stored = + try + val m = oracleConsumer.receive(15, TimeUnit.SECONDS) + assert(m != null, "the oracle consumer received no message") + (m.getKey, m.getValue, m.getProducerName, m.getProperties.asScala.toMap) + finally oracleConsumer.close() + val (storedKey, storedValue, storedProducer, storedProps) = stored + assert( + stored == (key, value, producerName, properties), + s"the broker stored something else than we published: $stored" + ) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.waitMessages(1) + cs.pauseFromToolbar() + cs.assertState("paused") + + cs.clickFirstMessage() + assertThat(cs.messageDetails).isVisible() + + // --- Value tab (the default one) --- + // A STRING key/value crosses the wire JSON-encoded (messageConverters: `msg.getKey.asJson`), + // so the panel renders it JSON-quoted - assert that exact rendering, not a substring. + assertThat(cs.messageDetails.getByTestId("cs-cell-key")).hasText(s"\"$storedKey\"", hasText) + // The value itself is shown in a Monaco viewer (JsonView), so assert its rendered text. + assertThat(cs.messageDetails.locator(".monaco-editor")).containsText(storedValue, containsText) + + // --- Metadata tab --- + cs.messageDetails.getByTestId("cs-details-tab-metadata").click() + assertThat(cs.messageDetails.getByTestId("cs-cell-topic")).hasText(fqn, hasText) + assertThat(cs.messageDetails.getByTestId("cs-cell-key")).hasText(s"\"$storedKey\"", hasText) + // The producer name is broker-reported, and we pinned it on the producer - so it is a real + // metadata round-trip, not a value the UI could have echoed from the row it was clicked on. + assertThat(cs.messageDetails).containsText(storedProducer, containsText) + + // --- Properties tab --- + val propertiesTab = cs.messageDetails.getByTestId("cs-details-tab-properties") + assertThat(propertiesTab).hasText(s"Properties ${storedProps.size}", hasText) // count in the tab title + propertiesTab.click() + // Only the active tab is mounted, so these are exactly the read-only property key/value inputs. + val propertyInputs = cs.messageDetails.locator("input") + assertThat(propertyInputs).hasCount(storedProps.size * 2, count(storedProps.size * 2)) + val shown = propertyInputs.all().asScala.toList + .map(_.inputValue()) + .grouped(2) + .map(pair => pair.head -> pair.last) + .toMap + assert(shown == storedProps, s"the Properties tab showed $shown, the broker has $storedProps") + } + test("CS-24: MessageDetails closes only via its close button") { val cs = loadedPaused(3) cs.clickFirstMessage() @@ -32,6 +155,152 @@ class CsDetailsSpec extends DekafSuite: assertThat(cs.messageDetails).isHidden() } + test("CS-24R: MessageDetails width is resizable and persists across closing and reload") { + val cs = loadedPaused(3) + cs.openTools() + cs.clickFirstMessage() + assertThat(cs.messageDetailsResizeHandle).isVisible() + + // Preserve the former 600rem default (540 CSS px under Dekaf's 0.9px root scale). + val defaultWidth = cs.messageDetailsDomWidth + assert(math.abs(defaultWidth - 540) <= 3, + s"the resizable inspector changed its old 600rem default: $defaultWidth") + + // Restore an oversized preference, then recreate the runtime view. The grid caps it to 85% + // without rewriting storage, and the first drag must start from that VISIBLE capped width. + cs.setStoredPaneSize("consumer-session-message-inspector", 1080) + page.reload() + cs.setStartFrom("Earliest message") + cs.play() + cs.waitMessages(3) + cs.pauseFromToolbar() + cs.assertState("paused") + cs.clickFirstMessage() + assertThat(cs.messageDetailsResizeHandle).isVisible() + + val before = cs.messageDetailsDomWidth + val contentWidth = before + cs.tableDomWidth + assert(before >= contentWidth * 0.80 - 3, + s"inspector should be draggable well beyond the old 65% cap: $before of $contentWidth") + assert(before <= contentWidth * 0.85 + 3, + s"oversized inspector preference was not capped: $before of $contentWidth") + assert(cs.storedPaneSize("consumer-session-message-inspector") == 1080, + s"responsive cap must not overwrite the preference: ${cs.storedPaneSize("consumer-session-message-inspector")}") + + // Drive the inspector's left edge right. For a right-hand pane that must SHRINK its live width. + cs.resizeMessageDetailsBy(100) + val resized = cs.messageDetailsDomWidth + assert(resized < before - 40, s"dragging the inspector edge right did not shrink it: $before -> $resized") + val stored = cs.storedPaneSize("consumer-session-message-inspector") + assert(stored > 0, s"the resized inspector width was not persisted: $stored") + + // Closing a message only removes the pane; selecting one again must restore the chosen width. + cs.messageDetailsClose.click() + assertThat(cs.messageDetails).isHidden() + cs.clickFirstMessage() + assertThat(cs.messageDetailsResizeHandle).isVisible() + val afterReopen = cs.messageDetailsDomWidth + assert(math.abs(afterReopen - resized) <= 3, + s"MessageDetails width changed after close/reopen: $resized -> $afterReopen") + + // The two optional panes can coexist: opening the bottom panel must not collapse or reset the + // right pane (both affect the same ConsumerSession grid). The panel is open by default, so it + // is first genuinely closed and then re-OPENED - the coexistence claim is about the act of + // opening, and closeTools() alone here used to prove nothing (and failed the very next line, + // which asserts the handle only an OPEN panel renders). + cs.closeTools() + cs.openTools() + assertThat(cs.toolsResizeHandle).isVisible() + assertThat(cs.messageDetailsResizeHandle).isVisible() + assert(cs.toolsPanelDomHeight > 150, s"Tools collapsed while MessageDetails was open: ${cs.toolsPanelDomHeight}") + assert(math.abs(cs.messageDetailsDomWidth - resized) <= 3, + s"opening Tools changed the inspector width: $resized -> ${cs.messageDetailsDomWidth}") + cs.closeTools() + + // Reload resets the running session, but the browser-local pane preference must survive. + page.reload() + cs.setStartFrom("Earliest message") + cs.play() + cs.waitMessages(3) + cs.pauseFromToolbar() + cs.assertState("paused") + cs.clickFirstMessage() + assertThat(cs.messageDetailsResizeHandle).isVisible() + assert(cs.storedPaneSize("consumer-session-message-inspector") == stored, + s"stored inspector width changed across reload: $stored -> ${cs.storedPaneSize("consumer-session-message-inspector")}") + val afterReload = cs.messageDetailsDomWidth + assert(math.abs(afterReload - resized) <= 3, + s"persisted MessageDetails width was not re-applied after reload: $resized -> $afterReload") + + // The lower bound is intentionally compact too, for users prioritising the message table. + cs.resizeMessageDetailsBy(2000) + assert(cs.messageDetailsDomWidth <= 175, + s"inspector could not be dragged down to its expanded minimum range: ${cs.messageDetailsDomWidth}") + } + + test("CS-24S: switching Value and Metadata keeps a scrolled message list anchored when Tools is open") { + // A short list cannot expose the regression: react-virtuoso needs enough off-screen rows for a + // tab-driven layout recalculation to have a different scroll position available to jump to. + val messageCount = 120 + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", messageCount) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.waitHeader() + cs.awaitLoaded(messageCount) + cs.pauseFromToolbar() + cs.assertState("paused") + + // Open Tools first, because its reduced vertical viewport is part of the reported interaction. + // Settle before establishing the scroll anchor so expected resizing is not mistaken for a jump. + cs.openTools() + assertThat(cs.toolsResizeHandle).isVisible() + page.waitForTimeout(500) + + cs.scrollTableToTop() + val firstAtTop = cs.firstRenderedIndex + cs.scrollTableDown(900) + val firstAfterScroll = cs.firstRenderedIndex + assert(firstAfterScroll > firstAtTop, + s"the virtualized list did not reach a meaningful middle position: $firstAtTop -> $firstAfterScroll") + assert(cs.tableScrollTop > 100, + s"the virtualized list did not gain a meaningful pixel scroll offset: ${cs.tableScrollTop}") + + // Select a row that is already inside the viewport. Opening the inspector is allowed to settle; + // the invariant starts only once both optional panes and the default Value editor are visible. + cs.messages.nth(2).click() + assertThat(cs.messageDetails).isVisible() + assertThat(cs.messageDetailsResizeHandle).isVisible() + val valueTab = cs.messageDetails.getByTestId("cs-details-tab-value") + val metadataTab = cs.messageDetails.getByTestId("cs-details-tab-metadata") + assertThat(cs.messageDetails.locator(".monaco-editor")).isVisible( + new LocatorAssertions.IsVisibleOptions().setTimeout(30000) + ) + page.waitForTimeout(750) + + val anchored = messageViewport(cs) + + // Repeat: the original report was intermittent, and remounting Monaco on every return to Value + // is the more demanding half of the transition. Check both the exact scroll offset and the + // virtualized row anchor after each direction, while also pinning the viewport geometry. + (1 to 3).foreach { cycle => + metadataTab.click() + assertThat(cs.messageDetails.getByTestId("cs-cell-topic")).isVisible() + page.waitForTimeout(500) + assertViewportUnchanged(anchored, messageViewport(cs), s"after Metadata (cycle $cycle)") + + valueTab.click() + assertThat(cs.messageDetails.locator(".monaco-editor")).isVisible( + new LocatorAssertions.IsVisibleOptions().setTimeout(30000) + ) + page.waitForTimeout(500) + assertViewportUnchanged(anchored, messageViewport(cs), s"after Value (cycle $cycle)") + } + } + test("CS-25: click-to-copy a cell copies + toasts; the Topic cell is a link") { context.grantPermissions(java.util.List.of("clipboard-read", "clipboard-write")) val cs = loadedPaused(3) diff --git a/e2e/src/test/scala/features/consumersession/CsExportSpec.scala b/e2e/src/test/scala/features/consumersession/CsExportSpec.scala index 154232e0f..6afe62b03 100644 --- a/e2e/src/test/scala/features/consumersession/CsExportSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsExportSpec.scala @@ -1,6 +1,7 @@ package features.consumersession import harness.DekafSuite +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.microsoft.playwright.Download import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import org.apache.pulsar.client.api.Schema @@ -9,11 +10,72 @@ import scala.jdk.CollectionConverters.* class CsExportSpec extends DekafSuite: /** Export runs JSON.parse(message.value) - values MUST be valid JSON, else the export throws. */ - private def produceJson(fqn: String, n: Int): Unit = + private def produceKeyedJson(fqn: String, keyed: Seq[(String, String)]): Unit = val p = client.newProducer(Schema.STRING).topic(fqn).create() - try (0 until n).foreach(i => p.send(s"""{"n":$i}""")) + try keyed.foreach((k, v) => p.newMessage().key(k).value(v).send()) finally p.close() + /** A JSON string literal for `s` - the exact encoding both the server (circe `noSpaces`) and the + * exporter (`JSON.stringify`) emit, so expectations can be built from the produced values. */ + private def jsonQuoted(s: String): String = + "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"") + "\"" + + /** Every non-directory entry of a downloaded .zip, as name -> UTF-8 text. */ + private def zipEntries(download: Download): Map[String, String] = + val zipPath = java.nio.file.Files.createTempFile("cs-export", ".zip") + download.saveAs(zipPath) + fixtures.onCleanup(() => java.nio.file.Files.deleteIfExists(zipPath)) + val zip = new java.util.zip.ZipFile(zipPath.toFile) + try + zip.entries().asScala + .filterNot(_.isDirectory) + .map(e => e.getName -> new String(zip.getInputStream(e).readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)) + .toMap + finally zip.close() + + /** ONE exported message, reduced to the fields a test can predict. Everything else in the + * descriptor is the broker's (message id, publish/event times, size, producer name, sequence id) + * and is deliberately not pinned. */ + private case class Exported(index: Int, key: String, value: String, topic: String) + + private val mapper = new ObjectMapper() + + /** Parse an exported "message per array entry" file into records. + * + * PARSING is the point. The previous version searched the concatenated file text for each index, + * each key and each value INDEPENDENTLY - which is satisfied by an export that paired message 1's + * key with message 4's value, or wrote every field of every message into one record, as long as + * all the substrings appeared somewhere. Records make the association itself assertable. */ + private def exportedMessages(fileText: String): List[Exported] = + val root = mapper.readTree(fileText) + assert(root.isArray, s"the exported file is not a JSON array:\n$fileText") + root.elements().asScala.toList.map { node => + def field(name: String): JsonNode = + val v = node.get(name) + assert(v != null && !v.isNull, s"exported message has no '$name':\n${node.toString}") + v + // Each of these also pins the ENCODING LEVEL, which the recorded values below depend on: a + // string field that stopped being a JSON string, or an index that became one, fails here + // rather than quietly comparing something else. + def jsonString(name: String): String = + val v = field(name) + assert(v.isTextual, s"'$name' was expected to be a JSON string, got ${v.toString}") + v.asText + val index = field("index") + assert(index.isInt, s"'index' is not a JSON number: ${index.toString}") + Exported( + index = index.intValue, + // Both of these arrive at the browser JSON-ENCODED, and the exporter treats them + // differently: `value` is JSON.parse'd once (so it comes back out as the produced text), + // `key` is not (so it comes back out still carrying its own quotes). Pin what is really + // written rather than what one might assume - `asText` peels exactly one JSON string level + // off each, and `isTextual` above is what stops that from hiding a change. + key = jsonString("key"), + value = jsonString("value"), + topic = jsonString("topic") + ) + } + private def loadedPaused(t: String, ns: String, topic: String, n: Int): ConsumerSessionPage = val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) @@ -24,10 +86,21 @@ class CsExportSpec extends DekafSuite: cs.assertState("paused") cs - test("CS-28: Export modal offers 4 formats and downloads a .zip containing the messages") { + // Previously this asserted 4 formats + that indices 1..5 appeared somewhere in the .zip - an + // exporter that dropped every value, or exported the wrong topic's messages, still passed. It now + // asserts the exported BYTES: exact content for the value-only format, and the parsed RECORDS - + // (index, key, value, topic) per message, in order - for the default full-descriptor format. + // + // Records rather than substrings because the fields are what a broken exporter mixes up. Every key + // and every value is DISTINCT and their positions differ (key i is the i-th key, value i the i-th + // value), so a record-for-record comparison fails on a swap that a "does the file contain this + // string" search cannot see at all. + test("CS-28: Export modal offers 4 formats and the .zip carries exactly the produced messages") { val (t, ns, topic) = fixtures.freshTopicParts() val fqn = s"persistent://$t/$ns/$topic" - produceJson(fqn, 5) + val keys = (1 to 5).map(i => s"cs28-key-$i").toList + val values = (1 to 5).map(i => s"""{"n":$i}""").toList + produceKeyedJson(fqn, keys.zip(values)) val cs = loadedPaused(t, ns, topic, 5) cs.exportOpen.click() @@ -36,25 +109,39 @@ class CsExportSpec extends DekafSuite: assert(modal.formatOptionCount == 4, s"expected 4 formats, got ${modal.formatOptionCount}") assertThat(modal.fieldRows.first()).isVisible() // field-config list present (reorder disabled - see NOTES) + // --- "value per array entry": the file is nothing but the values, so pin it EXACTLY - a + // dropped, duplicated, reordered or mangled message all fail here. The messages sort by + // publishTime ascending (stable), i.e. production order. + modal.selectFormat("json-value-per-entry") + val valueEntries = zipEntries(page.waitForDownload(() => modal.runButton.click())) + assert(valueEntries.size == 1, s"expected one exported file, got ${valueEntries.keys.toList}") + val expectedValuesJson = values.map(jsonQuoted).mkString("[", ",", "]") + assert( + valueEntries.head._2 == expectedValuesJson, + s"exported values were:\n${valueEntries.head._2}\nexpected:\n$expectedValuesJson" + ) + + // --- default "message per array entry": full descriptors. Timestamps and message ids are + // non-deterministic, so the exported file is PARSED and compared record for record on the + // fields that are: index, key, value and topic. + modal.selectFormat("json-message-per-entry") val download: Download = page.waitForDownload(() => modal.runButton.click()) assert(download.suggestedFilename().endsWith(".zip"), download.suggestedFilename()) + val entries = zipEntries(download) + // One file, and its NAME is the index range it holds: the exporter chunks by size and names each + // chunk `-.json`, inside a timestamped export folder. + assert(entries.size == 1, s"expected one exported file, got ${entries.keys.toList}") + val (entryName, entryText) = entries.head + assert(entryName.endsWith("/1-5.json"), s"unexpected exported file name: $entryName") - val zipPath = java.nio.file.Files.createTempFile("cs-export", ".zip") - download.saveAs(zipPath) - fixtures.onCleanup(() => java.nio.file.Files.deleteIfExists(zipPath)) - - val zip = new java.util.zip.ZipFile(zipPath.toFile) - try - val text = zip.entries().asScala - .filterNot(_.isDirectory) - .map(e => new String(zip.getInputStream(e).readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)) - .mkString("\n") - assert(text.nonEmpty, "empty export") - // The default format exports the full message descriptor; the raw value is JSON-escaped inside - // a string field, so assert the (unescaped) per-message index 1..5 is present. - (1 to 5).foreach(i => - assert(text.contains(s"\"index\":$i"), s"missing message index $i in export")) - finally zip.close() + val exported = exportedMessages(entryText) + val expected = keys.zip(values).zipWithIndex.map { case ((k, v), i) => + Exported(index = i + 1, key = jsonQuoted(k), value = v, topic = fqn) + } + assert( + exported == expected, + s"the exported messages are not the produced ones.\n exported: $exported\n expected: $expected" + ) } test("CS-29: export config persists across reopen") { diff --git a/e2e/src/test/scala/features/consumersession/CsFiltersSpec.scala b/e2e/src/test/scala/features/consumersession/CsFiltersSpec.scala index d3454fc82..ac2fc7262 100644 --- a/e2e/src/test/scala/features/consumersession/CsFiltersSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsFiltersSpec.scala @@ -20,7 +20,6 @@ class CsFiltersSpec extends DekafSuite: val (t, ns, topic) = fixtures.freshTopicParts() val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() val f = cs.sessionFilterPanel f.addFilter() @@ -50,7 +49,6 @@ class CsFiltersSpec extends DekafSuite: val (t, ns, topic) = fixtures.freshTopicParts() val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() val f = cs.sessionFilterPanel f.addFilter() @@ -71,7 +69,9 @@ class CsFiltersSpec extends DekafSuite: cs.setStartFrom("Earliest message") cs.play() val values = eventually() { - val vs = cs.columnValues("value") + // Whole-list read: a broken filter that let every skip-* through could push the older keeps + // out of the mounted viewport, and a mounted-rows read would then miss the leak. + val vs = cs.allColumnValues("value") assert(vs.contains("keep-last"), s"sentinel not loaded yet: $vs") vs } @@ -119,7 +119,6 @@ class CsFiltersSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() val f = cs.sessionFilterPanel f.addFilter() f.setOp("string includes") diff --git a/e2e/src/test/scala/features/consumersession/CsFlowControlSpec.scala b/e2e/src/test/scala/features/consumersession/CsFlowControlSpec.scala new file mode 100644 index 000000000..b87d39ade --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsFlowControlSpec.scala @@ -0,0 +1,437 @@ +package features.consumersession + +import harness.DekafSuite +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions + +/** The start-from flow-control and guard behaviors only a REAL broker can prove. + * + * The merge's watermark arithmetic, the give-up window and the guards are all pinned by the + * server suite with injected clocks and plain values. What no unit test can reach is the LAST + * HOP of each: `consumer.pause()` issued from inside an armed MessageListener callback while the + * broker keeps dispatching (CS-FC-1), the give-up degradation travelling broker -> merge -> + * progress frame -> STICKY banner (CS-FC-2), and the creation-time refusals surfacing as a + * user-visible error rather than a silent wrong session (CS-FC-3/4). + * + * WHY NOT A DISPATCH-RATE THROTTLE: the obvious way to starve one stream is a namespace + * dispatch rate, and it was tried first - this standalone broker does not enforce it (verified + * with the broker's own CLI consumer sailing through a 1-msg/10s policy), with or without the + * `dispatchThrottlingOnNonBacklogConsumerEnabled` / `preciseDispatcherFlowControl` flags. The + * scenarios below use levers that are deterministic here instead: a LARGE OLD BACKLOG whose + * drop phase takes real seconds (pause pressure without any silence), and a FORCE-DELETE of a + * topic while the session is paused (absolute silence - a deleted ledger cannot deliver its + * recorded end, whatever any cursor believes; admin cursor jumps proved unreliable on + * NonDurable subscriptions). + */ +class CsFlowControlSpec extends DekafSuite: + private def vis(ms: Int) = new LocatorAssertions.IsVisibleOptions().setTimeout(ms.toDouble) + + /** Consumer connection stamps for the session's subscription on `fqn` - the connection EPOCH + * the broker's per-consumer counters live inside (a reconnect replaces the consumer and its + * dispatch counter; same bracketing as CS-DM-D6G's). Empty while no consumer is attached. */ + private def consumerConnectedSince(fqn: String): List[String] = + import scala.jdk.CollectionConverters.* + admin.topics().getStats(fqn).getSubscriptions.values().asScala.toList + .flatMap(_.getConsumers.asScala.toList.map(_.getConnectedSince)) + + test("CS-FC-1: the per-stream watermark engages under the armed listener, and the exact SET survives") { + // B: 30,000 OLD messages. A: 3,000 NEW ones. Global skip of 31,995 must drop every B message + // FIRST (they are globally oldest), which takes real seconds - and in that window all of A + // arrives and can do nothing but QUEUE, sailing past the per-stream watermark (1,000), so + // A's consumer is paused from inside its own listener callback and resumed as the queue + // drains, cycling until the budget lands. What this pins end to end: pausing a consumer + // mid-listener neither deadlocks the session (a hang fails the await) nor loses, duplicates or + // mis-selects anything - and WHICH 1,005 survive is asserted, not just how many. A count is + // reached by 1,005 wrong messages just as easily as by the right ones, and one dropped message + // plus one duplicate reaches it too. What it deliberately does not claim: that a silently + // non-pausing consumer.pause() would be detected - that would only weaken the memory bound, + // which no black-box assertion can see; the hook INVOCATION is pinned by the ordering-layer + // unit suite. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + // DISTINCT PREFIXES, not decoration: with both topics carrying `msg-1..` a B message wrongly + // delivered instead of an A message is the same string as the A message it displaced, and the + // exact-set assertion below could not see it. + fixtures.produceStringsFast(fqnB, 30000, prefix = "b") // first, so B is the globally-oldest block + // Every one of B's publish times is stamped before its producer flush returned, so this instant + // separates the two blocks. The exact-set expectation below depends on that separation, and an + // instant taken here is what makes it checkable without reading 30,000 messages back. + val betweenBlocks = System.currentTimeMillis() + fixtures.produceStrings(fqnA, 3000, prefix = "a") + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("31995") + // PINNED, not inherited: the watermark under test is the ordering MERGE's, so this test is + // meaningful only with an engaged, non-stalling merge that keeps FOLLOWING. Fastest builds no + // merge at all, and Guaranteed is an exact replay that auto-pauses at its boundary - a + // different lifecycle whose flow-control cell is CS-FC-5. The cycling-under-pressure path + // pinned here is Best effort's, whatever the session default is. + cs.setDeliveryOrder("Best effort") + cs.play() + cs.assertState("running") + + cs.awaitLoaded(1005, timeoutMs = 120000) + page.waitForTimeout(1500) // anything past 1,005 would arrive right behind it + // The count is KEPT on purpose: it is read from the server's own numMessageSent after a quiet + // window, so it catches a straggling extra delivery arriving after the set below first matched + // - the one failure a set comparison cannot see. It is no longer the whole assertion. + assert(cs.loadedCount == 1005, s"exactly 1005 must remain after skipping 31995 of 33000, got ${cs.loadedCount}") + + // Slow positioning is not degraded positioning: every stream kept speaking. + assertThat(cs.startFromDegradedBanner).not().isVisible() + + // WHICH 1,005. The cut is global and B is entirely older than A, so the survivors are A's last + // 1,005 - read back from the BROKER in append order rather than assumed from the produce call. + // The boundary premise is checked too: A's oldest message must be newer than the instant B + // finished. If the two blocks ever overlapped in time a different set would be correct, and + // this expectation - not the app - would be the thing that is wrong. + val onA = fixtures.readAllMessages(fqnA) + val oldestA = onA.head.getPublishTime + assert( + oldestA > betweenBlocks, + s"arrangement broken: A's oldest message ($oldestA) is not newer than the instant B finished " + + s"($betweenBlocks), so the global cut does not fall where this expectation assumes" + ) + val expected = onA.map(_.getValue).toList.drop(1995) + assert(expected.size == 1005, s"expected-set arithmetic is off: ${expected.size}") + + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 15000) + assert(cs.retainedCount == 1005, s"the browser holds ${cs.retainedCount} of 1005 rows - the export would be partial") + DeliveredMessages.assertExactly(cs.exportedValues(), expected, "the surviving set after a global skip of 31,995") + } + + test("CS-FC-2: a stream whose recorded end stops being deliverable degrades VISIBLY after the give-up window") { + // The retention/trim race, reproduced without racing anything: B holds 200,000 messages, far + // more than the drop phase can consume before the session is PAUSED moments after it starts. + // With every consumer frozen, B's subscription cursor is jumped past its whole backlog - the + // recorded end silently stops being deliverable - and the session resumes into at most a + // prefetched tail followed by permanent silence on a still-waited stream. The give-up window + // (30s, granted FRESH at resume - paused time proves nothing) expires, the merge abandons B, + // and the session must SAY SO: the sticky best-effort banner, carried on the very frames the + // give-up drain emits. The budget (150,000) is deliberately unspendable, so the banner - not + // delivery - is the observable outcome, exactly like a real over-sized skip against a topic + // that retention trimmed mid-positioning. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + // A MILLION, deliberately: the merge's drop phase runs at up to a few hundred thousand + // messages per second, so anything smaller can be fully consumed - recorded end delivered, + // stream no longer waited - before a UI-timed pause can possibly land. At this size the + // pause is guaranteed to catch B mid-backlog, whatever the machine's speed. + fixtures.produceStringsFast(fqnB, 1000000) + fixtures.produceStrings(fqnA, 100) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + // ARRANGE THE HIDDEN CONSOLE: the tools pane now opens BY DEFAULT (a deliberate UI change on + // this branch - `consumerSessionToolsOpen` defaults true), so the collapsed-console layout + // oracle below needs its premise arranged, or it reads the OPEN pane's honest 360px and + // fails on a layout that in fact survived (observed 2026-08-09). + cs.closeTools() + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("600000") + // PINNED, not inherited: the give-up degradation IS Best effort's contract. Guaranteed waits + // indefinitely by design and can never reach the banner this test exists to prove, so the mode + // must be explicit here whatever the session default is - this is the only end-to-end proof of + // the degradation disclosure. + cs.setDeliveryOrder("Best effort") + cs.play() + cs.assertState("running") + + // Freeze the world, then make the backlog unreachable for good. The pre-pause second drops + // at most a couple hundred thousand of B's million (harmless - the budget stays unspendable + // and B stays mid-backlog, so it is still WAITED on), and force-deleting B destroys the + // rest, recorded end included. Auto-creation may resurrect the NAME as an empty topic; the + // old ledger never comes back, which is exactly the trimmed-partition condition. + cs.pauseFromToolbar() + cs.assertState("paused") + fixtures.forceDeleteTopic(fqnB) + cs.play() + cs.assertState("running") + + // Give-up at ~30s of post-resume silence (+ the 2s sweep cadence). + assertThat(cs.startFromDegradedBanner).isVisible(vis(75000)) + val banner = cs.startFromDegradedBanner.textContent() + assert(banner.contains("Best effort"), s"unexpected banner text: '$banner'") + assert(banner.contains("1 topic"), s"the banner should count the abandoned topics, got: '$banner'") + // The banner names the TOPIC the user owns, not the internal `@` stream id. + assert(banner.contains(fqnB), s"the banner should name the silent topic, got: '$banner'") + assert(!banner.contains("@"), s"internal stream ids must not reach the screen, got: '$banner'") + + // THE LAYOUT SURVIVES THE DISCLOSURE. The banner used to be a fourth direct grid child in a + // three-row grid: the content area collapsed into the console's zero-height row and the + // supposedly hidden console became the visible one. Height is the only honest oracle here. + val contentHeight = page.locator("[data-testid='cs-session'] > *:nth-child(2)").boundingBox().height + assert(contentHeight > 50, s"the content row must keep real height under the banner, got $contentHeight") + // <= 8: the console keeps its 4px top border even in a zero-height grid row; what the broken + // layout produced was the console's full content height in an implicit fourth row. + val consoleBox = Option(page.getByTestId("cs-console").boundingBox()) + assert( + consoleBox.forall(_.height <= 8), + s"a hidden console must stay collapsed under the banner, got ${consoleBox.map(_.height)}" + ) + + // Nothing was delivered - the budget is unspendable by design - and the session is still + // alive and positioning, not crashed: degradation is a disclosure, never a failure. + // A COUNT AND NOT A SET, deliberately: zero IS the exact set here. There is no identity to + // compare, and any non-zero value is a failure whatever its payload. + assert(cs.loadedCount == 0, s"an unspendable budget must deliver nothing, got ${cs.loadedCount}") + assert(cs.state == "running", s"a degraded session keeps running, got '${cs.state}'") + + // Stop EXPLICITLY: this session's B-consumer is reconnect-looping against a deleted topic, + // and leaving it running turns the server log into a firehose for the rest of the suite. + cs.stop() + } + + test("CS-FC-3: skip-n REFUSES overlapping targets with a visible reason - and per-view modes still work") { + // Two enabled targets, both "Current Topic": a counted skip cannot mean anything predictable + // over duplicated subscriptions (one shared budget, two copies of every message), so Play + // must fail with the reason on screen. The SAME session with a plain mode then works, and + // shows each target's own copy - the per-view contract, pinned end to end. + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 3) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.addTarget() + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("1") + cs.play() + + val refusal = page.getByText( + java.util.regex.Pattern.compile("two enabled targets select the same topic", java.util.regex.Pattern.CASE_INSENSITIVE) + ).first() + assertThat(refusal).isVisible(vis(15000)) + + // The refusal is MODE-specific: Earliest on the same two targets delivers each target's own + // counted set - three messages, two views, six rows. + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(6) + } + + test("CS-FC-4: latest-n REFUSES a read-compacted target with a visible reason - Latest message still works") { + // Latest-n counts STORED entries; a compacted read shows one message per key. The walk + // cannot see the compacted view, so the session must refuse rather than promise a count it + // cannot keep. 'Latest message' needs no counting and must keep working on the same target. + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 3) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.toggleTargetCompacted() + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("2") + cs.play() + + val refusal = page.getByText( + java.util.regex.Pattern.compile("read compacted", java.util.regex.Pattern.CASE_INSENSITIVE) + ).first() + assertThat(refusal).isVisible(vis(15000)) + + cs.setStartFrom("Latest message") + cs.play() + cs.assertState("running") + } + + test("CS-FC-5: the BYTE watermark pauses a hot stream before any count watermark can") { + // THE ORACLE IS EPOCH-BRACKETED, and the history is worth keeping: this cell first shipped + // reading the broker's dispatch counter bare, and went red in 2 of 4 otherwise-identical + // runs with "400 of 400 dispatched". That was the ORACLE's defect, not the product's: the + // counter counts DISPATCHES, not unique messages, and it belongs to the broker-side consumer + // - a transient reconnect replaces that consumer, resets the counter, and redelivers the + // entire un-acked held window (256+ messages the merge already holds and deduplicates), so + // the reading mixes redeliveries with admission. The counter is therefore interpreted per + // CONNECTION EPOCH (the same `connectedSince` bracketing CS-DM-D6G uses): within the epoch + // that began at Play it counts unique admitted messages and carries the full discriminating + // bounds; after a transient reconnect only the redelivery-inclusive ceiling is checkable + // from it, and the fixpoint properties below carry the rest. (The server side of the 2026-08-09 + // investigation is separately pinned: admission under the replay barrier is serialized, with + // its own deterministic unit tests.) + // + // Every other payload in this suite is a handful of bytes, so the merge's held-BYTES cap + // (256 MiB - globalStartFrom.startFromMergePauseBytesAt) was unreachable: a count watermark + // (10,000 total / 1,000 per stream) always tripped first, and a byte cap with the wrong unit, + // one that never fires, or one that fires on the first message was invisible. Here the counts + // are made UNREACHABLE instead: 400 messages of 1 MiB each can never trip a count watermark + // (400 < 1,000), while 400 MiB > 256 MiB guarantees the byte cap has something to do - so if + // anything pauses the stream, it is the byte cap. + // + // WHAT MAKES THE HOLD DETERMINISTIC changed with the exact-replay redesign (2026-08-09). The + // old arrangement held A behind a 100-partition topic that never spoke - but a stream that + // recorded nothing at the boundary is now trivially FINISHED and never waited for, so empty + // silence holds nothing any more (CS-DM-D1G pins exactly that). The hold that remains + // e2e-drivable is the one the replay cannot argue with: a recorded range that stops being + // DELIVERABLE. C records three million messages before Play and is force-deleted mid-drain; + // its recorded end can then never arrive, the replay may not finish past it, and - because + // the give-up stays Best effort's alone - the merge waits forever. Everything A dispatches + // is NEWER than C's unfinished range, so it queues in the heap for as long as we choose, + // and held-bytes walk up to the cap. Three million because the drop phase is FAST (a 1M + // topic was fully consumed inside the delete's own REST round trip when this was first + // tried); the arrangement guard below turns a lost race into a named failure. + // + // The skip budget (5,000,000 over a 3,000,400-message arrangement) is the browser shield: an + // unspendable counted cut consumes what the merge emits without delivering a row, so neither + // C's drained prefix nor A's 400 MiB can ever strain the shared Chromium. And B - the + // 100-partition topic that never speaks - stays in the arrangement for its OTHER old job: + // 102 consumers shrink the per-consumer prefetch to 5,000/102 = 49 messages + // (mergeReceiverQueueSizeFor), which is what makes the dispatch plateau tight enough to + // discriminate: after the pause the broker can have dispatched at most the trip point (256) + // plus prefetch (49) plus a little in-flight slack. + // + // COUNTS ARE THE PROPERTY HERE, not a weaker stand-in for an exact set - do not "strengthen" + // them into one. A byte watermark is a statement about HOW MANY bytes were admitted before the + // stream was held, so the only thing that can answer it is a quantity: the broker's dispatch + // counter, bounded on both sides. And the delivered set is asserted exactly - it must be + // EMPTY - because the budget is unspendable by design. + // + // The oracle is the BROKER's dispatch counter for A's one subscription: + // - plateau (two agreeing reads) >= 256 - the cap did not fire early (a bytes-vs-KB unit + // bug fires orders of magnitude too soon and fails this bound); + // - plateau <= 360 - the cap DID fire (a never-firing cap dispatches all 400 and fails + // this bound; no count watermark can produce a pause below 1,000 held). + // The held 256 MiB is deliberately NEVER released into the discard: the byte-RESUME drain is + // pinned at the server tier (the pause arbiter and watermark unit suites), while the e2e + // resume path is CS-FC-1's count-watermark cycling through the same arbiter. What closes the + // loop here instead: the 40s fixpoint (the give-up must NOT fire under Guaranteed - a + // wrongly-armed one would abandon C, drain the holds and resume A past the plateau), the + // caught-up banner staying ABSENT (an unfinished stream must never announce a boundary), and + // a clean stop that releases the paused consumer on the broker. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tC, nsC, topicC) = fixtures.freshTopicParts() + val fqnC = s"persistent://$tC/$nsC/$topicC" + val tB = fixtures.createTenant() + val nsB = fixtures.createNamespace(tB) + val topicB = s"silent-${System.currentTimeMillis()}" + val fqnB = s"persistent://$tB/$nsB/$topicB" + fixtures.admin.topics().createPartitionedTopic(fqnB, 100) + + // C first, A second: every C publish time is stamped before A's first, so A's whole block is + // strictly newer than C's unfinished range and must queue behind it. + val cBacklog = 3000000L + fixtures.produceStringsFast(fqnC, cBacklog.toInt) + fixtures.produceSized(fqnA, count = 400, payloadBytes = 1024 * 1024) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnC, fqnB)) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("5000000") + // Guaranteed is the point here, and pinned: only the no-give-up barrier turns "C's recorded + // end stopped being deliverable" into "A's dispatches are all HELD", which is what walks + // held-bytes up to the cap. Best effort would abandon C after 30s and drain everything. + cs.setDeliveryOrder("Guaranteed") + val playedAt = System.currentTimeMillis() + cs.play() + cs.assertState("running") + // The connection epoch the whole oracle is bracketed by - see the header comment. + val connectionAtPlay = consumerConnectedSince(fqnA) + + // Make C's recorded end undeliverable while its drop phase is provably still in flight. + // Auto-creation may resurrect the NAME as an empty topic; the old ledger never comes back, + // which is exactly the trimmed-partition condition. + fixtures.forceDeleteTopic(fqnC) + // THE ARRANGEMENT GUARD: the delete must beat the drop phase to at least part of C's range, + // or there is no undeliverable remainder and the whole cell proves nothing (observed + // 2026-08-09 at C = 1M: the drop phase consumed the entire topic inside the delete's own + // multi-second REST round trip, the barrier finished, and the session legitimately caught + // up). C is sized at 3M for a several-fold margin, and a lost race fails HERE, by name, + // instead of as a mysterious full-dispatch plateau below. + assertThat(page.getByTestId("cs-replay-caught-up")).not().isVisible() + val progress = page.getByTestId("cs-start-from-progress") + val skippedAfterDelete = + if progress.isVisible then Option(progress.getAttribute("data-cs-skipped")).flatMap(_.toLongOption).getOrElse(0L) + else 0L // no progress frame yet - nothing claimed, the race is safely won + assert( + skippedAfterDelete < cBacklog, + s"the delete lost the race with the drop phase: $skippedAfterDelete of $cBacklog were already claimed - " + + "grow C so its recorded range outlives the delete" + ) + + // The hold phase. C can never finish, so nothing of A's is emitted, and A's dispatch counter + // must SETTLE (within one connection epoch) rather than keep climbing. + val plateau = harness.Eventually.eventually(timeoutMs = 150000, intervalMs = 1000) { + val c1 = consumerConnectedSince(fqnA) + val d1 = fixtures.subscriptionDispatchCount(fqnA) + page.waitForTimeout(1500) + val d2 = fixtures.subscriptionDispatchCount(fqnA) + val c2 = consumerConnectedSince(fqnA) + assert(c1 == c2 && c2.nonEmpty, s"A's consumer is mid-reconnect: $c1 then $c2") + assert(d1 == d2, s"A's dispatch counter is still moving: $d1 then $d2") + // The early-fire floor is only meaningful while the counter still counts unique messages - + // i.e. in the epoch that began at Play. A reconnect resets the broker-side counter, and a + // byte-paused consumer can legitimately show a near-zero fresh epoch. + if c2 == connectionAtPlay then + assert(d2 >= 256, s"A was paused after only $d2 dispatched 1-MiB messages - the byte cap fired too early") + (d2, c2) + } + val (plateauDispatched, plateauEpoch) = plateau + if plateauEpoch == connectionAtPlay then + assert( + plateauDispatched <= 360, + s"the byte watermark never engaged: $plateauDispatched of 400 1-MiB messages were dispatched while C was " + + s"unfinished, with no reconnect to blame (cap 256 + prefetch 49 + slack should bound this at ~360; the " + + s"delete landed with $skippedAfterDelete of $cBacklog already claimed)" + ) + else + // A transient reconnect replaced the broker-side consumer: its fresh counter mixes a + // redelivery of the already-held window (<= 360, deduplicated by the merge) with whatever + // cap-bounded admission remained (<= 360), so only the redelivery-inclusive ceiling is + // checkable from the counter in this epoch. loadedCount and the fixpoint below still carry + // the leak detection at full strength. + assert( + plateauDispatched <= 720, + s"even counting a full held-window redelivery, $plateauDispatched dispatches in the post-reconnect epoch " + + s"exceed what a working byte cap admits (<= 360 unique + <= 360 redelivered)" + ) + assert(cs.loadedCount == 0, s"an unspendable budget must deliver nothing, got ${cs.loadedCount}") + + // The hold must be a FIXPOINT that outlives the best-effort give-up window (30s + sweep): + // under Guaranteed the give-up is disabled, and a wrongly-armed one would abandon C, drain + // the holds, RESUME A (dispatch rises past the plateau) and spend the budget's next 400. + // The wait is arrangement - outliving the window - not readiness. + harness.Eventually.eventually(timeoutMs = 60000, intervalMs = 500) { + val elapsed = System.currentTimeMillis() - playedAt + assert(elapsed >= 40000, s"only ${elapsed}ms of the 40s give-up-window margin has elapsed") + } + val fixpointEpoch = consumerConnectedSince(fqnA) + val fixpointDispatched = fixtures.subscriptionDispatchCount(fqnA) + if fixpointEpoch == plateauEpoch then + assert( + fixpointDispatched == plateauDispatched, + s"dispatch resumed past the plateau after the give-up window - a give-up fired under Guaranteed: " + + s"$fixpointDispatched after settling at $plateauDispatched" + ) + else + // The counter reset with a reconnect inside the window; the ceiling is what remains + // checkable from it, and the loaded/degraded asserts below are what a real give-up + // cannot get past (it would spend the budget's next 400 into the discard and disclose). + assert( + fixpointDispatched <= 720, + s"the post-reconnect epoch dispatched $fixpointDispatched - beyond a full held-window redelivery, " + + "so the hold was released during the give-up window" + ) + assert(cs.loadedCount == 0, s"the hold leaked deliveries after the give-up window, got ${cs.loadedCount}") + assertThat(cs.startFromDegradedBanner).not().isVisible() + // An unfinished stream is not a finished chunk: the replay may not announce caught-up while + // C's recorded range has never been delivered. + assertThat(page.getByTestId("cs-replay-caught-up")).not().isVisible() + assert(cs.state == "running", s"holding is not stalling - the session stays healthy, got '${cs.state}'") + + // A clean stop must release the byte-paused consumer: the session's NonDurable subscription + // disappears from the broker once its consumer disconnects - a wedged pause would leave it. + cs.stop() + harness.Eventually.eventually(timeoutMs = 30000, intervalMs = 500) { + val subs = fixtures.admin.topics().getStats(fqnA).getSubscriptions + assert(subs.isEmpty, s"the paused consumer's subscription survived stop: ${subs.keySet()}") + } + } diff --git a/e2e/src/test/scala/features/consumersession/CsLifecycleSpec.scala b/e2e/src/test/scala/features/consumersession/CsLifecycleSpec.scala index 57f9707aa..968a38738 100644 --- a/e2e/src/test/scala/features/consumersession/CsLifecycleSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsLifecycleSpec.scala @@ -1,10 +1,46 @@ package features.consumersession import harness.DekafSuite +import harness.Eventually.eventually import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.Locator +import org.apache.pulsar.client.api.Schema +import scala.jdk.CollectionConverters.* class CsLifecycleSpec extends DekafSuite: + private def hasText = new LocatorAssertions.HasTextOptions().setTimeout(20000) + private def count(n: Int) = new LocatorAssertions.HasCountOptions().setTimeout(20000) + + /** The toolbar's "processed" counter (the server's `numMessageProcessed` carried on the last + * streamed message). `ConsumerSessionPage` only exposes `loaded`, so address it directly. */ + private def processed: Locator = page.getByTestId("cs-processed") + + private def produce(fqn: String, values: Seq[String]): Unit = + val p = client.newProducer(Schema.STRING).topic(fqn).create() + try values.foreach(p.send) finally p.close() + + /** BROKER ORACLE: how many of Dekaf's OWN consumers are attached to `fqn` right now. + * + * The Consumer Session mints `__dekaf_` (ConsumerSession.tsx) and `buildConsumer` reuses + * it as the consumer AND subscription name, subscribing NON-DURABLY - so a `__dekaf_`-prefixed + * subscription in the topic stats can only exist while a live UI-driven consumer is attached, and + * the broker drops it as soon as that consumer detaches. Prefix-scoping (rather than counting any + * consumer) keeps a stray subscription from another test or a leftover reader out of the count. */ + private def dekafConsumerCount(fqn: String): Int = + val subs = admin.topics().getStats(fqn).getSubscriptions + subs.keySet().asScala.toList + .filter(_.startsWith("__dekaf_")) + .map(name => subs.get(name).getConsumers.size) + .sum + + private def awaitDekafConsumers(fqn: String, n: Int): Unit = + eventually(timeoutMs = 30000, intervalMs = 300) { + // The topic can momentarily 404 while the broker unloads/GCs it; treat that as "not yet". + val got = try dekafConsumerCount(fqn) catch case _: Throwable => -1 + assert(got == n, s"expected $n Dekaf consumer(s) attached to $fqn, got $got") + } + private def topicWith(n: Int): (String, String, String) = val (t, ns, topic) = fixtures.freshTopicParts() fixtures.produceStrings(s"persistent://$t/$ns/$topic", n) @@ -20,6 +56,73 @@ class CsLifecycleSpec extends DekafSuite: cs.assertState("running") cs + // CS-16/17 previously proved only "rows appeared" / "rows disappeared" - a UI that rendered a + // cached list would pass both. These two assert the thing the names claim: a REAL broker-side + // consumer for the lifetime of the session, and counters that track produced messages. + + test("CS-16: a running session holds a real broker-side consumer and its counters advance with production") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(fqn, 5) // "msg-1".."msg-5" + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + + // The table is virtualized, so assert the COUNTERS, not DOM rows. + cs.awaitLoaded(5) + cs.assertState("running") + assertThat(processed).hasText("5", hasText) + + // Broker-side proof that the session is actually consuming, not replaying a client-side cache. + awaitDekafConsumers(fqn, 1) + + // Counters must ADVANCE with newly produced messages. "live-sentinel" is produced LAST on a + // single-partition topic, so its arrival means every earlier value was already delivered - the + // set assertion below is then exact, never a transient count. + produce(fqn, Seq("live-1", "live-2", "live-sentinel")) + cs.awaitLoaded(8) + assertThat(processed).hasText("8", hasText) + + val expected = (1 to 5).map(i => s"msg-$i").toList ++ List("live-1", "live-2", "live-sentinel") + val values = eventually() { + val vs = cs.columnValues("value") + assert(vs.contains("live-sentinel"), s"sentinel not rendered yet: $vs") + vs + } + assert(values == expected, s"loaded values were: $values (expected exactly $expected)") + + // Still attached after the second wave - the consumer lives for the whole running session. + assert(dekafConsumerCount(fqn) == 1, s"the session's broker consumer vanished while running on $fqn") + } + + test("CS-17: Stop detaches the broker-side consumer, clears the messages and resets the counters") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(fqn, 5) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(5) + assertThat(processed).hasText("5", hasText) + awaitDekafConsumers(fqn, 1) + + cs.stop() + + // Client side: the session is torn down to a fresh one - no rows, both counters back to zero. + assertThat(cs.messages).hasCount(0, count(0)) + cs.assertState("new") + assertThat(cs.loaded).hasText("0", hasText) + assertThat(processed).hasText("0", hasText) + + // Broker side: the consumer is really gone (a non-durable subscription disappears with it), so + // Stop releases the broker resource instead of leaking a consumer per stopped session. + awaitDekafConsumers(fqn, 0) + } + test("CS-18: wheel-scroll (up) on a running session transitions to paused") { val cs = runningSession(5) cs.wheelUpOverTable() diff --git a/e2e/src/test/scala/features/consumersession/CsMergeOrderSpec.scala b/e2e/src/test/scala/features/consumersession/CsMergeOrderSpec.scala new file mode 100644 index 000000000..4473de6c1 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsMergeOrderSpec.scala @@ -0,0 +1,477 @@ +package features.consumersession + +import harness.DekafSuite +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +import scala.jdk.CollectionConverters.* + +/** Delivery ordering across topics and partitions. + * + * WHAT ONLY AN END-TO-END TEST CAN SEE HERE. The server suite pins the floor rule against + * injected clocks; the jest suite pins the config round-trip. Neither can tell whether two real + * consumers racing two real backlogs actually reach the browser as ONE stream in publish-time + * order - that is a property of the whole pipeline (broker read speed, listener threads, the + * merge, the gRPC stream, the table). + * + * THE ORACLE IS DELIVERY ORDER, NOT DISPLAY ORDER. The table sorts by publish time on its own, + * which would mask everything; each row's index cell carries its session-order ordinal, so the + * (ordinal, value) pairs reconstruct the order messages actually ARRIVED in. + * + * The fixture produces values round-robin across the topics with batching off and synchronous + * sends, so publish times strictly increase across the whole set: the expected merged order IS + * the production order. + */ +class CsMergeOrderSpec extends DekafSuite: + + private def vis(timeoutMs: Double) = new com.microsoft.playwright.assertions.LocatorAssertions.IsVisibleOptions().setTimeout(timeoutMs) + + private val bestEffortLabel = "Best effort" + private val guaranteedLabel = "Guaranteed" + private val fastestLabel = "Fastest" + + /** Values in the order the session DELIVERED them (row index-cell ordinal, ascending). + * + * The table is VIRTUALIZED: only a viewport's worth of rows is mounted at any moment, and how + * many that is moves with every layout change - a fixed hasCount on mounted rows broke the + * moment the tools pane grew. So the scrape pauses the session first (instant now, and it + * stops the running-state auto-scroll from fighting the walk), scrolls the virtuoso pane from + * the top, and harvests (ordinal, value) pairs until every expected ordinal has been seen. + */ + private def deliveredValues(expected: Int): List[String] = + if page.getByTestId("cs-session").getAttribute("data-cs-state") == "running" then + val cs = ConsumerSessionPage(page) + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 5000) + + val byOrdinal = scala.collection.mutable.Map.empty[Int, String] + // ONE evaluate per pass, not per-cell Locator reads: virtuoso mounts a spacer row without + // cells, and a Locator innerText on a cell that is not there auto-waits its full default + // timeout - one spacer froze the whole walk. The page-side snapshot also cannot go stale + // mid-read. The scroller is looked up INSIDE the message table: the tools pane's tabs render + // their own virtuoso, and a document-wide lookup scrolled whichever came first in the DOM. + def harvestAndStep(): Unit = + val json = page + .evaluate( + """() => { + | const pairs = Array.from(document.querySelectorAll("[data-testid='cs-table'] tbody tr")) + | .map(r => [ + | r.querySelector("[data-testid='cs-message']")?.innerText?.trim() ?? null, + | r.querySelector("[data-testid='cs-cell-value']")?.innerText?.trim() ?? null + | ]) + | .filter(p => p[0] !== null && p[1] !== null); + | const s = document.querySelector("[data-testid='cs-table'] [data-virtuoso-scroller]"); + | if (s) { + | if (s.scrollTop + s.clientHeight >= s.scrollHeight - 2) s.scrollTop = 0; + | else s.scrollTop = s.scrollTop + s.clientHeight * 0.8; + | } + | return JSON.stringify(pairs); + |}""".stripMargin + ) + .toString + // Minimal parse of [["1","\"m-00\""], ...]: strip the outer brackets, split the pairs. + "\\[\"(\\d+)\",\"(.*?)\"\\]".r + .findAllMatchIn(json.replace("\\\"", "\u0000")) + .foreach { m => + val value = m.group(2).replace("\u0000", "").stripPrefix("\"").stripSuffix("\"") + byOrdinal.update(m.group(1).toInt, value) + } + + val deadline = System.currentTimeMillis() + 30000 + while byOrdinal.size < expected && System.currentTimeMillis() < deadline do + harvestAndStep() + Thread.sleep(150) + + assert(byOrdinal.size == expected, s"collected ${byOrdinal.size} of $expected rows: ordinals ${byOrdinal.keys.toList.sorted}") + byOrdinal.toList.sortBy(_._1).map(_._2) + + private def twoToppedUpTopics(values: Seq[String]): (String, String, (String, String, String)) = + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + if values.nonEmpty then fixtures.produceRoundRobin(Seq(fqnA, fqnB), values) + (fqnA, fqnB, (tA, nsA, topicA)) + + test("CS-MO-0: a new consumer session defaults to Guaranteed ordering by publish time") { + // Owner decision (2026-08-11, direct instruction): the default is Guaranteed - the THIRD move + // of this default (the plan file's decision log is the record), superseding the 2026-08-09 + // Best effort default. The default is an EXACT REPLAY that delivers everything recorded up to + // Play and then auto-pauses caught-up (`cs-replay-caught-up`); Best effort stays the explicit + // choice for live following. CsDeliveryModesSpec pins what each mode means across every + // liveness state. This cell reads the config through openForTopicDefaults - the ONE + // navigation that leaves the configuration untouched, because the shared openForTopic pins + // Best effort for the generic feature lanes (see its comment). A future default change + // touches this cell and that pin, nothing else. + val (t, ns, topic) = fixtures.freshTopicParts() + val cs = ConsumerSessionPage(page) + + cs.openForTopicDefaults(t, ns, topic) + + assertThat(page.getByTestId("cs-delivery-order")).hasValue("guaranteed") + assertThat(page.getByTestId("cs-delivery-order-key")).hasValue("publish-time") + } + + test("CS-MO-0B: Guaranteed on a single non-persistent stream is an INSTANT caught-up - nothing recorded means nothing to replay") { + // Guaranteed refuses MULTI-stream non-persistent sessions (CS-DM-D4G); a SINGLE + // non-persistent stream is legal - and under the exact-replay contract its recorded range is + // empty by definition, so Play answers with the instant caught-up pause (observed 2026-08-09: + // the old "runs live and delivers" expectation failed with the session already 'paused'). + // Pinned through the control rather than inherited: whatever the session default is (today + // it is Guaranteed, CS-MO-0), this cell is about the explicit selection. + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.NonPersistentNonPartitioned) + val cs = ConsumerSessionPage(page) + + cs.openForTopic(t, ns, topic, persistency = "non-persistent") + cs.setDeliveryOrder(guaranteedLabel) + cs.play() + assertThat(page.getByTestId("cs-replay-caught-up")) + .isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + assert(cs.loadedCount == 0, s"an empty replay delivers nothing, got ${cs.loadedCount}") + + // A word published into the pause is GONE - a non-persistent topic keeps no history for any + // later chunk to replay - and the paused view says exactly that. + fixtures.produceRoundRobin(Seq(fqn), Seq("live")) + page.waitForTimeout(2000) // ample time to be wrong, then assert nothing arrived + assert(cs.loadedCount == 0, s"a paused session on a non-persistent topic must deliver nothing, got ${cs.loadedCount}") + assertThat(page.getByTestId("cs-paused-non-persistent-note")).isVisible() + } + + test("CS-MO-1: a two-topic history replay is DELIVERED in global publish-time order") { + // The discriminating case: two backlogs read by two racing consumers. Without the merge the + // delivery order is whatever the readers' interleaving happens to be - with 7+7 messages the + // chance of it landing exactly alternating is nil - so this asserts the merge, not luck. + val values = (0 until 14).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(values.size) + + val delivered = deliveredValues(values.size) + assert(delivered == values.toList, s"delivered order was $delivered, expected ${values.toList}") + // An ordered run confesses nothing: no out-of-order warning mark beside the loaded count. + // (The always-on mode chip was removed 2026-08-11; ordered delivery itself is asserted above.) + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + } + + test("CS-MO-2: the live tail comes out ordered too - the reorder grace at work") { + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(Seq.empty) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Latest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.assertState("running") + + val values = (0 until 8).map(i => f"live-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), values) + cs.awaitLoaded(values.size) + + val delivered = deliveredValues(values.size) + assert(delivered == values.toList, s"delivered order was $delivered, expected ${values.toList}") + } + + test("CS-MO-3: skip-n WITH merge keeps the cut exact and the remainder ordered") { + // The two promises composed end to end: exactly the globally-first 12 are dropped, and what + // survives arrives merged - the same session, both counted and ordered. + val values = (0 until 20).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("12") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(8) + + val delivered = deliveredValues(8) + assert(delivered == values.drop(12).toList, s"delivered $delivered, expected ${values.drop(12).toList}") + assert(cs.loadedCount == 8, s"the cut must deliver exactly 8, got ${cs.loadedCount}") + } + + test("CS-MO-4: Fastest delivers every message once without engaging the merge") { + // The control: no order is asserted - arrival interleaving is the broker's business - only + // that every message arrives exactly once with the merge NOT engaged. + val values = (0 until 14).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(fastestLabel) + cs.play() + cs.awaitLoaded(values.size) + + val delivered = deliveredValues(values.size) + assert(delivered.sorted == values.toList, s"every message must arrive exactly once, got $delivered") + // Fastest orders nothing and therefore warns of nothing. + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + } + + test("CS-MO-5: LATEST-N with merge - the metadata-resolved cut composes with ordered delivery") { + // Latest-n never touches the streaming merge for its cut (it is resolved from entry metadata + // before anything is delivered); the merge orders whatever the anchors then stream. The two + // must compose: exactly the globally-last 8, and in publish-time order. + val values = (0 until 20).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("8") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(8) + + val delivered = deliveredValues(8) + assert(delivered == values.drop(12).toList, s"delivered $delivered, expected ${values.drop(12).toList}") + } + + test("CS-MO-6: a MIXED persistent/non-persistent session merges too - live-only streams join at the tail") { + // A non-persistent topic retains nothing, so it can only ever speak from now on; the merge + // must treat it as one more stream with no history, not wedge waiting for a backlog that + // cannot exist. Old history first, then the live batch, ordered. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB, fqnB) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.NonPersistentNonPartitioned) + val backlog = (0 until 6).map(i => f"old-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA), backlog) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.assertState("running") + + val live = (0 until 6).map(i => f"live-$i%02d") + fixtures.produceRoundRobin(Seq(fqnA, fqnB), live) + cs.awaitLoaded(backlog.size + live.size) + + val delivered = deliveredValues(backlog.size + live.size) + assert( + delivered == (backlog ++ live).toList, + s"delivered $delivered, expected ${(backlog ++ live).toList}" + ) + } + + test("CS-MO-G1: GUARANTEED replays two topics COMPLETELY in exact key order, then auto-pauses caught up") { + val values = (0 until 14).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.play() + // The exact replay delivers EVERYTHING recorded up to Play - the globally-last message + // included, which the old contract held forever (observed 2026-08-09: 14 arrived where 13 + // were expected) - and then auto-pauses at the boundary with the caught-up banner. + cs.awaitLoaded(values.size) + assertThat(page.getByTestId("cs-replay-caught-up")).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + + val delivered = deliveredValues(values.size) + assert(delivered == values.toList, s"delivered $delivered, expected ${values.toList}") + // (The replay's announcement is the caught-up banner asserted above; the always-on chip + // label was removed 2026-08-11.) + cs.stop() + } + + test("CS-MO-G2: GUARANTEED is an exact replay - words after Play wait for the NEXT chunk, and Resume replays the delta") { + val seeds = (0 until 4).map(i => f"seed-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(seeds) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.play() + // The whole recorded range replays (observed 2026-08-09: 4 arrived where the old hold + // expected 3), then the boundary pause. + cs.awaitLoaded(seeds.size) + assertThat(page.getByTestId("cs-replay-caught-up")).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + + // A speaks PAST the boundary: recorded for the next chunk, delivered into this one never - + // give the pipeline ample time to be wrong, then assert it was not. + fixtures.produceRoundRobin(Seq(fqnA), (0 until 3).map(i => f"a-only-$i%02d")) + page.waitForTimeout(3000) + assert(cs.loadedCount == seeds.size, s"nothing past the boundary may enter this chunk, got ${cs.loadedCount} loaded") + + // Resume extends the boundary to now and replays the delta - exactly the three new words, + // in key order behind the seeds. + page.getByTestId("cs-replay-resume").click() + cs.awaitLoaded(seeds.size + 3) + assertThat(page.getByTestId("cs-replay-caught-up")).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + val delivered = deliveredValues(seeds.size + 3) + val expected = (seeds ++ (0 until 3).map(i => f"a-only-$i%02d")).toList + assert(delivered == expected, s"delivered $delivered, expected $expected") + cs.stop() + } + + test("CS-MO-T1: GUARANTEED over BATCHED data with real publish-time TIES follows the documented tiebreak - the replay completes, then pauses caught up") { + // Production data is batched, and every message in a batch shares ONE publish timestamp - so + // publish-time ties are the norm out there, while every other ordering arrangement in this + // suite tick-separates its sends precisely so that ties can never occur. This is the one cell + // where they exist on both axes at once: WITHIN each stream (a batch is one publish time) and + // ACROSS the two streams (the fixture retries until an A-batch and a B-batch land on the same + // millisecond, and throws rather than run tie-free). + // + // The contract is the documented total order, MessageOrderKey: + // (order time, topic FQN, ledger id, entry id, batch index) - publish time first, then topic, + // then position within the log. Within one topic that collapses to append order, so the + // expectation is the two append-ordered read-backs merged by (publish time, topic FQN) with a + // stable sort - entirely broker-derived, never produce-order guessing. The replay contract + // delivers the WHOLE recorded set in that order (observed 2026-08-09: all 16 arrived where + // the old held-tail expectation said 12) and then auto-pauses at the boundary. + // + // What fails here [bite-checked against the running app]: + // - a replay barrier that cannot emit while two stream heads SHARE a timestamp (the + // "< vs <=" stall): delivery stops short of the expectation and the await times out; + // - a tiebreak that is not the documented one, or that is non-deterministic: the delivered + // sequence differs from the computed order; + // - a leak past the replay boundary or a duplicate: deliveredValues collects MORE rows than + // expected and its exact-count assertion fails. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + val (valuesA, valuesB) = fixtures.produceTiedBatches(fqnA, fqnB, messagesPerBatch = 4) + + // The broker-derived expectation: everything, merged by (publish time, topic FQN). + val merged = + (fixtures.readAllMessages(fqnA).map(m => (m.getPublishTime, fqnA, m.getValue)) ++ + fixtures.readAllMessages(fqnB).map(m => (m.getPublishTime, fqnB, m.getValue))) + .sortBy { case (time, fqn, _) => (time, fqn) } // stable: same-key entries keep append order + assert(merged.size == valuesA.size + valuesB.size, s"read-back lost messages: ${merged.map(_._3)}") + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.setOrderTime("Publish time") + cs.play() + cs.awaitLoaded(merged.size) + assertThat(page.getByTestId("cs-replay-caught-up")).isVisible(vis(20000)) + cs.assertState("paused", timeoutMs = 10000) + + val delivered = deliveredValues(merged.size) + val expectedDelivered = merged.map(_._3).toList + assert(delivered == expectedDelivered, s"delivered order was $delivered, expected $expectedDelivered") + cs.stop() + } + + test("CS-MO-K1: ordering by EVENT TIME follows the application's clock, not the producers'") { + // Event times ASCEND within each stream (the merge preserves per-stream append order by + // contract - it never re-sorts inside a log) but INTERLEAVE differently from publish order: + // A carries the odd event times, B the even ones, and A always publishes first. Publish-time + // or arrival ordering would deliver e-01 first; event-time ordering must deliver e-00. + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + val eventBase = 1_700_000_000_000L + val produced = Vector(1, 0, 3, 2, 5, 4).map(i => (f"e-$i%02d", eventBase + i * 1000L)) + val producerA = fixtures.client.newProducer(org.apache.pulsar.client.api.Schema.STRING).topic(fqnA).enableBatching(false).create() + val producerB = fixtures.client.newProducer(org.apache.pulsar.client.api.Schema.STRING).topic(fqnB).enableBatching(false).create() + try + produced.zipWithIndex.foreach { case ((v, eventTime), i) => + val producer = if i % 2 == 0 then producerA else producerB + producer.newMessage().value(v).eventTime(eventTime).send() + Thread.sleep(2) + } + finally + producerA.close() + producerB.close() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.setOrderTime("Event time") + cs.play() + cs.awaitLoaded(produced.size) + + val delivered = deliveredValues(produced.size) + val expected = (0 until 6).map(i => f"e-$i%02d").toList // ascending EVENT time + assert(delivered == expected, s"delivered $delivered, expected event-time order $expected") + // (The chip that used to name the selected time was removed 2026-08-11 with the always-on + // mode label; the event-time ORDER asserted above is the contract.) + } + + test("CS-MO-K2: BROKER publish time on a broker without entry metadata is refused with the remediation") { + // The no-admin-steps contract: Dekaf never requires broker configuration - it detects the + // gap and names exactly what an administrator WOULD change, and where. + val values = (0 until 4).map(i => f"m-$i%02d") + val (fqnA, fqnB, (t, ns, topic)) = twoToppedUpTopics(values) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.setOrderTime("Broker publish time") + cs.play() + + assertThat(page.getByText("Broker publish time is unavailable").first()).isVisible(vis(15000)) + assert(cs.loadedCount == 0, s"a refused session must deliver nothing, got ${cs.loadedCount}") + } + + test("CS-MO-8: a SINGLE-STREAM session builds no ordering layer - and the chip does not pretend it did") { + // The config may ask for best-effort order, but one resolved delivery stream is already in + // order: the server builds no layer, pays no reorder latency, and says so on the wire. A + // chip claiming a window that does not exist would be exactly the kind of latency folklore + // this feature must not create. + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 5) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(5) + + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + } + + test("CS-MO-7: LOAD - 20 partitions, 40,000 messages through the merge, delivered completely and on time") { + // The throughput half of the wide-session story on a real broker: a partitioned topic is 20 + // delivery streams racing through one continuous merge. The oracle is completion within the + // timeout - a wedged or quadratic merge fails this by never finishing - plus the mode's chip. + val t = fixtures.createTenant() + val ns = fixtures.createNamespace(t) + val topic = s"load-${System.currentTimeMillis()}" + val fqn = s"persistent://$t/$ns/$topic" + fixtures.admin.topics().createPartitionedTopic(fqn, 20) + fixtures.produceStringsFast(fqn, 40000) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(40000, timeoutMs = 120000) + + assert(cs.loadedCount == 40000, s"every message must arrive, got ${cs.loadedCount}") + assert(cs.state == "running", s"the session must still be healthy, got '${cs.state}'") + cs.stop() + } diff --git a/e2e/src/test/scala/features/consumersession/CsPauseLoopSpec.scala b/e2e/src/test/scala/features/consumersession/CsPauseLoopSpec.scala new file mode 100644 index 000000000..8b1ff9f85 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsPauseLoopSpec.scala @@ -0,0 +1,131 @@ +package features.consumersession + +import harness.DekafSuite + +/** PAUSE IS FAST AND LOSSLESS - both halves of the fast-pause contract, proven together. + * + * `paused` now lands on the server's confirmation (intake closed, everything unsent held), not on + * the old "quiet second" of the loaded-rate gauge - so the button flips in an RPC round trip and + * the in-flight tail finishes rendering into the paused view. What makes that flip honest is the + * server's ledger: prefetched messages refused at the closed gate are handed back for redelivery, + * the limiter's backlog waits unacknowledged, and whatever was already sent lands and is counted. + * + * This spec drives that ledger hard and then demands the books balance to the message: + * + * - a 50,000-message backlog is drained through repeated HOT pauses - clicked mid-flow with the + * prefetch queues full, which is exactly the refuse-at-the-gate / redeliver-on-resume path; + * - one more burst is produced INTO a paused session - the retention path: a paused session + * must deliver NOTHING and hand it all over on resume; + * - and the final set must be EXACTLY the 55,000 the broker holds, message for message. + * + * WHY THE SET AND NOT THE COUNT. This test used to end on `loaded == 55,000` alone, and that + * number cannot tell a correct run from a plausible one: one message lost anywhere plus one + * delivered twice anywhere reaches it exactly. Worse, `cs-loaded` is not even read from the + * delivered rows - it is the server's own `numMessageSent` - so a message counted but never + * shown, or shown but never counted, is invisible to it. Every payload here is therefore UNIQUE + * (`bk-*` for the backlog, `burst-*` for what is produced into the pause), the expectation is + * read back from the BROKER, and the delivered set is read out of the app's own export. + * `harness.DeliveredMessagesSpec` pins that this comparison really does catch what the count + * cannot. The counts are kept alongside it, deliberately - see the assertions at the end. + */ +class CsPauseLoopSpec extends DekafSuite: + + private val backlog = 50_000 + private val pausedBurst = 5_000 + private val total = backlog + pausedBurst + + /** Poll the loaded counter until it reaches `n` (the toolbar counter moves before rows paint, + * and mid-drain there is no exact value to wait for - only a threshold to cross). */ + private def awaitLoadedAtLeast(cs: ConsumerSessionPage, n: Int, timeoutMs: Long): Unit = + val deadline = System.currentTimeMillis() + timeoutMs + var current = cs.loadedCount + while current < n && System.currentTimeMillis() < deadline do + Thread.sleep(200) + current = cs.loadedCount + assert(current >= n, s"loaded $current, expected at least $n within ${timeoutMs}ms") + + test("CS-PL-1: 50,000 messages through a hot pause/resume loop - instant pauses, exact SET, zero loss") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceStringsFast(fqn, backlog, prefix = "bk") + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + // RETENTION IS PART OF THE ARRANGEMENT. A session drops its oldest messages past the display + // limit (1,000,000 by default since 2026-08-11; 10,000 when this cell was written, which would + // have left 45,000 of these unobservable). The explicit ask keeps the cell INDEPENDENT of that + // product default, whatever it becomes; `retainedCount` below proves the ask took effect. + cs.setNumDisplayItems(total + 5_000) + cs.play() + + // Four pauses clicked HOT - mid-drain, prefetch full, frames in flight. Each pause must + // confirm promptly (the RPC plus a React commit, nowhere near the old gauge's multi-second + // quiet window), and each resume must pick the drain back up without dropping or repeating + // whatever the closed gate refused. + (1 to 4).foreach { round => + awaitLoadedAtLeast(cs, n = round * 8000, timeoutMs = 120000) + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 5000) + cs.play() + cs.assertState("running", timeoutMs = 20000) + } + + cs.awaitLoaded(backlog, timeoutMs = 180000) + + // The retention leg: produce INTO the paused session. A paused session delivers NOTHING - and + // that is asserted from BOTH sources, the server's counter and the browser's own buffer, so a + // delivery that moved only one of them cannot pass. + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 5000) + val loadedWhilePaused = cs.loadedCount + val retainedWhilePaused = cs.retainedCount + fixtures.produceStringsFast(fqn, pausedBurst, prefix = "burst") + Thread.sleep(1500) + assert( + cs.loadedCount == loadedWhilePaused, + s"a paused session delivered messages: $loadedWhilePaused -> ${cs.loadedCount}" + ) + assert( + cs.retainedCount == retainedWhilePaused, + s"a paused session added rows: $retainedWhilePaused -> ${cs.retainedCount}" + ) + + cs.play() + cs.awaitLoaded(total, timeoutMs = 120000) + + // THE COUNTS, KEPT ON PURPOSE - do not "simplify" them away when the set assertion below + // already looks stronger. They are read from a DIFFERENT source than the set (the server's + // numMessageSent, and the browser's buffer length) and they are read AFTER a quiet window, so + // together they catch the one thing a set comparison cannot: a straggling extra delivery that + // arrives after the set first matched. + Thread.sleep(2000) + assert( + cs.loadedCount == total, + s"expected exactly $total, got ${cs.loadedCount} - a loss deflates, a duplicate inflates" + ) + assert(cs.state == "running", s"the session must still be healthy, got '${cs.state}'") + assert( + cs.retainedCount == total, + s"the browser holds ${cs.retainedCount} of $total messages - display retention dropped rows, so the " + + "exported set below would be a tail rather than the whole delivery (raise setNumDisplayItems)" + ) + + // THE SET. The expectation comes from the broker, not from the produce call: what the topic + // really holds is the only ground truth for what the session should have shown. + val onTheBroker = fixtures.readAllMessages(fqn).map(_.getValue).toList + assert( + onTheBroker.size == total && onTheBroker.distinct.size == total, + s"arrangement broken: the broker holds ${onTheBroker.size} message(s), ${onTheBroker.distinct.size} distinct, " + + s"expected $total unique - the payloads must be unique or the set below proves nothing" + ) + + cs.pauseFromToolbar() + cs.assertState("paused", timeoutMs = 15000) + DeliveredMessages.assertExactly( + cs.exportedValues(), + onTheBroker, + "the pause loop delivered a different set than the broker holds" + ) + cs.stop() + } diff --git a/e2e/src/test/scala/features/consumersession/CsProjectionColoringSpec.scala b/e2e/src/test/scala/features/consumersession/CsProjectionColoringSpec.scala index fe04cba17..12ff61e57 100644 --- a/e2e/src/test/scala/features/consumersession/CsProjectionColoringSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsProjectionColoringSpec.scala @@ -3,7 +3,9 @@ package features.consumersession import harness.DekafSuite import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.assertions.LocatorAssertions +import com.microsoft.playwright.options.SelectOption import org.apache.pulsar.client.api.Schema +import scala.jdk.CollectionConverters.* class CsProjectionColoringSpec extends DekafSuite: private def count(n: Int) = new LocatorAssertions.HasCountOptions().setTimeout(20000) @@ -15,22 +17,51 @@ class CsProjectionColoringSpec extends DekafSuite: val p = client.newProducer(Schema.STRING).topic(fqn).create() try values.foreach(p.send) finally p.close() - test("CS-12 (P0): a projection adds a column whose header equals the projection label") { + private def produceKeyed(fqn: String, keyed: Seq[(String, String)]): Unit = + val p = client.newProducer(Schema.STRING).topic(fqn).create() + try keyed.foreach((k, v) => p.newMessage().key(k).value(v).send()) finally p.close() + + /** The rendered PROJECTION cells, in row order. + * + * A projection `` carries no test-id of its own (the projections are a variable-length run of + * columns), but its position is fixed by `Message.tsx`: the projections sit immediately before + * the instrumented value cell. Addressing them relative to that anchor reads the real projected + * output without asking `ui/` for new instrumentation. */ + private def projectionCells(cs: ConsumerSessionPage): List[String] = + page.locator("[data-testid='cs-message-value']").locator("xpath=preceding-sibling::td[1]") + .allInnerTexts().asScala.toList.map(_.trim) + + test("CS-12 (P0): a projection adds a column whose header is its label and whose cells hold the projected value") { val (t, ns, topic) = fixtures.freshTopicParts() val fqn = s"persistent://$t/$ns/$topic" - produce(fqn, (1 to 3).map(i => s"m-$i")) + // Distinct key and value per message, so a projection OF THE KEY cannot be satisfied by + // rendering the value column twice - which is what an assertion on the header alone allowed. + produceKeyed(fqn, (1 to 3).map(i => s"k-$i" -> s"m-$i")) val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() cs.addProjection() cs.projectionLabel.first().fill("MyCol") + // Project the message KEY rather than the default (the whole value). The target select has no + // test-id; its option values are the discriminator, and are unique to this control on the page. + cs.sessionProjections.locator("select:has(option[value='BasicMessageFilterKeyTarget'])") + .first().selectOption(new SelectOption().setValue("BasicMessageFilterKeyTarget")) cs.setStartFrom("Earliest message") cs.play() assertThat(cs.messages).hasCount(3, count(3)) assertThat(cs.projectionColumnHeader).hasCount(1, count(1)) assertThat(cs.projectionColumnHeader).containsText("MyCol", contains) + + // THE assertion: the projected VALUES, exactly, row by row - the server really evaluated the + // projection and the results really landed in the right rows. The header is generated entirely + // from local config and would render identically for a projection that computed nothing. + assert( + projectionCells(cs) == List("\"k-1\"", "\"k-2\"", "\"k-3\""), + s"the projection column holds ${projectionCells(cs)}" + ) + // ... and it is the KEY, not a second copy of the value column beside it. + assert(cs.columnValues("value") == List("m-1", "m-2", "m-3"), s"the value column holds ${cs.columnValues("value")}") } test("CS-13: coloring modal picks a swatch, applies it, and a matching row is colored") { @@ -40,7 +71,6 @@ class CsProjectionColoringSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() cs.addColoringRule() // default rule = empty filter chain (matches all) cs.coloringBg.first().click() // open the "Pick a Color" modal @@ -63,7 +93,6 @@ class CsProjectionColoringSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.revealAdvanced() // Session rule -> red. cs.addColoringRule() @@ -86,37 +115,44 @@ class CsProjectionColoringSpec extends DekafSuite: test("CS-14: changing the deserializer changes how values decode") { val (t, ns, topic) = fixtures.freshTopicParts() val fqn = s"persistent://$t/$ns/$topic" - produce(fqn, Seq("\"apple\"")) // raw bytes are the JSON string "apple" (with quotes) + // The raw bytes on the wire are the seven characters "apple" - a JSON string INCLUDING its + // quotes. That is what makes the two deserializers distinguishable: one more decode step + // removes exactly one layer of quoting. + produce(fqn, Seq("\"apple\"")) val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) cs.setStartFrom("Earliest message") cs.play() assertThat(cs.messages).hasCount(1, count(1)) - val schemaRendered = cs.firstValueCell.innerText() // topic STRING schema -> "apple" (quoted) + // Topic STRING schema: the bytes decode to the 7-character string `"apple"`, which the value + // column then renders as JSON - so the quotes are escaped and a second pair is added. + // `columnValues` strips the outer rendering pair, leaving the decoded string itself. + assert(cs.columnValues("value") == List("\\\"apple\\\""), s"schema rendering was ${cs.columnValues("value")}") cs.stop() // back to config view cs.setDeserializer("Treat raw bytes as JSON") cs.play() assertThat(cs.messages).hasCount(1, count(1)) - val jsonRendered = cs.firstValueCell.innerText() // JSON parse -> apple (unquoted) - - assert(schemaRendered != jsonRendered, - s"deserializer change should alter value rendering: schema='$schemaRendered' json='$jsonRendered'") + // Raw bytes as JSON: the same bytes are PARSED, so the value is the string `apple` with no + // quotes of its own, rendered with one pair which `columnValues` strips. + // + // Both renderings are pinned exactly, not merely asserted to differ: "they differ" is satisfied + // by any two wrong answers - a decode that dropped a character, or an error placeholder in + // either leg - which is what this test used to accept. + assert(cs.columnValues("value") == List("apple"), s"JSON rendering was ${cs.columnValues("value")}") } - test("CS-15: the Advanced reveal is one-way (once revealed, stays)") { + test("CS-15: all session settings are visible without an advanced disclosure") { val (t, ns, topic) = fixtures.freshTopicParts() val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - assertThat(cs.advancedToggle).isVisible(vis) - assertThat(cs.sessionFilters).hasCount(0, count(0)) - - cs.revealAdvanced() + assertThat(page.getByTestId("cs-advanced-toggle")).hasCount(0, count(0)) + assertThat(page.getByText("Show advanced settings")).hasCount(0, count(0)) + assertThat(page.getByTestId("cs-delivery-order")).isVisible(vis) + assertThat(page.getByText("Limit num. display messages")).isVisible(vis) assertThat(cs.sessionFilters).isVisible(vis) - - cs.sessionFilterPanel.addFilter() // adds advanced config -> reveal locks on - assertThat(cs.advancedToggle).hasCount(0, count(0)) // toggle is gone - assertThat(cs.sessionFilters).isVisible(vis) // advanced section stays + assertThat(cs.sessionProjections).isVisible(vis) + assertThat(cs.sessionColoring).isVisible(vis) } diff --git a/e2e/src/test/scala/features/consumersession/CsRetentionIsolationSpec.scala b/e2e/src/test/scala/features/consumersession/CsRetentionIsolationSpec.scala new file mode 100644 index 000000000..cc1d57af1 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsRetentionIsolationSpec.scala @@ -0,0 +1,52 @@ +package features.consumersession + +import harness.DekafSuite + +import scala.jdk.CollectionConverters.* + +/** SUBSCRIPTION ISOLATION: what a consumer session does and does not touch on the broker. + * + * The promise the docs make is precise: Dekaf never mutates ANOTHER subscription - it reads + * through its own exclusive, non-durable one, which disappears with the session (its live + * cursor can delay retention only while the session exists; that part is documented, not + * denied). Both halves are pinned against a real broker here, because no unit test can. + */ +class CsRetentionIsolationSpec extends DekafSuite: + + test("CS-ISO-1: an application's durable subscription is untouched, and Dekaf's own cursor disappears on stop") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(fqn, 10) + + // The application's own durable subscription, parked at the beginning with a real backlog. + fixtures.admin.topics().createSubscription(fqn, "app-durable", org.apache.pulsar.client.api.MessageId.earliest) + def appSub() = fixtures.admin.topics().getStats(fqn).getSubscriptions.get("app-durable") + val backlogBefore = appSub().getMsgBacklog + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(10) + + // While Dekaf reads: its own __dekaf_ subscription exists alongside, and the app's cursor + // has not moved a message. + val subsWhileRunning = fixtures.admin.topics().getStats(fqn).getSubscriptions.keySet.asScala.toSet + assert(subsWhileRunning.exists(_.startsWith("__dekaf_")), s"expected a dekaf subscription, got $subsWhileRunning") + assert(appSub().getMsgBacklog == backlogBefore, s"the app's backlog moved: ${appSub().getMsgBacklog} != $backlogBefore") + + cs.stop() + + // The dekaf subscription (and with it the cursor that could pin retention) is GONE; the + // app's is exactly where it was. Poll briefly - the unsubscribe is async on the broker. + val deadline = System.currentTimeMillis() + 15000 + var remaining = Set.empty[String] + while + remaining = fixtures.admin.topics().getStats(fqn).getSubscriptions.keySet.asScala.toSet + remaining.exists(_.startsWith("__dekaf_")) && System.currentTimeMillis() < deadline + do Thread.sleep(250) + + assert(!remaining.exists(_.startsWith("__dekaf_")), s"dekaf subscription still present after stop: $remaining") + assert(remaining.contains("app-durable"), s"the app's subscription vanished: $remaining") + assert(appSub().getMsgBacklog == backlogBefore, s"the app's backlog moved after stop: ${appSub().getMsgBacklog}") + } diff --git a/e2e/src/test/scala/features/consumersession/CsSchemaValueSpec.scala b/e2e/src/test/scala/features/consumersession/CsSchemaValueSpec.scala new file mode 100644 index 000000000..b60fd3d11 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsSchemaValueSpec.scala @@ -0,0 +1,69 @@ +package features.consumersession + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.pulsar.client.api.Schema + +import java.nio.charset.StandardCharsets.UTF_8 + +/** A consumer session reading a topic with a REGISTERED Pulsar schema. + * + * Every other consumer-session fixture is `Schema.STRING` raw bytes, so the server's + * schema-fetch-and-decode path (`getSchemasByTopic` -> per-message decode) had no end-to-end + * coverage at all: a regression that decoded schema'd values to garbage, fetched the wrong + * schema, or silently fell back to raw bytes passed every session test in the suite. The only + * schema coverage was `SchemaSpec` - the schema MANAGEMENT page, a different code path - and the + * UI-side deserializer dropdown, which never involves the registry. + */ +class CsSchemaValueSpec extends StartFromSupport: + + private val mapper = new ObjectMapper() + + test("CS-SCH-1: a topic with a registered JSON schema renders DECODED objects, not raw bytes") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + + // Register the schema through the ADMIN api, then produce plain bytes with a schema-less + // producer (standalone does not enforce schema validation). Nothing but the registry tells + // the session these bytes are JSON - which is exactly the path under test. The fixture + // asserts the registration premise (type JSON really stored) before the UI is driven. + val schemaDef = + """{"type":"record","name":"TestOrder","fields":[{"name":"id","type":"int"},{"name":"item","type":"string"}]}""" + fixtures.registerJsonSchema(fqn, schemaDef) + + val payloads = Seq("""{"id":1,"item":"boots"}""", """{"id":2,"item":"socks"}""") + val producer = client.newProducer(Schema.BYTES).topic(fqn).enableBatching(false).create() + try payloads.foreach(p => producer.send(p.getBytes(UTF_8))) + finally producer.close() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.assertState("running") + cs.awaitLoaded(2) + + // Parsed, not substring-matched: each rendered cell must BE one of the two produced objects. + // What fails here [phase-2 bite-check - needs the running app]: + // - a raw-bytes fallback renders the document as a JSON STRING (`"{\"id\":1,...}"`); after + // the one-quote-pair strip that text still carries escape backslashes and fails the parse; + // - a broken or wrong-schema decode fails the parse or the object equality; + // - jackson-tree equality keeps the assertion independent of key order and whitespace, but + // if the app deliberately reshapes decoded values (truncation, annotation) this needs + // phase-2 eyes on the real rendering rather than a weaker assertion now. + val rendered = harness.Eventually.eventually(timeoutMs = 20000, intervalMs = 400) { + val cells = cs.allColumnValues("value") + assert(cells.size == 2, s"expected 2 rendered rows, got $cells") + cells + } + val renderedNodes = rendered.map { cell => + try mapper.readTree(cell) + catch + case t: Throwable => + fail(s"a value cell does not parse as JSON (raw-bytes fallback or broken decode?): '$cell' (${t.getMessage})") + } + renderedNodes.foreach(n => assert(n.isObject, s"a value cell decoded to ${n.getNodeType}, expected a JSON object: $n")) + assert( + renderedNodes.toSet == payloads.map(mapper.readTree).toSet, + s"decoded objects differ from the produced ones: rendered $renderedNodes, produced $payloads" + ) + } diff --git a/e2e/src/test/scala/features/consumersession/CsStartFromMatrixSpec.scala b/e2e/src/test/scala/features/consumersession/CsStartFromMatrixSpec.scala new file mode 100644 index 000000000..b10386a8b --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsStartFromMatrixSpec.scala @@ -0,0 +1,309 @@ +package features.consumersession + +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions +import org.apache.pulsar.client.api.{Message as PulsarMessage, Schema} + +/** The two COUNTING Start-From modes - "Skip first n messages" and "Latest n messages" - crossed + * with both ways an application writes to Pulsar (batched / unbatched) and with the whole topic + * matrix ({persistent, non-persistent} x {partitioned, non-partitioned}). + * + * Why the batching axis: until 2026-07-25 both modes were implemented on + * `PulsarAdmin.examineMessage`, which addresses ENTRIES. The Java producer packs many messages into + * one entry by default, so in any ordinary application "the 6th entry" and "the 6th message" are + * different things - and every fixture the suite had produced exactly one message per entry, which + * made the two indistinguishable. See `harness.BatchingFixtureSpec` for the broker-level proof of + * both facts, and `CsStartFromOutcomesSpec` for the same two modes examined in isolation. + * + * The contracts asserted here, as implemented. BOTH counting modes are GLOBAL in COUNT - n in + * total across the whole session, never n per partition - and the merge takes each partition in + * its own APPEND order, comparing publish times only across the partitions' current heads. On + * same-clock producers that is the publish-time answer; where producer clocks disagree, WHICH n + * can differ from a strict global-publish-time sort (a buried out-of-order timestamp is not dug + * out), while the count stays exactly n. globalStartFromTest pins both sides of that line. + * - **Skip first n** drops n and delivers everything else. On a single ordered log that is + * exactly "start at message n + 1"; across partitions it is the n oldest as the merge sees + * them, whichever partitions they came from. + * - **Latest n** delivers EXACTLY n in total. It used to be resolved per physical topic, so + * "latest 2" on a 3-partition topic returned six; CS-SFM-4 is the regression against that. + * + * The cells generated from the topic matrix still funnel their payload through ONE partition, where + * the two contracts coincide with "the first / last n of that log" - which is what makes their + * expectation a plain slice of the payload. The genuinely-spread cases, where a per-topic answer + * and the global one differ, are CS-SFM-3/4 at the bottom of this spec. + * + * NON-PERSISTENT quadrants get a different assertion, not a weaker one. Such a topic retains + * nothing - anything published while no consumer is attached is dropped by the broker forever, and + * PulsarAdmin will not examine one at all - so there is no history to count into. The app now says + * so up front: both counting modes are rendered DISABLED with a note, and the server rejects them + * for an all-non-persistent session rather than degrading into a silent "from now". Those cells + * assert the refusal, plus that the live tail still works and none of the pre-produced messages + * come back. + */ +class CsStartFromMatrixSpec extends StartFromSupport: + + /** 12 messages -> 3 entries when batched at 4/entry: entry positions and message positions are + * far enough apart that no off-by-a-factor can be mistaken for a correct answer. `m-12` is the + * sentinel and is in both expectations below. */ + private val payload = (1 to 12).map(i => f"m-$i%02d") + + /** A counting mode: the dropdown label, and the messages it must leave showing. */ + private case class Counting(id: String, uiLabel: String, n: String, expected: Seq[String]) + private val counting = List( + Counting("skip-first-n", "Skip first n messages", "5", payload.drop(5)), // m-06 .. m-12 + Counting("latest-n", "Latest n messages", "5", payload.takeRight(5)) // m-08 .. m-12 + ) + + /** The single ordered log the expectation is stated against. + * + * A partitioned topic is fed through ONE partition on purpose: "the first n messages" is only + * defined on a totally ordered log, and a partitioned topic has none. This still drives the + * partitioned code path end to end - the session subscribes to the parent, so every partition is + * expanded, subscribed and seeked - it only makes the expectation unambiguous. The genuinely + * spread-across-partitions cases are CS-SFM-3/4 below. */ + private def logOf(fqn: String, kind: fixtures.TopicKind): String = + if kind.isPartitioned then s"$fqn-partition-0" else fqn + + fixtures.TopicKind.all.foreach { kind => + if kind.retains then + Produce.values.foreach { produce => + counting.foreach { c => + val cell = s"[${kind.label}] [${produce.label}] [${c.id}]" + // The batched cells are the regression: before 2026-07-25 both modes counted broker + // ENTRIES, and with these 12 messages in 3 entries "skip 5" started at m-09 (it skipped + // eight) while "latest 5" showed all twelve. Both mechanisms are pinned in BATCH-2. + test(s"CS-SFM-1 $cell: shows exactly the expected MESSAGES") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + produceAs(produce, logOf(fqn, kind), payload) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom(c.uiLabel) + cs.startFromN.fill(c.n) + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, c.expected) + } + } + } + else + // ONE test per non-persistent kind. The batched/unbatched axis is decorative here: a + // non-persistent topic retains nothing, so the pre-produced payload is dropped by the broker + // regardless of how it was batched (and `produceBatched` cannot even verify itself without a + // managed ledger). BOTH counting modes must be refused, so they are asserted together rather + // than across four byte-identical (produce x counting) cells that only differ in which label + // they happen to check. + test(s"CS-SFM-2 [${kind.label}]: no history to count - both counting modes refused, the live stream is exact") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + produceAs(Produce.Unbatched, fqn, payload) // no consumer attached -> the broker drops these forever + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + // The counting modes cannot mean anything here, and are no longer offered as if they could: + // each is rendered disabled and a note says why. Asserting the refusal is stronger than + // asserting that a permitted-but-meaningless selection happened to behave. + counting.foreach { c => + assert( + cs.disabledStartFromLabels.contains(c.uiLabel), + s"'${c.uiLabel}' is still selectable on a non-persistent topic; disabled: ${cs.disabledStartFromLabels}" + ) + } + assertThat(cs.startFromNonPersistentNote).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(20000)) + + // What the topic CAN do still has to work, and the pre-produced messages must stay gone. + cs.setStartFrom("Latest message") + cs.play() + cs.assertState("running") + awaitConsumersFlowing(fqn, kind) + // Prove the stream is live BEFORE asserting the negative - an empty table only means + // "nothing was retained" once we know a message would have shown up. + awaitSessionStreaming(cs, fqn) + + val live = Seq("live-1", "live-2", "live-last") + produceAs(Produce.Unbatched, fqn, live) + assertLoadedExactly(cs, live) + // Full-list read: leaked history would be OLDER than the live rows, i.e. exactly what the + // auto-scrolled table unmounts first. + val rendered = cs.allColumnValues("value") + assert( + !rendered.exists(_.startsWith("m-")), + s"a non-persistent topic answered a counting Start-From with retained history: $rendered" + ) + } + } + + // ------------------------------------------------------------------------------------------- + // Messages spread across ALL partitions - where a per-topic answer and the global one diverge + // + // Everything above funnels its payload through one partition, where "the first n" and "the last + // n" of that single log ARE the global answer. These two put part of the payload on every + // partition, which is the only shape in which a per-partition implementation and the global + // contract disagree: "skip n" has to pick the n oldest across independent consumers, and + // "latest n" has to return n messages rather than n per partition. + // + // Deliberately NO message filter, value projection or coloring rule is configured here, so a + // failure here is about start-from and nothing else. + // + // The CONCURRENT-ENTRY hazard that used to live next door is fixed: a session still gets ONE + // GraalVM JS context (`ConsumerSessionContextPool` pins the pool to size 1) while a partitioned + // topic gives each partition its own listener thread, but `ConsumerSessionContext.exclusively` now + // leases that context for a WHOLE message, so two threads can no longer be inside it at once + // ("Multi threaded access ... is not allowed for language(s) js") nor interleave a + // `setCurrentMessage` with another message's chain. + // + // The ORDERING race that used to be described here is fixed as well: `ConsumerListener.received` + // now resolves AND processes inside `startFromOrdering.inOrder`, so the vector the merge chose is + // handed to the target handler under the very lock that chose it. It is no longer possible for two + // listener threads to resolve in one order and deliver in another. + // + // The OUTPUT stream's ORDER and TERMINATION hazards this comment used to file as still-open are now + // fixed too, and this arrangement does not exercise them either. Every write goes through + // `ConsumerSessionRunner.sendResponse`, which holds `sendLock`, and: + // - `sendResponse` now BUILDS the response, start-from progress and all, INSIDE that lock (behind + // an `if !streamCompleted` check), so an older incomplete frame can no longer overtake a newer + // complete one; + // - `stop()` now calls `observer.onCompleted()` INSIDE the lock behind a sticky `streamCompleted` + // terminal gate, so nothing is written after completion and completion never interleaves with an + // in-flight `onNext`. + // Both are pinned at the server tier by `sessionOutputSerializationTest` (suite "progress never goes + // backwards, and nothing follows the end of the stream"); see e2e/README.md §6. + // ------------------------------------------------------------------------------------------- + + private val spreadKind = fixtures.TopicKind.PersistentPartitioned + + /** Gap left between two publishes by `spread`. + * + * Both counting modes are defined over the merged stream ordered by PUBLISH TIME, so an + * expectation of the form "the globally-earliest n" is only well defined while no two messages + * share a millisecond. Enforced by ARRANGEMENT (and checked in `globalOrder`) rather than by + * re-implementing the server's tie-break here, which would make the test agree with a broken + * tie-break instead of catching it. */ + private val PublishGapMs = 20L + + /** What each partition actually holds, oldest first, read back from the broker - the ORACLE both + * multi-partition expectations are derived from. The router's starting partition is chosen at + * random, so hard-coding the split would be wrong; only the broker knows it. */ + private def partitionContents(fqn: String): Vector[Vector[PulsarMessage[String]]] = + (0 until spreadKind.partitions).toVector.map(p => fixtures.readAllMessages(s"$fqn-partition-$p")) + + /** The merged stream in the GLOBAL publish-time order both counting modes are defined over, + * derived entirely from the broker - never from the payload names. + * + * A stable sort by publish time is a total order only while no two messages sharing a publish + * time sit in DIFFERENT partitions; `spread` is what makes that true and this asserts it rather + * than assuming it. Equal times do survive inside a producer BATCH - one entry, one publish time, + * every message in it - but a batch lives in a single partition and is read back in produce + * order, which the stable sort preserves. */ + private def globalOrder(perPartition: Vector[Vector[PulsarMessage[String]]]): Vector[String] = + val flat = perPartition.zipWithIndex.flatMap { case (msgs, p) => msgs.map(m => (p, m)) } + flat.groupBy { case (_, m) => m.getPublishTime }.foreach { case (publishTime, group) => + val partitions = group.map { case (p, _) => p }.distinct + assert( + partitions.size == 1, + s"publish time $publishTime is shared by partitions $partitions, so 'the globally-earliest n' is " + + s"not decidable from publish time alone: ${group.map { case (_, m) => m.getValue }}" + ) + } + flat.sortBy { case (_, m) => m.getPublishTime }.map { case (_, m) => m.getValue } + + /** Put `values` on the partitioned topic so that EVERY partition ends up holding some of it, and + * so that publish times never collide across partitions. + * + * Unbatched goes through the parent and lets the default round-robin router spread it - ordinary + * produce traffic - one message at a time. Batched cannot go that way: with batching on, the + * round-robin router only switches partition every `batchingPartitionSwitchFrequencyByPublishDelay` + * x publish delay, so a burst lands entirely on ONE partition (observed - the whole payload on + * partition 1). Addressing the partitions directly is the only way to get batched entries onto + * more than one of them, and the gap then goes BETWEEN batches, a batch being one entry with one + * publish time shared by everything in it. + * + * The gaps are ARRANGEMENT, not readiness waits: the only way to put messages at distinct known + * instants is to publish them at distinct instants. */ + private def spread(produce: Produce, fqn: String, values: Seq[String]): Unit = produce match + case Produce.Unbatched => + // One producer for the whole payload, so the round-robin router really rotates through the + // partitions instead of restarting from a fresh random one per message. + val producer = client.newProducer(Schema.STRING).topic(fqn).enableBatching(false).create() + try + values.foreach { value => + val sentAt = System.currentTimeMillis() + producer.send(value) + awaitClockGap(sentAt, PublishGapMs) + } + finally producer.close() + case Produce.Batched => + val perPartition = math.ceil(values.size.toDouble / spreadKind.partitions).toInt + values.grouped(perPartition).zipWithIndex.foreach { case (chunk, i) => + val sentAt = System.currentTimeMillis() + fixtures.produceBatched(s"$fqn-partition-$i", chunk, MessagesPerBatch) + awaitClockGap(sentAt, PublishGapMs) + } + + Produce.values.foreach { produce => + + test(s"CS-SFM-3 [${spreadKind.label}] [${produce.label}] [skip-first-n]: drops the GLOBALLY-EARLIEST n across every partition") { + val skip = 3 + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(spreadKind) + spread(produce, fqn, payload) + val perPartition = partitionContents(fqn) + assert( + perPartition.flatten.map(_.getValue).toSet == payload.toSet, + s"the arrangement did not land: ${perPartition.map(_.map(_.getValue))}" + ) + // Every partition has to hold some of the payload, or the session never merges anything and + // the global contract is indistinguishable from the single-log one. + assert( + perPartition.forall(_.nonEmpty), + s"this test needs every partition to hold part of the payload, got ${perPartition.map(_.size)}" + ) + + // Not payload.drop(skip): WHICH messages are globally oldest is a fact about publish times + // that only the broker knows - the router decides where each message lands, and a batch shares + // one publish time across everything in it. + val expected = globalOrder(perPartition).drop(skip) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, spreadKind.scheme) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill(skip.toString) + cs.play() + cs.assertState("running") + // An exact SET, not a count. Dropping any other three messages also leaves nine, so a count + // would pass for an implementation that dropped the first three to ARRIVE rather than the + // three oldest - which is precisely what the per-topic implementation did. + assertLoadedExactlyWithCounter(cs, expected) + } + + test(s"CS-SFM-4 [${spreadKind.label}] [${produce.label}] [latest-n]: shows exactly the globally-latest n, not n per partition") { + val n = 2 + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(spreadKind) + spread(produce, fqn, payload) + val perPartition = partitionContents(fqn) + assert( + perPartition.flatten.map(_.getValue).toSet == payload.toSet, + s"the arrangement did not land: ${perPartition.map(_.map(_.getValue))}" + ) + + // "Latest n" is GLOBAL: exactly the n newest messages of the merged stream, however they are + // distributed. Derived from the broker's own publish times, not from the message names. + val expected = globalOrder(perPartition).takeRight(n) + assert(expected.size == n, s"the globally-latest $n should be $n messages, got $expected") + + // THE regression. "Latest n" used to be resolved per physical topic, so this arrangement + // answered with the last n of EACH partition - up to six. Asserting here that the two answers + // really do differ is what stops the test also passing on the old behavior. + val perTopicAnswer = perPartition.filter(_.nonEmpty).flatMap(_.takeRight(n)).map(_.getValue) + assert( + perTopicAnswer.size > n, + s"this only pins the GLOBAL contract while the per-partition answer is larger than $n: $perTopicAnswer" + ) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, spreadKind.scheme) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill(n.toString) + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, expected) + } + } diff --git a/e2e/src/test/scala/features/consumersession/CsStartFromOutcomesSpec.scala b/e2e/src/test/scala/features/consumersession/CsStartFromOutcomesSpec.scala new file mode 100644 index 000000000..a07461132 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsStartFromOutcomesSpec.scala @@ -0,0 +1,686 @@ +package features.consumersession + +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions + +import java.time.{Instant, LocalDateTime, ZoneId} + +/** Start-From OUTCOME coverage: for each mode, the exact set of messages the session ends up + * showing. + * + * Before this spec only Earliest and Latest had outcome coverage (`CsStartFromSpec` CS-2/CS-3); + * the five addressed modes - skip first n, latest n, message id, specific time, relative time - + * had none at all. Everything here runs on a PERSISTENT NON-PARTITIONED topic, which is the + * single-topic fast path in `handleStartFrom` (a message-id seek); `CsStartFromMatrixSpec` crosses + * the same two counting modes with the rest of the topic matrix. + * + * The two APPROXIMATE modes are both covered below - "Approximate position (% of data)" (CS-SF-11..14) and + * "Approximate position (% of time)" (CS-SF-16..19) - including CS-SF-17, where one topic is asked the same + * "50%" by both and answers differently. Being non-partitioned, these do not exercise what the two + * modes do across several physical topics - entry position resolving each independently, publish-time + * position pooling one min/max range over them: that is `CsApproximatePartitionedSpec` (CS-SF-20/21), + * with the pure arithmetic in `server/.../approximatePublishTimePositionTest`. + * + * Each test asserts an exact SET, never a row count: "skip the first 5" and "skip the first 50" + * both produce *a* count, and a virtualized table reaches a transient count for almost any bug. + * The last produced message doubles as the sentinel - it is in every expectation here, so the set + * cannot be satisfied until the whole stream has been observed. + * + * Batching is the axis this spec exists for. `PulsarAdmin.examineMessage`, which the two counting + * modes are built on, addresses ENTRIES; the Java producer packs many messages into one entry by + * default; and every fixture the suite had produced one message per entry, so entry positions and + * message positions always coincided and the two could never be told apart. See + * `harness.BatchingFixtureSpec` for the broker-level proof of both facts. + */ +class CsStartFromOutcomesSpec extends StartFromSupport: + + /** 12 messages -> 3 entries when batched at 4/entry, so an entry position and a message position + * can never be mistaken for one another. `m-12` is the sentinel. */ + private val payload = (1 to 12).map(i => f"m-$i%02d") + + private def openOn(fqn: String, t: String, ns: String, topic: String): ConsumerSessionPage = + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs + + // ------------------------------------------------------------------------------------------- + // Skip first n (NthMessageAfterEarliest) + // ------------------------------------------------------------------------------------------- + + test("CS-SF-1: Skip first n (unbatched) shows every message except the first n") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("5") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload.drop(5)) // m-06 .. m-12 + } + + // The regression this whole spec exists for. Until 2026-07-25 the mode was implemented as an + // entry-addressed `examineMessage("earliest", n + 1)` seek, and BATCH-2 pins why that cannot work: + // that call answers with the first message of the n-th ENTRY and, past the last entry, silently + // returns the last entry instead of failing. With these 12 messages in 3 entries, "skip 5" asked + // for position 6, got entry 3, and started at m-09 - it skipped EIGHT. Nothing caught it because + // every fixture in the suite produced one message per entry. + test("CS-SF-2: Skip first n (batched) skips n MESSAGES, not n entries") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceBatched(fqn, payload, MessagesPerBatch) // 12 messages -> 3 entries + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("5") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload.drop(5)) // m-06 .. m-12 + } + + // ------------------------------------------------------------------------------------------- + // Latest n (NthMessageBeforeLatest) + // ------------------------------------------------------------------------------------------- + + test("CS-SF-3: Latest n (unbatched) shows exactly the last n messages") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("5") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload.takeRight(5)) // m-08 .. m-12 + } + + // The counterpart regression, and the old failure shape differed from CS-SF-2's: counting entries + // BACK from the end does not clamp - `examineMessage("latest", n)` past the first entry THROWS + // (BATCH-2). The seek swallowed that and fell back to Earliest, so "show me the latest 5" showed + // all twelve. A row count would have called that "5 or more" and moved on; the exact set will not. + test("CS-SF-4: Latest n (batched) shows the last n MESSAGES, not the last n entries") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceBatched(fqn, payload, MessagesPerBatch) // 12 messages -> 3 entries + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("5") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload.takeRight(5)) // m-08 .. m-12 + } + + // ------------------------------------------------------------------------------------------- + // The n = 0 boundary - the two modes are deliberately asymmetric there + // ------------------------------------------------------------------------------------------- + + test("CS-SF-9: Skip first n with n = 0 skips nothing") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceBatched(fqn, payload, MessagesPerBatch) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("0") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload) // "skip zero" is Earliest + } + + test("CS-SF-10: Latest n with n = 0 shows nothing, and still streams what arrives after play") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceBatched(fqn, payload, MessagesPerBatch) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Latest n messages") + cs.startFromN.fill("0") + cs.play() + cs.assertState("running") + // "the last zero messages" is Latest: none of the history, and the empty state is the proof. + assertThat(cs.awaitingText).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(20000)) + assert(cs.columnValues("value").isEmpty, s"n = 0 loaded history: ${cs.columnValues("value")}") + + // An empty table alone would also be what a dead session looks like, so prove it is live. + val live = Seq("live-1", "live-2", "live-last") + fixtures.produceUnbatched(fqn, live) + assertLoadedExactlyWithCounter(cs, live) + } + + // The one join neither the jest tests nor the server tests can reach: the server really populating + // `start_from_progress`, travelling over gRPC, and arriving in the rendered panel. Both ends are + // covered in isolation; this is the wire between them. + // + // The panel is deliberately silent at or below 1,000,000 messages to skip, so the only way to see + // it is to ask for a skip larger than that - hence a number with no relation to the 12 messages + // actually on the topic. That is also what makes the test cheap: nothing has to be produced to + // reach the threshold, because `messagesToSkip` is what was ASKED for, not what exists. + test("CS-SF-15: a very large skip reports its progress from the server into the rendered panel") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("2000000") + cs.play() + cs.assertState("running") + + // The raw counts, not the prose. `skipped` must be strictly POSITIVE, and that is the whole + // point of the test: the server reports as soon as the discard claims its first message, so a + // positive count is the only thing that separates a real progress callback - listener -> gRPC + // -> panel - from a zero-state frame the UI could have rendered from its own initial state. + // Accepting `>= 0` (which this test used to do) accepted exactly that empty frame. + val skipped = harness.Eventually.eventually(timeoutMs = 30000, intervalMs = 400) { + assert(cs.startFromProgress.count() > 0, "the progress panel never appeared") + val reported = cs.startFromProgress.getAttribute("data-cs-skipped").toLong + assert(reported > 0, s"the panel is still at its zero state: skipped = $reported") + reported + } + val toSkip = cs.startFromProgress.getAttribute("data-cs-to-skip") + assert(toSkip == "2000000", s"to-skip was $toSkip") + // Only 12 messages exist, so a skip of two million can never claim more than those twelve, can + // never finish, and can never deliver anything. + assert(skipped <= payload.size, s"skipped $skipped of a ${payload.size}-message topic") + assert(cs.columnValues("value").isEmpty, s"a skip of 2,000,000 delivered messages: ${cs.columnValues("value")}") + } + + // ------------------------------------------------------------------------------------------- + // Approximate position (% of data) (ApproximateEntryPosition) + // + // Deliberately ENTRY-addressed (it has to resolve in constant time at any topic size), so the + // expectation is derived from the broker's real entry count rather than from the message count - + // the same percentage lands somewhere different on batched and unbatched data, and that is the + // documented contract, not a defect. Asserting it from the oracle is what keeps these two tests + // honest: hard-coding message positions would quietly encode the unbatched case as "the" answer. + // ------------------------------------------------------------------------------------------- + + /** The messages expected after positioning `percent` through the retained ENTRIES of `fqn`. + * + * `messages` MUST be the values actually produced to `fqn`, in order. It defaults to the shared + * `payload` because most cells use exactly that - but CS-SF-17 arranges its own set, and the + * default silently gave a confident answer about a topic it had never looked at. + * + * The guard compares against `messages` rather than `payload` for the same reason: with `payload` + * it passed by COINCIDENCE, both arrangements happening to hold 12 messages, so the mistake + * surfaced as a baffling set mismatch instead of naming the real problem. + */ + private def expectedFromEntryPercent( + fqn: String, + percent: Int, + messagesPerEntry: Int, + messages: Seq[String] = payload + ): Seq[String] = + val entries = fixtures.numberOfEntries(fqn) + assert(entries == messages.size / messagesPerEntry, s"unexpected arrangement: $entries entries for ${messages.size} messages") + // The broker is the oracle for WHICH messages, not just how many. The count check above is too + // weak on its own: CS-SF-17 arranged 12 messages of its own while `messages` defaulted to a + // DIFFERENT 12, so the count matched and the helper confidently described the wrong topic. + val onTopic = fixtures.readAllMessages(fqn).map(_.getValue) + assert(onTopic == messages, s"`messages` does not match $fqn: expected $messages, topic holds $onTopic") + val entriesLeftBehind = math.floor(percent / 100.0 * entries).toInt + messages.drop(entriesLeftBehind * messagesPerEntry) + + test("CS-SF-11: data position (unbatched) starts at the selected entry percentage") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) // 12 entries, 1 message each + + val expected = expectedFromEntryPercent(fqn, 50, messagesPerEntry = 1) // m-07 .. m-12 + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("50") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, expected) + } + + test("CS-SF-12: data position follows ENTRIES even when batch sizes are deliberately uneven") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val largeFirstEntry = payload.take(8) + val singletonEntry = payload.slice(8, 9) + val finalEntry = payload.drop(9) + fixtures.produceBatches(fqn, Seq(largeFirstEntry, singletonEntry, finalEntry)) + + val entries = fixtures.numberOfEntries(fqn) + assert(entries == 3, s"the uneven arrangement should contain exactly 3 entries, got $entries") + assert( + fixtures.readAllMessages(fqn).map(_.getValue) == payload, + s"the uneven batches did not preserve message order on $fqn" + ) + + // 50% of three entries leaves the large first entry behind, so all four messages in the two + // remaining entries are delivered. A message-based midpoint would leave only m-07..m-12 and + // therefore cannot satisfy this exact set. + val expected = singletonEntry ++ finalEntry + assert(expected != payload.drop(payload.size / 2), s"entry and message midpoints accidentally coincide: $expected") + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("50") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, expected) + } + + test("CS-SF-13: data position endpoints - 0% is Earliest, 100% is Latest") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("0") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload) + + // 100% is the other endpoint: past the last retained message, i.e. the live tail. + val (t2, ns2, topic2) = fixtures.freshTopicParts() + val fqn2 = s"persistent://$t2/$ns2/$topic2" + fixtures.produceUnbatched(fqn2, payload) + + cs.openForTopic(t2, ns2, topic2) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("100") + cs.play() + cs.assertState("running") + assertThat(cs.awaitingText).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(20000)) + assert(cs.columnValues("value").isEmpty, s"100% loaded history: ${cs.columnValues("value")}") + + val live = Seq("live-1", "live-2", "live-last") + fixtures.produceUnbatched(fqn2, live) + assertLoadedExactlyWithCounter(cs, live) + } + + test("CS-SF-14: data position rejects a percentage outside 0-100 without changing the session") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("150") // the model stores a fraction in [0, 1]; 150% has no meaning + assertThat(cs.startFromEntryFractionError).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(10000)) + + // Play with the rejected text STILL ON SCREEN. This is the whole test: the invalid value must + // never reach the session, so the run has to come out at the last VALID value - the 50% default. + // Correcting the field to 50 first (which this test used to do) only proved that validation + // recovers, and a session that had silently accepted 150% would have passed it unchanged. + // + // NOTE ON THE PRECONDITION: today the toolbar leaves Play ENABLED while the fraction is + // invalid, so "does not change the session" has to mean "runs at the last valid value". If the + // app is later changed to propagate validity and DISABLE Play (an open product question - an + // even stronger way to honour the same contract), this assertion is the one to rewrite: drop + // the play and assert the disabled button instead. It is asserted rather than assumed so that + // change surfaces here, with this sentence attached, instead of as a mystery timeout. + assert(cs.playButton.isEnabled, "Play is disabled while the fraction is invalid - see the note above") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, expectedFromEntryPercent(fqn, 50, messagesPerEntry = 1)) + + // ... and the field is not simply inert: a VALID change on the same control does move the + // session. Without this leg the assertion above would also hold for a mode that ignored the + // fraction entirely and always started half way in. + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("0") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload) + } + + // ------------------------------------------------------------------------------------------- + // Message with specific ID (MessageId) + // ------------------------------------------------------------------------------------------- + + /** The broker's own message ids are the oracle - the UI takes the serialized id as hex. */ + private def messageIdHexOf(fqn: String, value: String): String = + val msgs = fixtures.readAllMessages(fqn) + val m = msgs.find(_.getValue == value).getOrElse(fail(s"$value is not on $fqn: ${msgs.map(_.getValue)}")) + fixtures.messageIdHex(m.getMessageId) + + test("CS-SF-5: Message with specific ID starts AT that message and shows the rest") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Message with specific ID") + cs.startFromMessageId.fill(messageIdHexOf(fqn, "m-04")) + cs.play() + cs.assertState("running") + // Inclusive: the addressed message is where the session starts, so it is shown too. + assertLoadedExactlyWithCounter(cs, payload.drop(3)) // m-04 .. m-12 + } + + test("CS-SF-5b: Message with specific ID resolves a message inside a BATCH") { + // A batched message id carries a batch index; the id-addressed path has to honour it and start + // mid-entry rather than at the entry's first message. + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceBatched(fqn, payload, MessagesPerBatch) // m-05..m-08 share the 2nd entry + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Message with specific ID") + cs.startFromMessageId.fill(messageIdHexOf(fqn, "m-06")) // index 1 within its entry + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload.drop(5)) // m-06 .. m-12 + } + + // ------------------------------------------------------------------------------------------- + // Specific time (DateTime) and Relative time ago (RelativeDateTime) + // ------------------------------------------------------------------------------------------- + + private val older = (1 to 4).map(i => f"old-$i%02d") + private val newer = (1 to 4).map(i => f"new-$i%02d") // "new-04" is the sentinel + + /** Produce `older`, leave a real gap on the wall clock, produce `newer`, and return the broker's + * publish times keyed by value - the oracle both time-addressed modes are asserted against. */ + private def produceStraddling(fqn: String, gapMs: Long): Map[String, Long] = + fixtures.produceUnbatched(fqn, older) + awaitClockGap(System.currentTimeMillis(), gapMs) + fixtures.produceUnbatched(fqn, newer) + val times = fixtures.readAllMessages(fqn).map(m => m.getValue -> m.getPublishTime).toMap + assert( + (older ++ newer).forall(times.contains), + s"the arrangement did not land on the broker: ${times.keys.toList.sorted}" + ) + assert( + older.map(times).max < newer.map(times).min, + s"the two groups did not straddle the gap: old=${older.map(times)} new=${newer.map(times)}" + ) + times + + test("CS-SF-6: Specific time shows only messages published at or after that instant") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + // 2s: the picker is second-granular, so the two groups have to sit in different seconds. + val times = produceStraddling(fqn, gapMs = 2000) + // The start of the second the first new message landed in: strictly after every old message + // (the gap guarantees it) and at or before every new one. + val cutoffMs = newer.map(times).min / 1000 * 1000 + assert(cutoffMs > older.map(times).max, s"cutoff $cutoffMs does not separate the groups: $times") + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Specific time") + cs.setStartFromDateTime(LocalDateTime.ofInstant(Instant.ofEpochMilli(cutoffMs), ZoneId.systemDefault)) + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, newer) + } + + test("CS-SF-7: Relative time ago with a window that covers everything shows the whole topic") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + fixtures.produceUnbatched(fqn, payload) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Relative time ago") + cs.setStartFromRelative(1, "hour") // everything here was published seconds ago + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, payload) + } + + test("CS-SF-8: Relative time ago with a narrow window excludes the older messages") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + // The window is resolved server-side at seek time, so the arrangement has to leave room for the + // round trip on both sides: with a 20s gap and a 10s window the cutoff lands ~10s after the old + // group and ~10s before the new one, wherever in that span the seek actually happens. + produceStraddling(fqn, gapMs = 20000) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Relative time ago") + cs.setStartFromRelative(10, "second") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, newer) + } + + // ------------------------------------------------------------------------------------------- + // Approximate position (% of time) (ApproximatePublishTimePosition) + // + // The sibling of data position above, and the reason both exist: "about half way in" is two + // different questions. Where entry position counts stored entries, this one interpolates between + // the first and last PUBLISH TIMES and seeks to the instant that falls out - so the same 50% lands + // somewhere else entirely on a topic whose messages did not arrive evenly. CS-SF-17 is that + // contrast, asserted on one topic with both modes. + // + // Every expectation here is derived from the broker's own publish times rather than written down, + // for the same reason the entry-addressed ones are: hard-coding a set would encode one particular + // arrangement as "the" answer. The arrangements leave SECONDS of margin between the cutoff and the + // nearest message, so a produce round trip cannot move a message across it. + // ------------------------------------------------------------------------------------------- + + /** Publish `groups` in order with `gapMs` of real wall clock between the start of each, and answer + * with the broker's publish time for every value produced. + * + * This is ARRANGEMENT: the only way to give a topic a time range is to publish across one. The + * gaps are also the test's safety margin - each group is asserted to sit strictly after the one + * before it, so a cutoff computed to fall inside a gap cannot accidentally land on a message. */ + private def produceSpacedGroups(fqn: String, groups: Seq[Seq[String]], gapMs: Long): Map[String, Long] = + var groupStartedAt = System.currentTimeMillis() + groups.zipWithIndex.foreach { (group, index) => + if index > 0 then + awaitClockGap(groupStartedAt, gapMs) + groupStartedAt = System.currentTimeMillis() + fixtures.produceUnbatched(fqn, group) + } + + val times = fixtures.readAllMessages(fqn).map(m => m.getValue -> m.getPublishTime).toMap + assert(groups.flatten.forall(times.contains), s"the arrangement did not land on the broker: ${times.keys.toList.sorted}") + groups.map(_.map(times)).sliding(2).foreach { + case Seq(before, after) => assert(before.max < after.min, s"the groups do not straddle their gap: $before then $after") + case _ => () + } + times + + /** The exact messages `percent` of the topic's TIME RANGE must deliver, computed from the broker's + * publish times: the cutoff is `first + floor(percent/100 * (last - first))` and everything + * published at or after it is shown. */ + private def expectedFromPublishTimePercent(times: Map[String, Long], percent: Int): Seq[String] = + val first = times.values.min + val last = times.values.max + val cutoffMs = first + math.floor(percent / 100.0 * (last - first)).toLong + times.filter((_, publishedAt) => publishedAt >= cutoffMs).keys.toSeq.sorted + + test("CS-SF-16: publish-time position lands proportionally through ELAPSED TIME") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + // Three groups six seconds apart, so the range is ~12s and both fractions below fall in the + // middle of a gap - three seconds from the nearest message on either side. + val early = Seq("t-01", "t-02") + val middle = Seq("t-03", "t-04") + val late = Seq("t-05", "t-06") + val times = produceSpacedGroups(fqn, Seq(early, middle, late), gapMs = 6000) + + // 25% of a 12s range is ~3s in: past the early group, well short of the middle one. + val quarter = expectedFromPublishTimePercent(times, 25) + assert(quarter.toSet == (middle ++ late).toSet, s"25% of the time range resolved to $quarter") + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("25") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, quarter) + + // 75% is ~9s in: past the middle group, short of the late one. A mode that ignored the fraction + // and always seeked to one end would satisfy one of these two assertions but never both. + val threeQuarters = expectedFromPublishTimePercent(times, 75) + assert(threeQuarters.toSet == late.toSet, s"75% of the time range resolved to $threeQuarters") + + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("75") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, threeQuarters) + } + + test("CS-SF-17: at the same 50%, publish-time and entry positions land in different places") { + // THE motivating case, and the whole reason one mode became two. Two messages long ago and then + // a burst of ten: half the TIME is back in the empty stretch, while half the ENTRIES is inside + // the burst. One control could not have meant both. + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val old = Seq("old-1", "old-2") + val burst = (1 to 10).map(i => f"b-$i%02d") + val times = produceSpacedGroups(fqn, Seq(old, burst), gapMs = 10000) + + // Half of a ~10s range is ~5s in - five seconds after the old pair and five before the burst. + val byPublishTime = expectedFromPublishTimePercent(times, 50) + assert(byPublishTime.toSet == burst.toSet, s"50% of the publish-time range resolved to $byPublishTime") + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("50") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, byPublishTime) + + // The same 50%, counted over the 12 entries instead: six are left behind, so it starts on the + // seventh message overall - the fifth of the burst. + val byEntry = expectedFromEntryPercent(fqn, 50, messagesPerEntry = 1, messages = old ++ burst) + assert(byEntry.toSet == burst.drop(4).toSet, s"50% of the entries resolved to $byEntry") + assert( + byEntry.size < byPublishTime.size, + s"the two modes must not coincide here: entries=$byEntry publishTime=$byPublishTime" + ) + + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Approximate position (% of data)") + cs.setStartFromEntryPercent("50") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, byEntry) + } + + test("CS-SF-18: publish-time endpoints - 0% is everything, 100% includes every final timestamp tie") { + // Deliberately unlike entry position, whose 100% means "past the end" and shows nothing: the + // time range ends AT the latest publish timestamp. A timestamp seek cannot split messages that + // share that timestamp, so the final entry is deliberately a multi-message batch and every + // member must be shown. + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val bulk = (1 to 6).map(i => f"e-$i%02d") + val finalBatch = Seq("e-last-1", "e-last-2", "e-last-3") + fixtures.produceBatches(fqn, Seq(bulk)) + awaitClockGap(System.currentTimeMillis(), 2000) + fixtures.produceBatches(fqn, Seq(finalBatch)) + val retained = fixtures.readAllMessages(fqn) + val times = retained.map(m => m.getValue -> m.getPublishTime).toMap + val finalTimes = finalBatch.map(times) + assert(finalTimes.distinct.size == 1, s"the final batch does not share one publish timestamp: $finalTimes") + assert( + bulk.map(times).max < finalTimes.head, + s"the final batch is not strictly newer than the bulk: bulk=${bulk.map(times)}, final=$finalTimes" + ) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("0") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, bulk ++ finalBatch) + + val atTheEnd = expectedFromPublishTimePercent(times, 100) + assert(atTheEnd.toSet == finalBatch.toSet, s"100% of the publish-time range resolved to $atTheEnd") + + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("100") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, finalBatch) + } + + test("CS-SF-19: publish-time position rejects a percentage outside 0-100 without changing the session") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val early = Seq("t-01", "t-02") + val late = Seq("t-03", "t-04") + val times = produceSpacedGroups(fqn, Seq(early, late), gapMs = 6000) + + val cs = openOn(fqn, t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("150") // the model stores a fraction in [0, 1]; 150% has no meaning + // The publish-time mode's own error, not entry position's: both render the same control, so this is + // also the check that they are separately addressable in the real DOM. + assertThat(cs.startFromPublishTimeFractionError).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(10000)) + assertThat(cs.startFromEntryFraction).hasCount(0) + + // Play with the rejected text STILL ON SCREEN - the same claim CS-SF-14 makes for the data + // mode, and for the same reason: correcting the field first proves validation recovery, not + // that an invalid value cannot reach the session. + assert(cs.playButton.isEnabled, "Play is disabled while the fraction is invalid - see the note in CS-SF-14") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, expectedFromPublishTimePercent(times, 50)) + + // ... and the control is not inert: a valid change does move the session. + val atTheEnd = expectedFromPublishTimePercent(times, 100) + assert( + atTheEnd.toSet != expectedFromPublishTimePercent(times, 50).toSet, + s"100% and 50% resolve to the same set here, so the second leg proves nothing: $atTheEnd" + ) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("100") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, atTheEnd) + } + + test("CS-SF-22: publish-time position resolves ONE cutoff across separate logical topics") { + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + + // Re-aimed 2026-08-09. This cell asserted the PRE-P2.15 contract (a range per logical topic) + // and had been failing since the session-wide cutoff landed - the implementation, its server + // pins ("P2.15: SEPARATE topics share ONE cutoff") and docs/consume/modes-comparison.md all + // specify one instant for the whole selection. The old per-topic answer is kept below as the + // counterexample this cell now exists to EXCLUDE, so a regression back to it fails here. + // + // Give A a short early range and B a short late range, with a larger empty gap between them. + // At 50%, one range per topic would keep each topic's newer message. The session-wide range + // instead lands in the inter-topic gap: it keeps both B messages and drops all of A. + fixtures.produceUnbatched(fqnA, Seq("a-old")) + awaitClockGap(System.currentTimeMillis(), 1000) + fixtures.produceUnbatched(fqnA, Seq("a-new")) + awaitClockGap(System.currentTimeMillis(), 3000) + fixtures.produceUnbatched(fqnB, Seq("b-old")) + awaitClockGap(System.currentTimeMillis(), 1000) + fixtures.produceUnbatched(fqnB, Seq("b-new")) + + def publishTimes(fqn: String): Map[String, Long] = + fixtures.readAllMessages(fqn).map(message => message.getValue -> message.getPublishTime).toMap + + val timesA = publishTimes(fqnA) + val timesB = publishTimes(fqnB) + // What the session must do: one range across every selected topic, one cutoff instant. + val sessionWide = expectedFromPublishTimePercent(timesA ++ timesB, 50) + assert(sessionWide.toSet == Set("b-old", "b-new"), s"the session-wide range resolved to $sessionWide") + + // What it must NOT do: interpolate inside each topic's own history. + val perTopic = expectedFromPublishTimePercent(timesA, 50) ++ expectedFromPublishTimePercent(timesB, 50) + assert(perTopic.toSet == Set("a-new", "b-new"), s"the per-topic counterexample resolved to $perTopic") + assert(sessionWide.toSet != perTopic.toSet, s"session-wide and per-topic answers unexpectedly coincide: $sessionWide") + + val cs = openOn(fqnA, tA, nsA, topicA) + cs.setTargetTopicsSpecific(Seq(fqnA, fqnB)) + cs.setStartFrom("Approximate position (% of time)") + cs.setStartFromPublishTimePercent("50") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, sessionWide) + } diff --git a/e2e/src/test/scala/features/consumersession/CsStartFromSpec.scala b/e2e/src/test/scala/features/consumersession/CsStartFromSpec.scala index 7a72baf21..4c005f884 100644 --- a/e2e/src/test/scala/features/consumersession/CsStartFromSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsStartFromSpec.scala @@ -60,7 +60,9 @@ class CsStartFromSpec extends DekafSuite: // When the sentinel arrives every post-play message is in, so assert the EXACT loaded set: only // the new ones, none of the old - a bare hasCount(3) could pass transiently even if old loaded too. val values = eventually() { - val vs = cs.columnValues("value") + // Whole-list read: a leaked old-* row is older than every new-* one, i.e. the first row the + // auto-scrolled virtualized table pushes out of the mounted range. + val vs = cs.allColumnValues("value") assert(vs.contains("new-last"), s"sentinel not loaded yet: $vs") vs } diff --git a/e2e/src/test/scala/features/consumersession/CsTableSpec.scala b/e2e/src/test/scala/features/consumersession/CsTableSpec.scala index 992c63c95..05e2550c5 100644 --- a/e2e/src/test/scala/features/consumersession/CsTableSpec.scala +++ b/e2e/src/test/scala/features/consumersession/CsTableSpec.scala @@ -1,6 +1,7 @@ package features.consumersession import harness.DekafSuite +import scala.jdk.CollectionConverters.* import org.apache.pulsar.client.api.Schema class CsTableSpec extends DekafSuite: @@ -111,3 +112,36 @@ class CsTableSpec extends DekafSuite: assert(math.abs(domAfterReload - domAfterResize) <= 3, s"restored column rendered at ${domAfterReload}px, expected ~${domAfterResize}px (persisted width not re-applied)") } + + test("CS-TBL-REORDER: a message column dragged onto another lands before it and is REMEMBERED") { + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 3) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(3) + + def headerKeys(): List[String] = + page.locator("[data-testid^='cs-th-']").all().asScala.toList + .map(_.getAttribute("data-testid").stripPrefix("cs-th-")) + + // `.all()` does not wait; the header commits a frame after the first rows do. + page.getByTestId("cs-th-topic").waitFor() + val before = headerKeys() + assert(before.indexOf("topic") > before.indexOf("value"), s"unexpected default order: $before") + + // Drag TOPIC onto VALUE: topic must land immediately before value, rows following the header. + page.getByTestId("cs-th-topic").dragTo(page.getByTestId("cs-th-value")) + val after = headerKeys() + assert( + after.indexOf("topic") == after.indexOf("value") - 1, + s"dragged column should sit immediately before its target, got $after" + ) + // The sticky pair stays put in front. + assert(after.take(2) == List("index", "publishTime"), s"sticky pair must stay first, got ${after.take(2)}") + + val stored = page.evaluate("() => localStorage.getItem('table:consumer-session-messages:column-order') || ''").toString + assert(stored.contains("topic"), s"expected a persisted message-column order, got '$stored'") + } diff --git a/e2e/src/test/scala/features/consumersession/CsTopicKindsSpec.scala b/e2e/src/test/scala/features/consumersession/CsTopicKindsSpec.scala new file mode 100644 index 000000000..3466d00ac --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsTopicKindsSpec.scala @@ -0,0 +1,227 @@ +package features.consumersession + +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat +import com.microsoft.playwright.assertions.LocatorAssertions + +/** The Consumer Session across the whole topic matrix - {persistent, non-persistent} x + * {partitioned, non-partitioned}. Every test is generated from `TopicKind.all`, so all four + * quadrants run and a failure names its own quadrant. + * + * Why the matrix matters: `consumer/session_runner/handleStartFrom.scala` behaves differently per + * quadrant - a partitioned topic is expanded into its partitions and each gets its own consumer and + * its own seek, while a non-persistent one has no history to seek into at all - and the suite's + * only start-from coverage lived on persistent non-partitioned topics; the other three quadrants + * had none. + * + * Three broker facts shape the assertions here: + * - a NON-PERSISTENT topic retains nothing. Whatever is published while no consumer is attached + * is dropped forever, so "pre-produce, then Start From = Earliest" is meaningless there; only + * produce-AFTER-play is assertable. `kind.retains` gates which shape a quadrant gets. + * - producing to a PARTITIONED topic round-robins, and the session runs one consumer per + * partition, so the merged view has no total order. Every assertion below is on a SET. + * - a partitioned topic's partitions are what the session actually subscribes to, so + * "the session is attached" has to be checked per partition (see `awaitConsumersFlowing`). + * + * The arrangement and assertion helpers come from `StartFromSupport`, which this spec used to + * carry near-identical private copies of. The shared ones are stronger in the way that matters + * here: `assertLoadedExactlyWithCounter` also pins the toolbar's `cs-loaded` counter and requires + * it to STAY there, which the local copy could not - it compared the currently rendered rows of a + * VIRTUALIZED table, so a message loaded off-screen, or one that arrived just after the first + * matching poll, passed. + */ +class CsTopicKindsSpec extends StartFromSupport: + private def vis = new LocatorAssertions.IsVisibleOptions().setTimeout(30000) + + private def produce(fqn: String, values: Seq[String]): Unit = fixtures.produceUnbatched(fqn, values) + + /** Messages published BEFORE the session exists. Every test that arranges them also asserts none + * of them came back, so the prefix is what a stray one is recognized by. */ + private val OldPrefix = "old-" + + /** What the toolbar counter already stood at before a test's payload was produced. + * + * Zero on the retaining quadrants - nothing is loaded there until the payload is. The + * non-persistent ones first have to prove the stream is live by getting a handshake row rendered, + * and handshakes count towards `cs-loaded` too, so their baseline is whatever the counter settled + * at once the handshakes stopped arriving. */ + private def streamingBaseline(cs: ConsumerSessionPage, fqn: String, kind: fixtures.TopicKind): Int = + if kind.retains then 0 + else + awaitSessionStreaming(cs, fqn) + settledLoaded(cs) + + fixtures.TopicKind.all.foreach { kind => + + test(s"CS-TK-1 [${kind.label}]: the session mounts on the topic and starts") { + val (t, ns, topic, _) = fixtures.freshTopicPartsOfKind(kind) + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + + // The configuration view comes first - the message table only mounts once a message exists. + assertThat(cs.startFromSelect).isVisible(vis) + assertThat(cs.playButton).isVisible(vis) + assert(cs.state == "new", s"unexpected initial session state: ${cs.state}") + + cs.play() + // 'running' is set only when CreateConsumer returns OK, which makes it a real server-side + // gate: the target resolved this topic kind, the consumers subscribed, and the start-from + // seek did not throw. A failure here is a broken quadrant, not a slow one. + cs.assertState("running") + assertThat(cs.awaitingText).isVisible(vis) + } + + test(s"CS-TK-2 [${kind.label}]: Start From = Latest loads exactly the messages produced after play") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + // Sentinels published BEFORE the session exists. Without them this test could not tell + // "Latest" from "Earliest" at all: on a topic that was EMPTY at play time both modes deliver + // exactly the rows produced afterwards, so an implementation that ignored the selected mode + // passed. With them, the retaining quadrants have history on disk that only Latest keeps out - + // and it is the same history CS-TK-3 proves Earliest does deliver, so the two tests now + // disagree about the same topic contents rather than agreeing by construction. + // (On a non-persistent topic the broker drops these anyway - CS-TK-5 is where that is the + // point; here they are simply harmless.) + val old = (1 to 3).map(i => s"${OldPrefix}tk-$i") + produce(fqn, old) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Latest message") + cs.play() + cs.assertState("running") + awaitConsumersFlowing(fqn, kind) + val base = streamingBaseline(cs, fqn, kind) + + val payload = Seq("tk-1", "tk-2", "tk-3", "tk-last") // "tk-last" is produced last + produce(fqn, payload) + assertLoadedExactlyWithCounter(cs, payload, base) + // The whole virtualized list, not the mounted viewport: a leaked OLD row is the oldest on + // screen and is exactly what the auto-scrolled table pushes out of the mounted range first. + val rendered = cs.allColumnValues("value") + assert( + !rendered.exists(_.startsWith(OldPrefix)), + s"Start From = Latest delivered a message published before the session started: $rendered" + ) + cs.waitHeader() // the real message table rendered, not just the empty state + } + + if kind.retains then + test(s"CS-TK-3 [${kind.label}]: Start From = Earliest loads the whole pre-produced set") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + val pre = Seq("pre-1", "pre-2", "pre-3", "pre-last") + produce(fqn, pre) // round-robins across partitions when the kind is partitioned + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Earliest message") + cs.play() + cs.assertState("running") + assertLoadedExactlyWithCounter(cs, pre) + } + + test(s"CS-TK-4 [${kind.label}]: Start From = Skip first n skips exactly the first n messages") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + // The partitioned kind is fed through ONE partition deliberately: "the first n messages" is + // only well defined on a single ordered log, and a partitioned topic has no total order. + // This still drives the partitioned branch end to end - the topic is expanded into its + // partitions and every partition gets its own consumer and its own seek - it just makes the + // expectation unambiguous. + val log = if kind.isPartitioned then s"$fqn-partition-0" else fqn + val all = (1 to 6).map(i => s"m-$i") + produce(log, all) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("2") + cs.play() + cs.assertState("running") + // The control is labelled "Skip first n messages", so n = 2 out of 6 must leave m-3..m-6. + assertLoadedExactlyWithCounter(cs, all.drop(2)) + } + + if kind.isPartitioned then + // UNTAGGED on 2026-07-25 - green in the normal lane. This was `KnownBug` twice over: first + // for a real defect (skipping across partitions dropped nothing at all), then because the + // expectation below - `all.drop(2)` - is a statement about a GLOBAL order that the + // then-current contract did not promise. Skip-N was resolved per partition, so it dropped + // two arbitrary messages and runs disagreed about which (m-1 + m-6 in one, m-4 + m-5 in + // another). + // + // Skip-N is now GLOBAL: it drops the n oldest messages of the merged stream by publish time, + // whichever partitions they came from. The payload below is produced through the parent one + // blocking send at a time, so publish order is m-1 .. m-6 and the two globally-oldest are + // m-1 and m-2 - exactly what `all.drop(2)` says. + // + // The SET is the assertion, not the count: dropping any other two also leaves four. + // + // The one residual assumption is that six blocking sends land in six distinct milliseconds - + // if two shared one, the global order between their partitions would be decided by the + // merge tie-break rather than by produce order. A blocking send costs a broker round + // trip, and this ran green six times running, so it is stated rather than defended here. + // `CsStartFromMatrixSpec` CS-SFM-3 covers the same contract without the assumption: it spaces + // its publishes and derives the expectation from the broker's own publish times. + test(s"CS-TK-6 [${kind.label}]: Skip first n works when messages are spread across ALL partitions") { + // CS-TK-4 deliberately funnels every message through partition-0 to make "first n" + // unambiguous - which also means it never exercises the real multi-partition path. Here + // the default RoundRobinPartition router spreads 6 messages over 3 partitions (2 each), + // which is what ordinary produce traffic looks like. + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + val all = (1 to 6).map(i => s"m-$i") + produce(fqn, all) // parent topic -> round-robin across the 3 partitions + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + cs.setStartFrom("Skip first n messages") + cs.startFromN.fill("2") + cs.play() + cs.assertState("running") + // Skipping 2 of 6 must leave 4 messages, whichever partitions they came from. + assertLoadedExactlyWithCounter(cs, all.drop(2)) + } + else + test(s"CS-TK-5 [${kind.label}]: nothing is retained - every history mode is refused, the live session still streams") { + val (t, ns, topic, fqn) = fixtures.freshTopicPartsOfKind(kind) + val dropped = (1 to 4).map(i => s"${OldPrefix}$i") + produce(fqn, dropped) // no consumer attached -> the broker drops these forever + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic, kind.scheme) + // "Nothing is retained" is now enforced in the selector, not just observable afterwards: + // every history-based mode is disabled here, INCLUDING "Earliest message" - which used to be + // the misleading one, since with nothing retained it quietly behaved as "from now". + // Asserting the disabled set is what this test's title has always claimed to prove. + assert( + cs.disabledStartFromLabels.toSet == Set( + "Earliest message", + "Message with specific ID", + "Specific time", + "Relative time ago", + "Skip first n messages", + "Latest n messages", + // Both approximate modes need a history to be a proportion OF: one asks the broker for + // the topic's entry count, the other for its first and last publish times, and + // examineMessage answers neither on a non-persistent topic. + "Approximate position (% of data)", + "Approximate position (% of time)" + ), + s"disabled modes on a non-persistent topic were: ${cs.disabledStartFromLabels}" + ) + assertThat(cs.startFromNonPersistentNote).isVisible(vis) + + cs.setStartFrom("Latest message") // the only position a non-persistent topic has + cs.play() + cs.assertState("running") + awaitConsumersFlowing(fqn, kind) + // Prove the session is really streaming BEFORE asserting the negative: an empty table only + // means "nothing was retained" once we know a message would have shown up. + val base = streamingBaseline(cs, fqn, kind) + + val live = Seq("new-1", "new-2", "new-last") + produce(fqn, live) + assertLoadedExactlyWithCounter(cs, live, base) + // Full-list read for the same reason as CS-TK-2: a leaked OLD row scrolls out of the + // mounted viewport first. + val rendered = cs.allColumnValues("value") + assert(!rendered.exists(_.startsWith(OldPrefix)), s"a non-persistent topic retained a pre-produced message: $rendered") + } + } diff --git a/e2e/src/test/scala/features/consumersession/CsTopicPositionsSpec.scala b/e2e/src/test/scala/features/consumersession/CsTopicPositionsSpec.scala new file mode 100644 index 000000000..d64d29bb6 --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsTopicPositionsSpec.scala @@ -0,0 +1,293 @@ +package features.consumersession + +import harness.DekafSuite +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +/** The Topic Positions tab: the per-topic debug view in the Tools panel. + * + * WHAT ONLY AN END-TO-END TEST CAN SEE HERE. The server suite pins the arithmetic and the cursor + * bookkeeping against plain values; the jest suite pins the row model against a hand-built + * protobuf. Neither can tell whether the numbers a REAL broker produces survive the whole path - + * whether the entry counts add up to what was actually published, and whether a figure the server + * genuinely does not know arrives as a blank cell rather than as a confident 0%. + * + * POLLING IS GATED BY THE TAB ITSELF: opening it is the request, and only the tab on screen pays + * the per-partition broker cost (the jest suite pins the hidden-tab half). The session records + * its read position unconditionally either way, so the tab shows the full truth whenever it is + * opened - there is nothing to remember to turn on first. + */ +class CsTopicPositionsSpec extends DekafSuite: + + /** Count the real grpc-web calls without delaying or replacing them. The script is installed + * before navigation because a page evaluation would be discarded by the navigation itself. */ + private def installTopicPositionsRequestCounter(): Unit = + context.addInitScript( + """(() => { + | const marker = 'ConsumerService/GetTopicPositions'; + | const openOrig = XMLHttpRequest.prototype.open; + | const sendOrig = XMLHttpRequest.prototype.send; + | XMLHttpRequest.prototype.open = function (method, url) { + | this.__isTopicPositionsRequest = String(url).indexOf(marker) !== -1; + | return openOrig.apply(this, arguments); + | }; + | XMLHttpRequest.prototype.send = function () { + | if (this.__isTopicPositionsRequest) { + | window.__topicPositionsRequestCount = (window.__topicPositionsRequestCount || 0) + 1; + | } + | return sendOrig.apply(this, arguments); + | }; + |})();""".stripMargin + ) + + private def topicPositionsRequestCount: Int = + page.evaluate("() => window.__topicPositionsRequestCount || 0") match + case n: Number => n.intValue() + case value => throw new AssertionError(s"Topic Positions request counter is not numeric: $value") + + test("CS-TP-1: before the session is started the tab says so - no table, no raw server error") { + // The tab is reachable the moment the page loads, so opening it before Play is the ORDINARY + // case. Nothing exists to ask about yet: no poll is armed (the jest suite pins the zero-RPC + // half), so no FAILED_PRECONDITION can reach the screen and the generated session name cannot + // leak at the one moment the user has done nothing wrong. + val (t, ns, topic) = fixtures.freshTopicParts() + fixtures.produceStrings(s"persistent://$t/$ns/$topic", 10) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.openTools() + + val tools = ToolsPanel(page) + assertThat(tools.topicPositionsTab).isVisible() + tools.topicPositionsTab.click() + + assertThat(tools.topicPositionsNotStarted).isVisible() + assertThat(tools.topicPositionsTable).not().isVisible() + assertThat(tools.topicPositionsError).not().isVisible() + // The internal session name must not reach the screen. + assertThat(page.getByText("__dekaf_")).not().isVisible() + } + + test("CS-TP-2: first and last consumed are tracked before Topic Positions is opened") { + // Read the broker's real ids before starting Dekaf. They are the end-to-end oracle: a unit + // fixture cannot prove the id survived Pulsar -> listener bookkeeping -> protobuf -> browser. + val (t, ns, topic) = fixtures.freshTopicParts() + val topicFqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(topicFqn, 5) + val retained = fixtures.readAllMessages(topicFqn) + val expectedFirst = fixtures.messageIdHex(retained.head.getMessageId) + val expectedLast = fixtures.messageIdHex(retained.last.getMessageId) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.closeTools() + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(5) + // More tools is deliberately closed before consumption. Listener-side history must not depend on the + // expensive broker-inspection tab having mounted or polled while consumption happened. + + cs.openTools() + val tools = ToolsPanel(page) + assertThat(tools.topicPositionsCell(topicFqn, TopicPositionsColumn.FirstConsumedId)).hasText(expectedFirst) + assertThat(tools.topicPositionsCell(topicFqn, TopicPositionsColumn.LastConsumedId)).hasText(expectedLast) + + val cells = tools.topicPositionsCells(topicFqn) + assert(cells(TopicPositionsColumn.FirstConsumedPublished).trim != "-", + s"first consumed published time should be known, got '${cells(TopicPositionsColumn.FirstConsumedPublished)}'") + assert(cells(TopicPositionsColumn.LastConsumedPublished).trim != "-", + s"last consumed published time should be known, got '${cells(TopicPositionsColumn.LastConsumedPublished)}'") + } + + test("CS-TP-3: a running session reports each partition, and the entry counts ADD UP") { + // The arithmetic check that only a real broker can settle: three partitions, 30 messages, and + // the per-partition entry totals must sum to exactly what was published. + val (t, ns, topic, topicFqn) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.PersistentPartitioned) + fixtures.produceStrings(topicFqn, 30) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.closeTools() + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(30) + cs.openTools() + + val tools = ToolsPanel(page) + tools.topicPositionsTab.click() + + val partitions = (0 until 3).map(i => s"$topicFqn-partition-$i") + partitions.foreach(p => assertThat(tools.topicPositionsRow(p)).isVisible()) + + // WAIT for a settled frame first: the table refreshes once a second, and the frame on screen + // when the tab opens can legally predate the last messages read - parsing its '-' cells would + // test the refresh cycle's phase, not the arithmetic. + assertThat(tools.topicPositionsRow("All topics")).containsText("30 / 30") + + // "ordinal / total" in the RETAINED ENTRY POSITION column; the denominators are what must + // reconcile with the number published. + val totals = partitions.map { p => + val cells = tools.topicPositionsCells(p) + assert(cells(TopicPositionsColumn.FirstConsumedId).trim != "-", + s"$p should report its first consumed id after reading with the tab closed") + assert(cells(TopicPositionsColumn.FirstConsumedPublished).trim != "-", + s"$p should report its first consumed publish time after reading with the tab closed") + assert(cells(TopicPositionsColumn.LastConsumedId).trim != "-", + s"$p should report its last consumed id after reading with the tab closed") + assert(cells(TopicPositionsColumn.LastConsumedPublished).trim != "-", + s"$p should report its last consumed publish time after reading with the tab closed") + cells(TopicPositionsColumn.RetainedEntryPosition).split('/').last.trim.replace(",", "").toInt + } + assert( + totals.sum == 30, + s"per-partition entry totals ${totals.mkString(" + ")} = ${totals.sum}, but 30 messages were published to $topicFqn" + ) + + // The ALL TOPICS row sums what the per-partition rows show - the whole session, one line. + val aggregate = tools.topicPositionsCells("All topics") + assert(aggregate(TopicPositionsColumn.RetainedEntryPosition).replace(",", "").contains("30 / 30"), + s"aggregate retained entry position should be 30 / 30, got '${aggregate(TopicPositionsColumn.RetainedEntryPosition)}'") + assert(aggregate(TopicPositionsColumn.EntriesAfterCursor).trim == "0", + s"aggregate entries after cursor should be 0, got '${aggregate(TopicPositionsColumn.EntriesAfterCursor)}'") + // Message ids from independent partition logs cannot be combined honestly, but their publish + // time envelope can: earliest first-consumed and latest last-consumed. + assert(aggregate(TopicPositionsColumn.FirstConsumedId).trim == "-", + s"aggregate first consumed id should be blank, got '${aggregate(TopicPositionsColumn.FirstConsumedId)}'") + assert(aggregate(TopicPositionsColumn.LastConsumedId).trim == "-", + s"aggregate last consumed id should be blank, got '${aggregate(TopicPositionsColumn.LastConsumedId)}'") + assert(aggregate(TopicPositionsColumn.FirstConsumedPublished).trim != "-", + s"aggregate first consumed published time should be known") + assert(aggregate(TopicPositionsColumn.LastConsumedPublished).trim != "-", + s"aggregate last consumed published time should be known") + } + + test("CS-TP-4: a figure the server does not know is BLANK, not 0%") { + // The distinction the whole view rests on. Starting at LATEST on a topic nobody is producing to + // means the session has read nothing, so it has no position - while the topic itself still has + // a first and a last message. Rendering the unknown as 0% would claim the session sits at the + // BEGINNING of the topic when it is in fact parked at the end. + val (t, ns, topic) = fixtures.freshTopicParts() + val topicFqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(topicFqn, 10) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Latest message") + cs.play() + cs.openTools() + + val tools = ToolsPanel(page) + tools.topicPositionsTab.click() + assertThat(tools.topicPositionsRow(topicFqn)).isVisible() + + val cells = tools.topicPositionsCells(topicFqn) + // The topic's own endpoints ARE known. + assert(cells(TopicPositionsColumn.OldestRetainedId).trim.nonEmpty && cells(TopicPositionsColumn.OldestRetainedId).trim != "-", + s"oldest retained message id should be known, got '${cells(TopicPositionsColumn.OldestRetainedId)}'") + assert( + cells(TopicPositionsColumn.NewestRetainedPublished).trim.nonEmpty && + cells(TopicPositionsColumn.NewestRetainedPublished).trim != "-", + s"newest retained published time should be known, got '${cells(TopicPositionsColumn.NewestRetainedPublished)}'") + // The session has consumed nothing - neither bound nor progress may be invented. + assert(cells(TopicPositionsColumn.FirstConsumedId).trim == "-", + s"first consumed id should be blank before anything is read, got '${cells(TopicPositionsColumn.FirstConsumedId)}'") + assert(cells(TopicPositionsColumn.FirstConsumedPublished).trim == "-", + s"first consumed published should be blank before anything is read, got '${cells(TopicPositionsColumn.FirstConsumedPublished)}'") + assert(cells(TopicPositionsColumn.LastConsumedId).trim == "-", + s"last consumed id should be blank before anything is read, got '${cells(TopicPositionsColumn.LastConsumedId)}'") + assert(cells(TopicPositionsColumn.LastConsumedPublished).trim == "-", + s"last consumed published should be blank before anything is read, got '${cells(TopicPositionsColumn.LastConsumedPublished)}'") + assert(cells(TopicPositionsColumn.PublishTimeLag).trim == "-", + s"publish-time lag should be blank without a cursor, got '${cells(TopicPositionsColumn.PublishTimeLag)}'") + assert(cells(TopicPositionsColumn.PublishTimeFraction).trim == "-", + s"publish-time position should be blank, got '${cells(TopicPositionsColumn.PublishTimeFraction)}'") + assert(cells(TopicPositionsColumn.EntryPositionFraction).trim == "-", + s"stored entry position percentage should be blank, got '${cells(TopicPositionsColumn.EntryPositionFraction)}'") + } + + test("CS-TP-5: polling runs only while Topic Positions is visible") { + installTopicPositionsRequestCounter() + val (t, ns, topic) = fixtures.freshTopicParts() + val topicFqn = s"persistent://$t/$ns/$topic" + fixtures.produceStrings(topicFqn, 2) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.closeTools() + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(2) + + // Topic Positions is the default tab internally, but the closed Tools panel is also a + // visibility gate. Starting and consuming must not silently arm its broker-heavy poller. + page.waitForTimeout(2200) + assert(topicPositionsRequestCount == 0, + s"Topic Positions made $topicPositionsRequestCount request(s) while Tools was closed") + + cs.openTools() + val tools = ToolsPanel(page) + assertThat(tools.topicPositionsRow(topicFqn)).isVisible() + val openedCount = topicPositionsRequestCount + assert(openedCount > 0, "opening Topic Positions did not make its initial request") + // A visible tab keeps polling - the contract in this cell's title. The window is deliberately + // GENEROUS and pins no cadence: the Table asks for 1000ms, but SWR floors the effective period + // at its dedupingInterval (2000ms by default), so refreshes actually land every ~2s. Measured + // on 2026-08-09 (samples at 500ms: first refresh at ~2.5-3s, then 5s, 7s, 9s), which is why the + // old fixed 2200ms wait failed deterministically - it expired just before the first refresh. + harness.Eventually.eventually(timeoutMs = 8000, intervalMs = 250) { + assert(topicPositionsRequestCount > openedCount, + s"visible Topic Positions did not refresh: request count stayed at $openedCount") + } + + // Switching tabs unmounts the Table poller. Give the state transition a moment, then observe + // more than two refresh intervals: no NEW request may start (an already-issued response may + // still finish, which is harmless and deliberately not confused with another request here). + tools.produceTab.click() + assertThat(tools.produceSend).isVisible() + page.waitForTimeout(150) + val hiddenTabCount = topicPositionsRequestCount + page.waitForTimeout(2200) + assert(topicPositionsRequestCount == hiddenTabCount, + s"Topic Positions kept polling behind Produce: $hiddenTabCount -> $topicPositionsRequestCount") + + // Reopening proves the counter and refresh lifecycle are live, then closing the whole panel + // exercises the second visibility gate independently of tab selection. + tools.topicPositionsTab.click() + assertThat(tools.topicPositionsTable).isVisible() + page.waitForTimeout(1200) + assert(topicPositionsRequestCount > hiddenTabCount, + s"Topic Positions did not resume polling after reopening: still $topicPositionsRequestCount") + + cs.closeTools() + page.waitForTimeout(150) + val closedPanelCount = topicPositionsRequestCount + page.waitForTimeout(2200) + assert(topicPositionsRequestCount == closedPanelCount, + s"Topic Positions kept polling with Tools closed: $closedPanelCount -> $topicPositionsRequestCount") + } + + test("CS-TP-6: sorting is live, and the ALL TOPICS row stays pinned on top through it") { + val (t, ns, topic, topicFqn) = fixtures.freshTopicPartsOfKind(fixtures.TopicKind.PersistentPartitioned) + fixtures.produceStrings(topicFqn, 30) + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.play() + cs.awaitLoaded(30) + cs.openTools() + + val tools = ToolsPanel(page) + tools.topicPositionsTab.click() + assertThat(tools.topicPositionsRow("All topics")).isVisible() + + // The table opens sorted by topic ASCENDING (its defaultSort), so ONE click flips it to + // descending: partition-2 must lead the real rows, and the aggregate must not move - it + // summarizes the table, it does not compete with it. Descending is the interesting direction: + // the Table implements it by reversing the sorted array, which is exactly the operation that + // would flip a naive comparator-based pin to the bottom. + tools.topicPositionsSortBy("topic") + val rows = page.locator("[data-testid='topic-positions'] tbody tr").allTextContents() + assert(rows.size() >= 4, s"expected the aggregate plus 3 partitions, got ${rows.size()}") + assert(rows.get(0).contains("All topics"), s"the aggregate must stay pinned first, got '${rows.get(0).take(60)}'") + assert(rows.get(1).contains("partition-2"), s"desc sort by topic should lead with partition-2, got '${rows.get(1).take(80)}'") + } diff --git a/e2e/src/test/scala/features/consumersession/CsWideTopologySpec.scala b/e2e/src/test/scala/features/consumersession/CsWideTopologySpec.scala new file mode 100644 index 000000000..abf2ea04f --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/CsWideTopologySpec.scala @@ -0,0 +1,113 @@ +package features.consumersession + +import harness.DekafSuite +import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat + +/** WIDTH: two hundred delivery streams through one session - the scale the merge's O(1) emission + * gate, the per-session prefetch budget and the guaranteed replay barrier were built for, + * exercised against a real broker end to end. + * + * A 200-partition topic IS 200 delivery streams: the runner subscribes and merges per partition, + * so partitions and distinct topics are the same width to everything downstream of target + * resolution (a 200-topic selector differs only in admin-side resolution cost, which the + * admission caps bound separately). + * + * The two tests are the two modes' CONTRASTING contracts at width, on identically shaped data: + * + * - BEST-EFFORT completes a finite backlog on its own and keeps FOLLOWING - bounded residence + * releases the tails, and the session stays running for whatever arrives next. + * - GUARANTEED is an exact replay (owner decision 2026-08-09): the whole recorded backlog + * delivers - a partition running dry holds nothing back, because history needs no proof + * from the future - and the session then AUTO-PAUSES at the boundary with the caught-up + * banner. New words arrive as the NEXT chunk, replayed exactly by an explicit Resume. + * + * That contrast is the whole reason both modes exist: one mode follows live traffic at a bounded + * lateness cost, the other replays immutable history exactly and hands the user the boundary. + */ +class CsWideTopologySpec extends DekafSuite: + + private val partitions = 200 + private val backlog = 20_000 + + private val bestEffortLabel = "Best effort" + private val guaranteedLabel = "Guaranteed" + + /** A fresh 200-partition topic with `backlog` messages spread round-robin PER MESSAGE, so every + * partition holds an equal share (batched bulk produce would leave most of a wide topic empty, + * and 200 NON-empty streams is the width the replay barrier has to finish across). */ + private def widePartitionedTopic(): (String, String, String, String) = + val t = fixtures.createTenant() + val ns = fixtures.createNamespace(t) + val topic = s"wide-${System.currentTimeMillis()}" + val fqn = s"persistent://$t/$ns/$topic" + fixtures.admin.topics().createPartitionedTopic(fqn, partitions) + fixtures.produceStringsFastRoundRobin(fqn, backlog) + (t, ns, topic, fqn) + + test("CS-WT-1: BEST-EFFORT at 200 partitions - a 20,000-message backlog completes on its own") { + val (t, ns, topic, _) = widePartitionedTopic() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(bestEffortLabel) + cs.play() + cs.awaitLoaded(backlog, timeoutMs = 180000) + + assert(cs.loadedCount == backlog, s"every message must arrive, got ${cs.loadedCount}") + assert(cs.state == "running", s"the session must still be healthy, got '${cs.state}'") + cs.stop() + } + + test("CS-WT-2: GUARANTEED at 200 partitions - the whole recorded backlog replays exactly, auto-pauses at the boundary, and Resume replays the delta") { + val (t, ns, topic, fqn) = widePartitionedTopic() + + val cs = ConsumerSessionPage(page) + cs.openForTopic(t, ns, topic) + cs.setStartFrom("Earliest message") + cs.setDeliveryOrder(guaranteedLabel) + cs.play() + + // Phase 1: the WHOLE recorded range. The old contract settled short of the backlog (a + // drained partition held the rest hostage); the replay contract delivers every one of the + // 20,000 - the barrier at width holds nothing back - and then the session AUTO-PAUSES at the + // boundary with the caught-up banner, on the ordinary paused state. + cs.awaitLoaded(backlog, timeoutMs = 180000) + assertThat(page.getByTestId("cs-replay-caught-up")) + .isVisible(new com.microsoft.playwright.assertions.LocatorAssertions.IsVisibleOptions().setTimeout(20000)) + cs.assertState("paused", timeoutMs = 10000) + Thread.sleep(2000) // the boundary must be a fixpoint, not a moment the counter passed through + assert(cs.loadedCount == backlog, s"the replay must stop exactly at the boundary, got ${cs.loadedCount}") + // Round-robin production is time-ordered within every partition, so exactness at width has + // nothing to confess: no source inversions, no late deliveries, and no seam flags. + assertThat(page.getByTestId("cs-order-warning")).not().isVisible() + + // Phase 2: one closing word per partition INTO the pause, ticked so publish times strictly + // increase past the whole backlog - 200 words recorded past the boundary, none of which may + // deliver into the closed chunk. + val closer = fixtures.client + .newProducer(org.apache.pulsar.client.api.Schema.STRING) + .topic(fqn) + .enableBatching(false) + .create() + try + (0 until partitions).foreach { i => + closer.send(f"close-$i%03d") + Thread.sleep(2) + } + finally closer.close() + + Thread.sleep(2000) // ample time to be wrong before asserting the boundary held + assert(cs.loadedCount == backlog, s"pause-window words belong to the next chunk, got ${cs.loadedCount}") + + // Phase 3: Resume extends the boundary - all 200 closing words replay exactly once, one per + // stream, and the banner returns at the new boundary. + page.getByTestId("cs-replay-resume").click() + cs.awaitLoaded(backlog + partitions, timeoutMs = 60000) + assertThat(page.getByTestId("cs-replay-caught-up")) + .isVisible(new com.microsoft.playwright.assertions.LocatorAssertions.IsVisibleOptions().setTimeout(20000)) + cs.assertState("paused", timeoutMs = 10000) + Thread.sleep(2000) + assert(cs.loadedCount == backlog + partitions, s"the delta must replay exactly once, got ${cs.loadedCount}") + cs.stop() + } diff --git a/e2e/src/test/scala/features/consumersession/StartFromSupport.scala b/e2e/src/test/scala/features/consumersession/StartFromSupport.scala new file mode 100644 index 000000000..58a15935d --- /dev/null +++ b/e2e/src/test/scala/features/consumersession/StartFromSupport.scala @@ -0,0 +1,168 @@ +package features.consumersession + +import harness.DekafSuite +import harness.Eventually.eventually +import org.apache.pulsar.common.policies.data.TopicStats + +import scala.jdk.CollectionConverters.* + +/** Shared arrangement + assertion helpers for the Start-From outcome specs + * (`CsStartFromOutcomesSpec`, `CsStartFromMatrixSpec`). + * + * The one assertion that matters: every test states the EXACT set of message values the session + * must end up showing. Counting rows cannot express "skip first 5" - a run that skipped 50 and one + * that skipped 5 both reach *some* count, and a transient count is reached by almost anything. + */ +/** How a test's data is written to the broker. Both shapes are ordinary application behavior; only + * the second one was ever exercised before, which is why entry-vs-message defects in Start-From + * went unseen. */ +enum Produce(val label: String): + /** Several messages share ONE broker entry - the Java producer's default behavior. */ + case Batched extends Produce("batched") + /** One message per broker entry. */ + case Unbatched extends Produce("unbatched") + +trait StartFromSupport extends DekafSuite: + + /** Messages per entry for `Produce.Batched`. Chosen so the test payloads below land in a handful + * of entries, i.e. entry positions and message positions are far apart and cannot coincide. */ + val MessagesPerBatch = 4 + + def produceAs(mode: Produce, fqn: String, values: Seq[String]): Unit = mode match + case Produce.Batched => fixtures.produceBatched(fqn, values, MessagesPerBatch) + case Produce.Unbatched => fixtures.produceUnbatched(fqn, values) + + /** Handshake rows: produced only to prove the live stream is flowing, and excluded from every + * payload assertion by this prefix. */ + val HandshakePrefix = "handshake-" + + /** The session must show EXACTLY `expected`, no more and no less. + * + * Set-based because a partitioned topic merges several consumers and has no total order. Waiting + * for the whole set (rather than a count) is what makes it non-transient: a missing message keeps + * polling until the deadline, and an extra one can never satisfy the equality. + * + * Rows come from [[ConsumerSessionPage.allColumnValues]], which walks the WHOLE virtualized + * list. The mounted-viewport read this used to be built on could only ever see the last + * viewport-worth of rows of the auto-scrolled table, so any expectation taller than one + * viewport failed on its OLDEST rows (CS-SF-7/9/13/14, all "missing m-01" the moment the + * tools pane opened by default and shrank the viewport). + * + * What it still does NOT prove is that nothing arrives AFTER the first matching poll - the + * shape a too-wide start-from produces. Prefer [[assertLoadedExactlyWithCounter]], which closes + * that with the toolbar counter; this bare form is for the quadrants where the counter cannot + * be predicted. */ + def assertLoadedExactly(cs: ConsumerSessionPage, expected: Seq[String]): Unit = + val rendered = eventually(timeoutMs = 45000, intervalMs = 400) { + val all = cs.allColumnValues("value") + val payload = all.filterNot(_.startsWith(HandshakePrefix)) + assert( + payload.toSet == expected.toSet, + s"the session shows ${payload.size} message(s): $payload\n expected exactly ${expected.size}: $expected" + + s"\n missing: ${expected.toSet.diff(payload.toSet)}\n unexpected: ${payload.toSet.diff(expected.toSet)}" + ) + payload + } + assert(rendered.size == expected.size, s"a message is rendered more than once: $rendered") + + /** Same, plus the toolbar's loaded counter - two things the rendered rows alone cannot say. + * + * The message table is VIRTUALIZED, so a set built from DOM rows cannot rule out messages loaded + * OFF-SCREEN; and the set assertion above is satisfied by the first poll that sees the right + * rows, so a message arriving AFTER that - the shape a too-wide start-from produces - would slip + * past it. The counter answers the first, and requiring the counter to STAY put answers the + * second. + * + * `base` is what the counter already stood at before `expected` was produced. It is 0 whenever no + * handshake rows are in play; a quadrant that needs them takes its baseline from + * [[settledLoaded]], because how many handshakes survived is not knowable up front (the early + * ones are dropped by design) yet all of them count towards `cs-loaded`. */ + def assertLoadedExactlyWithCounter(cs: ConsumerSessionPage, expected: Seq[String], base: Int = 0): Unit = + assertLoadedExactly(cs, expected) + val total = base + expected.size + cs.awaitLoaded(total) + val settled = settledLoaded(cs) + assert( + settled == total, + s"the session loaded $settled message(s) once the counter stopped moving, expected $total " + + s"(${expected.size} expected + $base already loaded before them). Rendered: ${cs.allColumnValues("value")}" + ) + + /** The loaded counter once it has STOPPED MOVING, i.e. two reads `quietMs` apart that agree. + * + * The window is quiescence, never readiness - nothing here is waiting for the app to catch up + * with a request, it is waiting for the app to prove it has nothing more to deliver. That is the + * only observable form "and no further message arrives" can take: a counter that has reached the + * right value tells you nothing about the message still in flight behind it. + * + * LIMIT, stated so it is not mistaken for airtight: the quiet window is finite (~1.2s), so an + * over-delivery arriving LATER than it - most plausibly a message the session left un-acked being + * redelivered on the broker's ack timeout, tens of seconds out - would land after this returns and + * still pass a "shows EXACTLY n" spec. The window is deliberately NOT widened to cover that: it is + * on the hot path of every counting cell, and a redelivery interval is far too long to wait per + * test. What actually forbids the over-delivery is pinned at the SERVER tier, where it is cheap and + * deterministic - the session acks every delivered message exactly once and a failing progress push + * cannot cost a skipped message its ack (`sessionOutputSerializationTest`, suite "a failing progress + * push must not cost a skipped message"), and the start-from discard is applied exactly once and is + * re-armed by neither a redelivery nor a resume (`startFromDiscardOnceTest`). This helper is the + * UI-level cross-check on top of that, not the primary guard against duplication. */ + def settledLoaded(cs: ConsumerSessionPage, quietMs: Int = 1200): Int = + eventually(timeoutMs = 30000, intervalMs = 200) { + val before = cs.loadedCount + page.waitForTimeout(quietMs) + val after = cs.loadedCount + assert(before == after, s"the loaded counter is still moving: $before then $after") + after + } + + /** Available-permit counts per consumer, keyed by the non-partitioned topic each is attached to. + * `availablePermits > 0` is the signal that the client has actually issued flow permits. */ + private def sessionConsumerPermits(fqn: String, kind: fixtures.TopicKind): Map[String, List[Int]] = + def permitsOf(st: TopicStats): List[Int] = + st.getSubscriptions.values().asScala.toList + .flatMap(sub => sub.getConsumers.asScala.toList.map(_.getAvailablePermits)) + if kind.isPartitioned then + admin.topics().getPartitionedStats(fqn, true).getPartitions.asScala.toMap.map { case (p, st) => p -> permitsOf(st) } + else Map(fqn -> permitsOf(admin.topics().getStats(fqn))) + + /** Wait until the session is consuming EVERY part of the topic - one attachment for a + * non-partitioned topic, one per partition otherwise. */ + def awaitConsumersFlowing(fqn: String, kind: fixtures.TopicKind): Unit = + val expected = math.max(kind.partitions, 1) + eventually(timeoutMs = 30000, intervalMs = 400) { + val permits = + try sessionConsumerPermits(fqn, kind) + catch case _: Throwable => Map.empty[String, List[Int]] // topic not materialized yet + assert( + permits.size == expected && permits.values.forall(_.exists(_ > 0)), + s"the session is not consuming every part of $fqn: expected $expected attachment(s) with permits > 0, got $permits" + ) + } + + /** Produce handshake messages until the session RENDERS one, i.e. until the whole path + * (broker -> consumer -> listener -> gRPC stream -> table) is provably live. + * + * Needed on NON-PERSISTENT topics: nothing is retained there, so anything published before the + * session's message handler is installed is acked, discarded and unrecoverable. A rendered row is + * the only observable proof that window has closed. The retries are why the prefix is filtered + * out of payload assertions. */ + def awaitSessionStreaming(cs: ConsumerSessionPage, fqn: String): Unit = + var attempt = 0 + eventually(timeoutMs = 60000, intervalMs = 1200) { + attempt += 1 + fixtures.produceUnbatched(fqn, Seq(s"$HandshakePrefix$attempt")) + val rendered = cs.columnValues("value") + assert(rendered.exists(_.startsWith(HandshakePrefix)), s"the session has not rendered a handshake message yet: $rendered") + } + + /** Block until `gapMs` has elapsed since `sinceMs`. + * + * This is ARRANGEMENT, not a readiness wait: the time-addressed Start-From modes can only be + * asserted if the test data straddles a known instant, and the only way to put messages on + * either side of one is to publish them at different times. Expressed as a polled predicate so + * it cannot silently become "sleep and hope the app caught up". */ + def awaitClockGap(sinceMs: Long, gapMs: Long): Unit = + eventually(timeoutMs = gapMs + 30000, intervalMs = 200) { + val elapsed = System.currentTimeMillis() - sinceMs + assert(elapsed >= gapMs, s"only ${elapsed}ms of the required ${gapMs}ms gap has elapsed") + } diff --git a/e2e/src/test/scala/features/library/LibraryNotesSpec.scala b/e2e/src/test/scala/features/library/LibraryNotesSpec.scala index c2229fe52..981917fdd 100644 --- a/e2e/src/test/scala/features/library/LibraryNotesSpec.scala +++ b/e2e/src/test/scala/features/library/LibraryNotesSpec.scala @@ -45,6 +45,68 @@ class LibraryNotesSpec extends DekafSuite: assertThat(lib.createFirstNoteButton).isVisible() // back to empty state } + /** Hold every `ListLibraryItems` response back by `delayMs`, from inside the browser. + * + * ARRANGEMENT, not a readiness wait: the panel's loading state is only observable while its first + * fetch is genuinely in flight, and on a local stack that window is a few milliseconds wide - far + * too narrow for a test to land in reliably. Widening it deterministically is the only way to + * make the state a test can be written against. + * + * Done by wrapping `XMLHttpRequest.send` in the page (grpc-web's transport) rather than with + * Playwright's `page.route`: a Java route handler that sleeps blocks the driver's dispatch loop, + * which would ALSO stall the test's own `isVisible` call and hide the very race being reproduced. + * A `setTimeout` in the page delays exactly one request and nothing else. The delay is bounded + * and small - the point is to be reliably slower than a click, not to test a timeout. */ + private def delayListLibraryItems(delayMs: Int): Unit = + context.addInitScript( + s"""(() => { + | const marker = 'LibraryService/ListLibraryItems'; + | const openOrig = XMLHttpRequest.prototype.open; + | const sendOrig = XMLHttpRequest.prototype.send; + | XMLHttpRequest.prototype.open = function (method, url) { + | this.__delayUrl = String(url); + | return openOrig.apply(this, arguments); + | }; + | XMLHttpRequest.prototype.send = function () { + | const args = arguments; + | if (this.__delayUrl && this.__delayUrl.indexOf(marker) !== -1) { + | window.__delayedListLibraryItems = (window.__delayedListLibraryItems || 0) + 1; + | setTimeout(() => sendOrig.apply(this, args), $delayMs); + | return; + | } + | return sendOrig.apply(this, args); + | }; + |})();""".stripMargin + ) + + // The regression for the Notes panel's readiness precondition. Until it was fixed, `createNote` + // asked `createFirstNoteButton.isVisible` - a question with no wait attached - while the panel was + // still rendering "Loading...". Neither button exists then, so the answer was `false`, the else + // branch clicked `lib-new-note`, and that button can never appear on a topic with no notes: the + // test burned its whole timeout and failed on an app that was working perfectly. + // + // Nothing pinned that, because ordinary tests never delay `ListLibraryItems` and the panel settles + // in milliseconds on a local stack - the flake needed a slow or loaded machine to appear at all. + test("LIB-22: a note can be created while the first ListLibraryItems is still in flight") { + delayListLibraryItems(3000) + val (t, ns, topic) = openTopicOverview() + val lib = LibrarySidebar(page) + lib.openNotesTab() + + // The precondition this test exists for: the panel really is in its unsettled state, with + // NEITHER button on screen. Asserted rather than assumed - if the delay ever stopped taking + // effect the test below would still pass, and would silently stop covering anything. + assert(!lib.createFirstNoteButton.isVisible, "the panel had already settled - the response delay did not take effect") + assert(!lib.newNoteButton.isVisible, "the panel had already settled - the response delay did not take effect") + + lib.createNote() + assertThat(lib.noteTab("Note 1")).hasCount(1, new LocatorAssertions.HasCountOptions().setTimeout(20000)) + + // ... and the delay applied to the real request, not to some other call that happened to match. + val delayed = page.evaluate("() => window.__delayedListLibraryItems || 0").asInstanceOf[Number].intValue + assert(delayed > 0, "no ListLibraryItems request was delayed") + } + // NOTE: instance-scope leg is NOT parallel-safe and leaks one instance note (no LibraryService teardown). // Run in a serial lane. See packet NOTES. test("LIB-16: the '⭐️ Updates' pseudo-note appears only on the Instance scope") { diff --git a/e2e/src/test/scala/features/producer/ProducerBytesSpec.scala b/e2e/src/test/scala/features/producer/ProducerBytesSpec.scala index 4bd7306b5..e7160e70a 100644 --- a/e2e/src/test/scala/features/producer/ProducerBytesSpec.scala +++ b/e2e/src/test/scala/features/producer/ProducerBytesSpec.scala @@ -33,7 +33,8 @@ class ProducerBytesSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.openTools() // reveal the (already-mounted) Produce console + cs.openTools() + page.getByTestId("console-tab-produce").click() dismissCredentialsIfPresent() page.getByTestId("produce-encoding").selectOption(new SelectOption().setLabel("Bytes (hex)")) diff --git a/e2e/src/test/scala/features/producer/ProducerSpec.scala b/e2e/src/test/scala/features/producer/ProducerSpec.scala index eb140a003..364277965 100644 --- a/e2e/src/test/scala/features/producer/ProducerSpec.scala +++ b/e2e/src/test/scala/features/producer/ProducerSpec.scala @@ -16,7 +16,8 @@ class ProducerSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) - cs.openTools() // open the console (force-click; its tooltip can overlay the button) + cs.openTools() + page.getByTestId("console-tab-produce").click() // A first Pulsar op may raise the credentials onboarding modal - dismiss it. page.waitForTimeout(500) diff --git a/e2e/src/test/scala/features/producer/ProducerValidationSpec.scala b/e2e/src/test/scala/features/producer/ProducerValidationSpec.scala index 7733f4114..61de81db9 100644 --- a/e2e/src/test/scala/features/producer/ProducerValidationSpec.scala +++ b/e2e/src/test/scala/features/producer/ProducerValidationSpec.scala @@ -18,6 +18,7 @@ class ProducerValidationSpec extends DekafSuite: val cs = ConsumerSessionPage(page) cs.openForTopic(t, ns, topic) cs.openTools() + page.getByTestId("console-tab-produce").click() page.waitForTimeout(500) if page.getByText("Pulsar Credentials").isVisible then page.getByRole(AriaRole.BUTTON, new GetByRoleOptions().setName("Done").setExact(true)).click() diff --git a/e2e/src/test/scala/harness/BatchingFixtureSpec.scala b/e2e/src/test/scala/harness/BatchingFixtureSpec.scala new file mode 100644 index 000000000..f10906746 --- /dev/null +++ b/e2e/src/test/scala/harness/BatchingFixtureSpec.scala @@ -0,0 +1,170 @@ +package harness + +import harness.Eventually.eventually + +import java.nio.charset.StandardCharsets.UTF_8 +import scala.jdk.CollectionConverters.* + +/** Broker-level spec for the batching fixtures - no UI, PulsarAdmin is both the arrange and the + * assert side. + * + * Two things are pinned here, and everything the batched start-from coverage claims rests on them: + * + * 1. `fixtures.produceBatched` really does put several messages into ONE broker entry, and + * `fixtures.produceUnbatched` really does put exactly one message in each. Without this the + * "batched" half of the matrix would be indistinguishable from the unbatched half and would + * silently prove nothing - the very blind spot that coverage exists to close. + * + * 2. `PulsarAdmin.examineMessage` addresses ENTRIES, not messages, and past the end of the log it + * answers with the wrong entry ("earliest") or throws ("latest") rather than saying so. That is + * the mechanism by which "skip the first 5 messages" silently skipped fifty while the two + * counting Start-From modes were built on it. They no longer are - the seek was rewritten on + * 2026-07-25 to count delivered MESSAGES - so this is now a pin on the primitive rather than on + * the app: it records why entry-addressing cannot implement a message-count contract, so + * nobody rebuilds those modes on it. + */ +class BatchingFixtureSpec extends DekafSuite: + + private def valueOf(m: org.apache.pulsar.client.api.Message[Array[Byte]]): String = + new String(m.getData, UTF_8) + + /** `numberOfEntries` is read back through the admin API; give it a beat to catch up with writes + * we already hold producer acks for. */ + private def awaitEntries(fqn: String, atLeast: Long): Long = + eventually(timeoutMs = 10000, intervalMs = 200) { + val n = fixtures.numberOfEntries(fqn) + assert(n >= atLeast, s"only $n entrie(s) visible on $fqn yet (want >= $atLeast)") + n + } + + test("BATCH-1: produceBatched packs many messages into few entries; produceUnbatched does not") { + val (t, ns, _) = fixtures.freshTopicParts() + + // --- batched: 100 messages, 50 per batch -> 2 entries ------------------------------------- + val batchedFqn = fixtures.createTopic(t, ns) + val values = (1 to 100).map(i => f"m-$i%03d") + fixtures.produceBatched(batchedFqn, values, messagesPerBatch = 50) + + val batchedEntries = awaitEntries(batchedFqn, 1) + assert(batchedEntries == 2, s"100 messages at 50/batch should be 2 broker entries, got $batchedEntries") + // The messages themselves must all be there - "few entries" must not mean "lost messages". + val read = fixtures.readAllMessages(batchedFqn).map(_.getValue) + assert(read == values.toVector, s"batched produce lost or reordered messages: got ${read.size} - $read") + + // --- unbatched: one entry per message ------------------------------------------------------ + val unbatchedFqn = fixtures.createTopic(t, ns) + fixtures.produceUnbatched(unbatchedFqn, values) + val unbatchedEntries = awaitEntries(unbatchedFqn, 100) + assert(unbatchedEntries == 100, s"unbatched produce should be 1 entry per message, got $unbatchedEntries for 100") + + // --- and the shape every pre-existing fixture produced -------------------------------------- + // `produceStrings` leaves the client default (batching ON) but sends with a BLOCKING send, which + // closes a one-message batch every time. That is why the suite could never see an entry-vs- + // message confusion before: every fixture in it produced entry-per-message data. + val legacyFqn = fixtures.createTopic(t, ns) + fixtures.produceStrings(legacyFqn, 100) + val legacyEntries = awaitEntries(legacyFqn, 100) + assert(legacyEntries == 100, s"blocking send should still be 1 entry per message, got $legacyEntries") + } + + test("BATCH-2: examineMessage is ENTRY-addressed; past the end it clamps (earliest) or fails (latest)") { + val (t, ns, _) = fixtures.freshTopicParts() + val fqn = fixtures.createTopic(t, ns) + val values = (1 to 100).map(i => f"m-$i%03d") + fixtures.produceBatched(fqn, values, messagesPerBatch = 50) // -> entry 1 = m-001.., entry 2 = m-051.. + assert(awaitEntries(fqn, 2) == 2) + + def examine(position: String, n: Long): String = + valueOf(admin.topics().examineMessage(fqn, position, n)) + + // Position 1 and 2 are the two ENTRIES - each answers with the FIRST message of that entry, not + // with the 1st and 2nd messages of the topic. + assert(examine("earliest", 1) == "m-001", s"entry 1 answered ${examine("earliest", 1)}") + assert(examine("earliest", 2) == "m-051", s"entry 2 answered ${examine("earliest", 2)} - if this is m-002, examineMessage is message-addressed") + + // Everything past the last entry answers with the last entry instead of failing. This is the + // whole defect mechanism for "skip first n": the mode asks for position n+1, gets entry 2, and + // silently skips 50 messages. + for n <- Seq(3L, 50L, 100L) do + assert( + examine("earliest", n) == "m-051", + s"examineMessage(earliest, $n) answered ${examine("earliest", n)}; expected the last entry's first message (m-051). " + + "Message-addressing would have answered m-003/m-050/m-100." + ) + + // Counting back from the end is entry-addressed too... + assert(examine("latest", 1) == "m-051", s"latest,1 answered ${examine("latest", 1)}") + assert(examine("latest", 2) == "m-001", s"latest,2 answered ${examine("latest", 2)}") + + // ...but past the end it does NOT clamp - it fails outright (ManagedLedgerException surfaced as + // an admin 500). Note the asymmetry with "earliest" above: a caller that only tested one + // direction would conclude examineMessage either always clamps or always fails, and both + // conclusions are wrong. + // + // This asymmetry is load-bearing in BOTH directions of the rewrite. It is why the old "Latest n + // messages" seek fell back to EARLIEST and showed every message there is: it caught the failure + // and could not tell "past the start of the log" from "the broker refused". And it is what the + // current `resolveLatestN` MERGED backward walk uses as its per-topic end-of-log signal - one + // cursor per topic steps back entry by entry while the merge accumulates message counts to n, + // and a cursor only leaves the merge when this failure (classified by `isEmptyLogAnswer`, so a + // genuine broker error still aborts instead) says its log has no older entry. Nothing may be + // built on "earliest" instead: that side clamps silently. + for n <- Seq(3L, 50L, 100L) do + val thrown = intercept[org.apache.pulsar.client.admin.PulsarAdminException] { + admin.topics().examineMessage(fqn, "latest", n) // must NOT return a message + } + assert( + Option(thrown.getMessage).exists(_.contains("Incorrect parameter input")), + s"examineMessage(latest, $n) failed for an unexpected reason: ${thrown.getMessage}" + ) + } + + test("BATCH-4: on a partitioned topic only the partitions holding data can answer examineMessage") { + // Why a per-partition search cannot be made to answer a global "n-th message" question: a + // partition holding no data cannot answer at all, so any algorithm that polls every partition + // sees candidates only from the ones that happen to hold data. This also pins that a partition + // of a partitioned topic is entry-addressed exactly like any other topic. + val kind = fixtures.TopicKind.PersistentPartitioned + val (t, ns, _, fqn) = fixtures.freshTopicPartsOfKind(kind) + val values = (1 to 12).map(i => f"m-$i%02d") + fixtures.produceBatched(s"$fqn-partition-0", values, 4) // -> 3 entries, all on partition 0 + + // Materialize every partition the way a session subscribing to the parent does. + val consumer = client.newConsumer().topic(fqn).subscriptionName("batch-4-probe").subscribe() + try + val partitions = admin.topics().getList(s"$t/$ns").asScala.toList.sorted + assert(partitions.size == 3, s"expected 3 materialized partitions, got $partitions") + + val loaded = partitions.filter(_.endsWith("-partition-0")) + val empty = partitions.filterNot(_.endsWith("-partition-0")) + + // The loaded partition answers entry-wise, and clamps past the end just like BATCH-2. + assert(valueOf(admin.topics().examineMessage(loaded.head, "earliest", 6)) == "m-09") + + // The empty ones cannot answer at all - so a multi-partition search only ever sees candidates + // from partitions that happen to hold data. + empty.foreach { p => + val thrown = intercept[org.apache.pulsar.client.admin.PulsarAdminException] { + admin.topics().examineMessage(p, "earliest", 6) + } + assert( + Option(thrown.getMessage).exists(_.contains("total message is zero")), + s"$p failed for an unexpected reason: ${thrown.getMessage}" + ) + } + finally consumer.close() + } + + test("BATCH-3: on unbatched data examineMessage positions coincide with message numbers") { + // The control for BATCH-2: with one message per entry, entry-addressing and message-addressing + // are the same thing - which is exactly why unbatched fixtures could never expose the defect. + val (t, ns, _) = fixtures.freshTopicParts() + val fqn = fixtures.createTopic(t, ns) + val values = (1 to 10).map(i => f"m-$i%03d") + fixtures.produceUnbatched(fqn, values) + assert(awaitEntries(fqn, 10) == 10) + + for n <- 1 to 10 do + val got = valueOf(admin.topics().examineMessage(fqn, "earliest", n.toLong)) + assert(got == f"m-$n%03d", s"examineMessage(earliest, $n) answered $got") + } diff --git a/e2e/src/test/scala/harness/DeliveredMessagesSpec.scala b/e2e/src/test/scala/harness/DeliveredMessagesSpec.scala new file mode 100644 index 000000000..26fb27681 --- /dev/null +++ b/e2e/src/test/scala/harness/DeliveredMessagesSpec.scala @@ -0,0 +1,77 @@ +package harness + +import features.consumersession.DeliveredMessages +import org.scalatest.funsuite.AnyFunSuite + +/** The fact the pause/flow specs' "zero loss" claim rests on: that their ORACLE can tell a correct + * delivery from a merely plausible one. + * + * `CsPauseLoopSpec` and `CsFlowControlSpec` used to conclude "nothing was lost, nothing was + * duplicated" from the final message COUNT. This suite runs that count, and the exact-set + * comparison that replaced it, over the SAME wrong data - a stream that lost one message and + * delivered another twice - and shows the count accepting it. That is the whole content of the + * finding, made executable and permanent: the browser-driven specs cannot demonstrate it (they + * would have to be fed a broken server), and without it a future edit could quietly relax the set + * assertion back to a count and nothing would go red. + * + * Pure - no browser, no broker, no Dekaf. It runs in the same lane as everything else. + */ +class DeliveredMessagesSpec extends AnyFunSuite: + + /** A well-behaved 5,000-message delivery: every produced value, exactly once. */ + private val produced: Vector[String] = (1 to 5000).map(i => s"bk-$i").toVector + + /** The same delivery with ONE message never delivered and ONE delivered twice. The size is + * unchanged - that is the point. */ + private val lostOneDuplicatedOne: Vector[String] = + produced.filterNot(_ == "bk-17").appended("bk-4242") + + test("SET-1: the exact-set oracle accepts a correct delivery, in any order") { + assert(DeliveredMessages.difference(produced, produced).isEmpty) + // Order is deliberately not part of the contract - a message refused at a closed pause gate is + // nacked and can come back behind a newer one - so a shuffle must still pass. + assert(DeliveredMessages.difference(scala.util.Random.shuffle(produced), produced).isEmpty) + } + + test("SET-2: THE DEFECT - a count accepts one loss plus one duplicate; the exact set does not") { + // Both halves of the same arrangement, on one line each, because their disagreement IS the + // finding: the count is blind here, and blindness is exactly what let the branch's original + // delivery bug survive its own regression test. + assert( + lostOneDuplicatedOne.size == produced.size, + "arrangement error: the wrong delivery must have the same SIZE as the right one, or the " + + "count would catch it and there would be nothing to demonstrate" + ) + val text = DeliveredMessages.difference(lostOneDuplicatedOne, produced).getOrElse( + fail("the exact-set oracle accepted a delivery that lost bk-17 and repeated bk-4242") + ) + // And it must say WHICH IS WHICH on the right line: a report that only says "not equal" over + // 5,000 values is not usable, and the two failures have unrelated causes (a message the closed + // gate dropped vs. a redelivery shown a second time). + def line(classification: String): String = + text.linesIterator.find(_.contains(classification)).getOrElse(fail(s"no $classification line in:\n$text")) + assert(line("MISSING").contains("bk-17"), s"the lost message is not reported as missing: $text") + assert(line("DUPLICATED").contains("bk-4242"), s"the repeated message is not reported as duplicated: $text") + } + + test("SET-3: a message from the WRONG stream is reported as foreign, not as a duplicate") { + // The cut-and-merge specs assert which messages survived, across topics whose payloads now + // carry distinct prefixes for this reason: a message delivered from the stream that should + // have been dropped entirely is a different defect from a repeat, and reads differently. + val fromTheOtherTopic = produced.filterNot(_ == "bk-17").appended("b-17") + val text = DeliveredMessages.difference(fromTheOtherTopic, produced).getOrElse( + fail("a message from another topic passed the exact-set oracle") + ) + assert(text.contains("FOREIGN"), s"the report does not classify the stray message: $text") + assert(!text.contains("DUPLICATED"), s"a stray message is not a duplicate: $text") + } + + test("SET-4: the failure report stays bounded when everything is wrong") { + // 55,000 mismatches must not print 55,000 lines - an unreadable report is a report nobody + // reads, and this oracle exists for sets that big. + val nothingInCommon = (1 to 55000).map(i => s"other-$i") + val text = DeliveredMessages.difference(nothingInCommon, produced).getOrElse(fail("disjoint sets compared equal")) + assert(text.linesIterator.size <= 6, s"the report is unbounded:\n$text") + assert(text.contains("delivered 55000 message(s), expected 5000"), s"the report does not state the sizes: $text") + assert(text.contains("more)"), s"the report does not say how much it elided: $text") + } diff --git a/e2e/src/test/scala/harness/MessageShapeFixtureSpec.scala b/e2e/src/test/scala/harness/MessageShapeFixtureSpec.scala new file mode 100644 index 000000000..4a6dfde71 --- /dev/null +++ b/e2e/src/test/scala/harness/MessageShapeFixtureSpec.scala @@ -0,0 +1,101 @@ +package harness + +import org.apache.pulsar.client.api.SubscriptionInitialPosition + +import java.util.concurrent.TimeUnit + +/** Broker-level spec for the blind-spot payload-shape fixtures - the same role + * `BatchingFixtureSpec` plays for `produceBatched`: prove on the REAL broker that each fixture's + * premise both holds and is self-asserted, so no consumer-session test built on one can silently + * run against data of the wrong shape. Pure broker specs - no page navigation - so they also run + * without a Dekaf instance. + * + * The shapes, and the defect each keeps reachable: + * - CHUNKING (SHAPE-1): one message across many entries - the exact dual of batching. An + * entry-addressed walk that miscounts chunked messages is invisible without one on the log. + * - PUBLISH-TIME TIES (SHAPE-2): the batched producer's normal output. A merge tie-break, or a + * Guaranteed barrier comparing tied heads, is unreachable on tick-separated fixtures. + * - SIZED PAYLOADS (SHAPE-3): megabyte values. The merge's held-BYTES watermark can never trip + * under 4-byte payloads, whatever the count watermarks do. + */ +class MessageShapeFixtureSpec extends DekafSuite: + + test("SHAPE-1: produceChunked spreads ONE message across several entries, and a chunk-capable consumer reassembles it") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + val value = "chunk-" + "x" * (40 * 1024) + + val chunks = fixtures.produceChunked(fqn, value, chunkBytes = 16 * 1024) + assert(chunks == 3, s"a ${value.length}-byte value at 16 KiB/chunk should be 3 chunks, got $chunks") + // produceChunked already asserted the entry count; restate the broker fact here so THIS spec, + // not only the fixture's internal guard, records it. + assert(fixtures.numberOfEntries(fqn) == chunks, s"expected $chunks entries for one chunked message") + + // A chunk-capable consumer (the session's own consumer is built with maxPendingChunkedMessage) + // must hand the message back as ONE message, byte-identical - and nothing else. + val consumer = client + .newConsumer(org.apache.pulsar.client.api.Schema.STRING) + .topic(fqn) + .subscriptionName(fixtures.unique("shape")) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .maxPendingChunkedMessage(2) + .subscribe() + try + val m = consumer.receive(15, TimeUnit.SECONDS) + assert(m != null, "the chunked message was never delivered whole") + assert( + m.getValue == value, + s"reassembled value differs from the produced one: ${m.getValue.length} chars vs ${value.length}" + ) + consumer.acknowledge(m) + val extra = consumer.receive(2, TimeUnit.SECONDS) + assert(extra == null, s"chunks leaked as extra messages: ${Option(extra).map(_.getValue.take(50))}") + finally + try consumer.unsubscribe() + catch case _: Throwable => () + consumer.close() + } + + test("SHAPE-2: produceTiedBatches manufactures REAL publish-time ties - within batches and across topics") { + val (tA, nsA, topicA) = fixtures.freshTopicParts() + val fqnA = s"persistent://$tA/$nsA/$topicA" + val (tB, nsB, topicB) = fixtures.freshTopicParts() + val fqnB = s"persistent://$tB/$nsB/$topicB" + + val (valuesA, valuesB) = fixtures.produceTiedBatches(fqnA, fqnB, messagesPerBatch = 4) + + // The fixture throws when its premises fail; re-derive them from the broker so this spec + // states the facts independently of the fixture's own guards. + val readA = fixtures.readAllMessages(fqnA) + val readB = fixtures.readAllMessages(fqnB) + assert( + readA.map(_.getValue) == valuesA && readB.map(_.getValue) == valuesB, + s"read-back does not match what was produced: A=${readA.map(_.getValue)} B=${readB.map(_.getValue)}" + ) + Seq(fqnA -> readA, fqnB -> readB).foreach { case (fqn, read) => + read.grouped(4).foreach { batch => + assert( + batch.map(_.getPublishTime).distinct.size == 1, + s"a batch on $fqn does not share one publish time: ${batch.map(m => m.getValue -> m.getPublishTime)}" + ) + } + } + val ties = readA.map(_.getPublishTime).toSet.intersect(readB.map(_.getPublishTime).toSet) + assert(ties.nonEmpty, "no cross-topic publish-time tie landed on the broker") + } + + test("SHAPE-3: produceSized lands count x size unbatched bytes on the broker") { + val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + + val values = fixtures.produceSized(fqn, count = 3, payloadBytes = 1024 * 1024) + assert(values.size == 3 && values.forall(_.length == 1024 * 1024), "the fixture must return the exact values it sent") + assert(fixtures.numberOfEntries(fqn) == 3, s"3 unbatched messages must be 3 entries, got ${fixtures.numberOfEntries(fqn)}") + assert( + admin.topics().getStats(fqn).getStorageSize >= 3L * 1024 * 1024, + s"the broker holds ${admin.topics().getStats(fqn).getStorageSize} bytes for 3 MiB of payload" + ) + // Byte-for-byte survival at this size, through the same Reader the start-from oracles use. + val read = fixtures.readAllMessages(fqn).map(_.getValue) + assert(read == values, s"sized payloads did not survive the round trip: got ${read.map(_.length)}") + } diff --git a/e2e/src/test/scala/harness/StackScriptsSpec.scala b/e2e/src/test/scala/harness/StackScriptsSpec.scala new file mode 100644 index 000000000..d3e38587e --- /dev/null +++ b/e2e/src/test/scala/harness/StackScriptsSpec.scala @@ -0,0 +1,204 @@ +package harness + +import org.scalatest.funsuite.AnyFunSuite + +import java.nio.file.{Files, Path, Paths} +import scala.jdk.CollectionConverters.* + +/** The shell scripts that bring the stack up and take it down - the one part of the harness that no + * UI test can reach, because a test cannot run them without destroying the stack it is running on. + * + * Scoped tightly to what is testable in isolation: `scripts/fresh-data-dir.sh`, which exists so + * that `run-dekaf.sh` and `stack-down.sh` agree on the throwaway `DEKAF_DATA_DIR` path - one per + * STACK, derived from the same `DEKAF_PORT` / `PULSAR_CONTAINER_NAME` both ends already read. + * + * Why it needs pinning at all: `run-dekaf.sh` ends in `exec sbt run`, so the process that creates + * the tree is replaced by the server, and CI then kills that whole process tree - no trap, atexit + * or JVM shutdown hook in the server's own lifetime can ever fire. Cleanup therefore has to come + * from OUTSIDE, from a step that knows the path without being told, and the previous `mktemp -d` + * made that impossible: a fresh unguessable name every build, one abandoned tree per build, forever, + * on a self-hosted runner nobody wipes. + * + * No `DekafSuite` here on purpose - these need no browser, no Pulsar and no Dekaf. + */ +class StackScriptsSpec extends AnyFunSuite: + + /** The scripts dir. sbt forks tests with the project base (`e2e/`) as the working directory; the + * repo-root fallback keeps the spec runnable from an IDE that chose differently. */ + private val scripts: Path = + Seq(Paths.get("scripts"), Paths.get("e2e/scripts")) + .find(p => Files.isDirectory(p)) + .getOrElse(fail(s"cannot locate e2e/scripts from ${Paths.get("").toAbsolutePath}")) + + private def run(script: String, args: Seq[String], env: Map[String, String]): (Int, String) = + val pb = new ProcessBuilder((Seq("bash", scripts.resolve(script).toString) ++ args).asJava) + pb.redirectErrorStream(true) + env.foreach((k, v) => pb.environment().put(k, v)) + val process = pb.start() + val out = new String(process.getInputStream.readAllBytes(), "UTF-8") + (process.waitFor(), out.trim) + + /** The environment of one stack: its runner temp plus the two variables that identify it. Both are + * already read by the scripts on either end (`run-dekaf.sh` serves on `DEKAF_PORT`, + * `stack-down.sh` removes `PULSAR_CONTAINER_NAME`), which is what lets startup and teardown agree + * on a path without anything being passed between them. */ + private def stackEnv(runnerTemp: Path, dekafPort: String, container: String): Map[String, String] = + Map("RUNNER_TEMP" -> runnerTemp.toString, "DEKAF_PORT" -> dekafPort, "PULSAR_CONTAINER_NAME" -> container) + + private def freshDataDirWith(env: Map[String, String]): String = + val (code, out) = run("fresh-data-dir.sh", Seq("path"), env) + assert(code == 0, s"fresh-data-dir.sh path exited $code: $out") + out + + private def freshDataDir(runnerTemp: Path): String = + freshDataDirWith(Map("RUNNER_TEMP" -> runnerTemp.toString)) + + test("STACK-1: the fresh data dir is a DETERMINISTIC path under $RUNNER_TEMP") { + val runnerTemp = Files.createTempDirectory("stack-1-runner-temp") + try + // Deterministic: asking twice gives the same answer. This is the whole property - `mktemp -d` + // answered differently every time, which is what made the tree unfindable afterwards. + val first = freshDataDir(runnerTemp) + val second = freshDataDir(runnerTemp) + assert(first == second, s"the path is not stable across invocations: $first then $second") + + // ... and it lives under the runner-scoped temp dir it was given, not somewhere of its own + // choosing. A path that ignored RUNNER_TEMP would be stable AND still outlive the job. + assert( + first.startsWith(runnerTemp.toString + "/"), + s"$first is not under the RUNNER_TEMP it was given ($runnerTemp)" + ) + + // A different runner temp really moves it - so the value is read, not merely mentioned. + val elsewhere = Files.createTempDirectory("stack-1-other-runner-temp") + try assert(freshDataDir(elsewhere) != first, s"the path ignores RUNNER_TEMP: $first for both $runnerTemp and $elsewhere") + finally Files.deleteIfExists(elsewhere) + finally Files.deleteIfExists(runnerTemp) + } + + test("STACK-2: `clean` removes the data tree, and succeeds when there is nothing to remove") { + val runnerTemp = Files.createTempDirectory("stack-2-runner-temp") + try + val dir = Paths.get(freshDataDir(runnerTemp)) + // Seed something shaped like what run-dekaf.sh puts there - nested, non-empty, so a `rmdir` + // or a single-file delete would not be enough. + Files.createDirectories(dir.resolve("library")) + Files.createDirectories(dir.resolve("js/dist")) + Files.writeString(dir.resolve("js/dist/libs.js"), "// seeded by STACK-2") + assert(Files.isDirectory(dir), s"the fixture did not create $dir") + + val (code, out) = run("fresh-data-dir.sh", Seq("clean"), Map("RUNNER_TEMP" -> runnerTemp.toString)) + assert(code == 0, s"clean exited $code: $out") + assert(!Files.exists(dir), s"$dir survived the clean: $out") + + // Teardown runs with `if: always()`, including on runs that never created the tree, so a + // second clean must not fail the job. + val (againCode, againOut) = run("fresh-data-dir.sh", Seq("clean"), Map("RUNNER_TEMP" -> runnerTemp.toString)) + assert(againCode == 0, s"a second clean exited $againCode: $againOut") + finally Files.deleteIfExists(runnerTemp) + } + + /** Recursive rm for this spec's fixtures - the trees it seeds are nested, and one half of STACK-4 + * is deliberately NOT removed by the script under test. */ + private def deleteTree(dir: Path): Unit = + if Files.exists(dir) then + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).iterator().asScala.foreach(Files.deleteIfExists) + + test("STACK-4: each stack gets its OWN tree, and tearing one down leaves the other's live data alone") { + // The defect this pins: the path used to be one constant name per machine, while run-dekaf.sh + // `rm -rf`s it at startup and stack-down.sh `rm -rf`s it at teardown. Two stacks side by side - + // which is exactly how a second Dekaf is run against the same box - therefore destroyed each + // other's LIVE data dir, and nothing in STACK-1/2 could notice: determinism and removal are both + // still true of a path that every stack shares. + val runnerTemp = Files.createTempDirectory("stack-4-runner-temp") + try + val a = stackEnv(runnerTemp, dekafPort = "8090", container = "dekaf-e2e-pulsar") + // Differs from `a` only by PORT: two Dekafs against ONE Pulsar container is a real shape (this + // repo runs one on :8090 and one on :8091), and it is the case a container-only identity misses. + val bSamePulsar = stackEnv(runnerTemp, dekafPort = "8091", container = "dekaf-e2e-pulsar") + // ... and differs only by CONTAINER, the case a port-only identity misses. + val cSamePort = stackEnv(runnerTemp, dekafPort = "8090", container = "dekaf-e2e-pulsar-2") + + val stacks = List("a" -> a, "b(same pulsar, other port)" -> bSamePulsar, "c(same port, other pulsar)" -> cSamePort) + val paths = stacks.map((name, env) => name -> freshDataDirWith(env)) + assert( + paths.map(_._2).distinct.size == stacks.size, + s"two distinct stacks resolved to the SAME data dir, so one would delete the other's live data: $paths" + ) + // Still deterministic PER STACK - asking twice must agree, or teardown could not find the tree + // startup made, which is the whole reason this script exists (STACK-1 checks the same property + // for one stack; an identity built from a timestamp or a PID would pass that and fail here). + stacks.zip(paths).foreach { case ((name, env), (_, path)) => + val again = freshDataDirWith(env) + assert(again == path, s"stack $name's path is not stable across invocations: $path then $again") + } + + // Now the real proof: two live trees, tear ONE down, and the other must survive byte for byte. + val dirA = Paths.get(paths.head._2) + val dirB = Paths.get(paths(1)._2) + def seed(dir: Path, marker: String): Unit = + Files.createDirectories(dir.resolve("library")) + Files.writeString(dir.resolve("library/item.json"), marker) + seed(dirA, "stack A's library item") + seed(dirB, "stack B's library item") + + val (code, out) = run("fresh-data-dir.sh", Seq("clean"), a) + assert(code == 0, s"clean exited $code: $out") + assert(!Files.exists(dirA), s"stack A's own tree survived its own teardown: $out") + assert(Files.isRegularFile(dirB.resolve("library/item.json")), s"tearing stack A down deleted stack B's data dir ($dirB): $out") + assert( + Files.readString(dirB.resolve("library/item.json")) == "stack B's library item", + s"tearing stack A down rewrote stack B's live data: ${Files.readString(dirB.resolve("library/item.json"))}" + ) + // The one hazard per-stack paths introduce is the opposite of the old one: a teardown run + // without its stack's variables now LEAKS a tree rather than destroying a live one. `clean` + // has to name what it left, or that leak is as invisible as the `mktemp -d` one was. + assert(out.contains(dirB.toString), s"clean did not report the tree it left behind ($dirB):\n$out") + + // Symmetric: B's own teardown removes B. (A one-way property would be satisfied by a script + // that simply never removed anything but its first argument's tree.) + val (bCode, bOut) = run("fresh-data-dir.sh", Seq("clean"), bSamePulsar) + assert(bCode == 0, s"clean exited $bCode: $bOut") + assert(!Files.exists(dirB), s"stack B's own tree survived its own teardown: $bOut") + finally deleteTree(runnerTemp) + } + + test("STACK-3: run-dekaf.sh and stack-down.sh both go through the shared path") { + // STATIC on purpose. The behaviour above is executed; this pins the WIRING, which cannot be: + // `run-dekaf.sh` installs npm dependencies, builds the UI and ends in `exec sbt run`, and + // `stack-down.sh` deletes the Pulsar container the rest of this suite is running against. The + // check is here rather than nowhere because the deterministic path is worthless if either end + // stops using it - and each of them would still pass its own tests. + def source(name: String): String = Files.readString(scripts.resolve(name)) + + val runDekaf = source("run-dekaf.sh") + assert( + runDekaf.contains("""fresh_data="$("$here/fresh-data-dir.sh" path)""""), + "run-dekaf.sh no longer takes its DEKAF_FRESH_DATA directory from fresh-data-dir.sh" + ) + // Comments are excluded deliberately: the script EXPLAINS why it is no longer on mktemp, and a + // whole-file search would match that sentence and never be able to fail for the real reason. + val runDekafCode = runDekaf.linesIterator.filterNot(_.trim.startsWith("#")).mkString("\n") + assert( + !runDekafCode.contains("mktemp"), + "run-dekaf.sh is back on mktemp - the tree becomes unfindable and leaks on every CI run" + ) + assert( + runDekaf.contains("""export DEKAF_DATA_DIR="$fresh_data""""), + "run-dekaf.sh no longer points DEKAF_DATA_DIR at that directory" + ) + assert( + source("stack-down.sh").contains("""fresh-data-dir.sh" clean"""), + "stack-down.sh no longer removes the fresh data dir - CI's teardown step is the only thing that can" + ) + + // Cheap correctness gate on all three: an edit that broke the syntax would otherwise only + // surface on CI, where these run once each and nothing else exercises them. + Seq("fresh-data-dir.sh", "run-dekaf.sh", "stack-down.sh").foreach { name => + val pb = new ProcessBuilder(Seq("bash", "-n", scripts.resolve(name).toString).asJava) + pb.redirectErrorStream(true) + val process = pb.start() + val out = new String(process.getInputStream.readAllBytes(), "UTF-8") + assert(process.waitFor() == 0, s"$name does not parse: $out") + } + } diff --git a/e2e/src/test/scala/harness/SuiteFactsSpec.scala b/e2e/src/test/scala/harness/SuiteFactsSpec.scala new file mode 100644 index 000000000..0bac68f03 --- /dev/null +++ b/e2e/src/test/scala/harness/SuiteFactsSpec.scala @@ -0,0 +1,162 @@ +package harness + +import org.scalatest.funsuite.AnyFunSuite + +import java.nio.file.{Files, Path, Paths} +import scala.jdk.CollectionConverters.* + +/** The README's countable claims about this suite, checked against the suite. + * + * `e2e/README.md` is the catalog, and several of its statements are facts about the source rather + * than prose: how many tests are `ignore`d, how many are `pending`, how many carry the `KnownBug` + * tag, how many `assume(...)` (and where), and how many tags `build.sbt` excludes from the green + * lane. Every one of those is a way coverage can be dropped from `sbt test` WITHOUT the run going + * red, so every one is pinned here against a machine-readable marker the README carries: + * + * {{{ + * + * }}} + * + * The point is not to correct a sentence once but to make the numbers derivable, so the next edit + * that adds a lane has to say so in the README or fail here. The lanes and why each hides coverage: + * + * - `ignore(...)` - a test that never runs and reports neither pass nor fail; + * - `pending` / `pendingUntilFixed` - reported yellow, i.e. neither passed nor failed; + * - `test("name", KnownBug)` (or `taggedAs KnownBug`) - excluded from `sbt test` by build.sbt. + * `known-bug=0` is also the standing rule that the bug lane stays EMPTY: the tag exists for a bug + * that is genuinely open, and tagging a red test to get a run green is the misuse it invites; + * - `assume(...)` - RUNTIME-cancels the test when its predicate is false, so on the wrong stack it + * just vanishes from the run, neither failing nor reported as ignored. Legitimate for the two + * topic-policy specs, which adapt to the broker's `topicLevelPoliciesEnabled` and therefore + * always cancel one branch - but ONLY there, which is why both the count and the location are + * pinned; + * - a second `-l ` exclusion in build.sbt - drops a whole tag from `sbt test` the same way + * KnownBug is dropped, without any test looking ignored or tagged. + * + * This spec's OWN source is excluded from the scan: it holds every pattern below as a string + * literal (e.g. the bare word `pending`), so scanning it would count the patterns themselves. It + * declares no lane of its own, so nothing is lost by skipping it. + */ +class SuiteFactsSpec extends AnyFunSuite: + + private val e2eRoot: Path = + Seq(Paths.get("."), Paths.get("e2e")) + .find(p => Files.isRegularFile(p.resolve("README.md")) && Files.isDirectory(p.resolve("src/test/scala"))) + .getOrElse(fail(s"cannot locate the e2e project from ${Paths.get("").toAbsolutePath}")) + + /** Drop comment lines from a source file: this spec's prose - and the README-quoting comments in + * the specs - would otherwise count as occurrences and a check could never fail for the real + * reason. */ + private def stripComments(source: String): String = + source.linesIterator + .filterNot(line => { val t = line.trim; t.startsWith("//") || t.startsWith("*") || t.startsWith("/*") }) + .mkString("\n") + + /** Every spec source except THIS one, with comment lines dropped. See the class doc for why this + * file is excluded. */ + private lazy val specs: List[(Path, String)] = + val root = e2eRoot.resolve("src/test/scala") + Files.walk(root).iterator().asScala + .filter(p => Files.isRegularFile(p) && p.toString.endsWith(".scala")) + .filterNot(_.getFileName.toString == "SuiteFactsSpec.scala") + .toList.sortBy(_.toString) + .map(p => p -> stripComments(Files.readString(p))) + + private lazy val readme: String = Files.readString(e2eRoot.resolve("README.md")) + + /** The declared counts, parsed out of the README's marker. */ + private lazy val declared: Map[String, Int] = + val body = """""".r.findFirstMatchIn(readme).map(_.group(1)).getOrElse( + fail("e2e/README.md has no `` marker - see §3") + ) + """(\S+)=(\d+)""".r.findAllMatchIn(body).map(m => m.group(1) -> m.group(2).toInt).toMap + + private def countIn(pattern: String): List[(Path, Int)] = + val re = pattern.r + specs.map((path, code) => path -> re.findAllMatchIn(code).size).filter(_._2 > 0) + + private def total(counts: List[(Path, Int)]): Int = counts.map(_._2).sum + + private def declaredEquals(key: String, counts: List[(Path, Int)], noun: String): Unit = + assert( + declared.get(key).contains(total(counts)), + s"e2e/README.md declares $key=${declared.get(key)} but the suite has ${total(counts)} $noun" + + s"${if counts.isEmpty then "" else s" in ${counts.map(_._1)}"}. Update BOTH the marker in §3 and " + + "the prose in §3/§6 - a change in this count is a change in what coverage a reader can rely on." + ) + + test("FACTS-1: the README's `ignored` count is the number of ignored tests") { + // Bare `ignore(` only: a ScalaTest ignored test is called as a statement, so anything reached + // through a receiver (`x.ignore(...)`) is a different method and not what the README counts. + declaredEquals("ignored", countIn("""(^|[^A-Za-z0-9_.`"])ignore\s*\("""), "ignored test(s)") + } + + test("FACTS-2: the README's `known-bug` count is the number of tagged tests, and every tag names an OPEN bug") { + // Both ways ScalaTest can carry a tag: the FunSuite tag argument this suite uses, + // `test("name", KnownBug) { ... }`, and the `taggedAs` form the other styles use - the latter + // with OR without parentheses (`taggedAs KnownBug` is a legal infix call), because either would + // take a test out of the green lane. Matching the usage rather than the bare identifier keeps an + // `import harness.KnownBug` from counting as a test. + val counts = + countIn("""(?s)\btest\s*\(\s*"(?:[^"\\]|\\.)*"\s*,\s*KnownBug""") ++ countIn("""taggedAs\s*\(?\s*KnownBug""") + declaredEquals("known-bug", counts, "tagged test(s)") + // The census is pinned BY NAME, not merely by number: a tag is for a bug that is genuinely + // OPEN, never a way to take a red test out of the green run - so a stray tag anywhere fails + // here, and fixing a listed bug (untagging its regression) shrinks this list in the same + // change. Currently EMPTY: the two 2026-08-09 replay-redesign entries (CsDeliveryModesSpec + // CS-DM-R3B, the live-edge boundary clobber; CsFlowControlSpec CS-FC-5, whose 2-of-4 red + // turned out to be an oracle defect reading a reconnect-reset dispatch counter) were fixed + // and untagged the same day, and both regressions run green in the normal lane. + val openBugRegressions = Map.empty[String, Int] + val census = counts.map((path, n) => path.getFileName.toString -> n).toMap + assert( + census == openBugRegressions, + s"the KnownBug census moved: expected exactly $openBugRegressions, found $census. " + + s"A NEW tag needs a genuinely open bug recorded here and in the README; a FIXED bug means untagging " + + s"the regression and shrinking this list (and the README marker) in the same change." + ) + } + + test("FACTS-3: nothing in the suite is `pending`") { + // A `pending` (or `pendingUntilFixed`) test is reported neither passed nor failed - yellow, not + // red - so it removes coverage as surely as an ignored one while a `sbt test` run still goes + // green. The README owns that count. + declaredEquals("pending", countIn("""\bpending(?:UntilFixed)?\b"""), "`pending` marker(s)") + } + + /** The only specs allowed to carry an `assume(...)`. Both adapt to whether the broker has + * `topicLevelPoliciesEnabled`, so exactly one branch runs per stack and the other runtime-cancels + * - legitimate config-gated coverage, but cancellation all the same, so it lives here and nowhere + * else. */ + private val assumeAllowedFiles = Set("TopicPolicySpec.scala", "TopicPolicyBreadthSpec.scala") + + test("FACTS-4: `assume(...)` cancellation is confined to the known policy specs and counted") { + // Bare `assume(` (statement form), guarded against a receiver call the same way `ignore` is. + val counts = countIn("""(^|[^A-Za-z0-9_.`"])assume\s*\(""") + val stray = counts.filterNot((path, _) => assumeAllowedFiles.contains(path.getFileName.toString)) + assert( + stray.isEmpty, + s"`assume(...)` outside the known policy specs $assumeAllowedFiles: ${stray.map(_._1)}. assume " + + "RUNTIME-cancels a test - on the wrong broker config it vanishes from the run, neither failing " + + "nor reported as ignored. If this is deliberate it belongs beside a documented config axis (like " + + "topicLevelPoliciesEnabled), not as a way to quiet a red test." + ) + declaredEquals("assume", counts, "`assume(...)` call(s)") + } + + test("FACTS-5: build.sbt excludes exactly the KnownBug tag from the green lane, nothing else") { + // A `-l ` in build.sbt drops that whole tag from `sbt test`. Exactly one is expected + // (KnownBug); a second is a way to remove tests from the green run without ignoring or tagging + // them - the very lane this census closes. + val code = stripComments(Files.readString(e2eRoot.resolve("build.sbt"))) + val excluded = "\"-l\"\\s*,\\s*\"([^\"]+)\"".r.findAllMatchIn(code).map(_.group(1)).toList + assert( + declared.get("excluded-tags").contains(excluded.size), + s"e2e/README.md declares excluded-tags=${declared.get("excluded-tags")} but build.sbt has " + + s"${excluded.size} `-l ` exclusion(s): $excluded. Update the marker in §3." + ) + assert( + excluded.toSet == Set("KnownBug"), + s"build.sbt's green lane excludes $excluded; only KnownBug may be excluded from `sbt test`." + ) + } diff --git a/e2e/src/test/scala/routes/ResilienceSpec.scala b/e2e/src/test/scala/routes/ResilienceSpec.scala index 121d2fde9..4534c47f1 100644 --- a/e2e/src/test/scala/routes/ResilienceSpec.scala +++ b/e2e/src/test/scala/routes/ResilienceSpec.scala @@ -1,14 +1,17 @@ package routes import harness.DekafSuite +import harness.Eventually.eventually import features.consumersession.ConsumerSessionPage +import features.library.LibrarySidebar import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.assertions.LocatorAssertions import com.microsoft.playwright.options.AriaRole import com.microsoft.playwright.Page.GetByRoleOptions -/** RES-1/2/3 - negative / resilience: bogus routes 404, a bad saved-session id degrades gracefully, - * and the unguarded non-persistent details route is documented. */ +/** RES-1/2/3 - negative / resilience: bogus routes 404, a saved-session id that is missing or + * points at a malformed persisted config degrades gracefully, and the unguarded non-persistent + * details route is documented. */ class ResilienceSpec extends DekafSuite: private def goHome = page.getByRole(AriaRole.BUTTON, new GetByRoleOptions().setName("Go Home")) @@ -24,7 +27,7 @@ class ResilienceSpec extends DekafSuite: assertThat(goHome).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(15000)) } - test("RES-2: opening a consumer session with a bad saved-session id degrades gracefully") { + test("RES-2: opening a consumer session with a MISSING saved-session id degrades gracefully") { val (t, ns, topic) = fixtures.freshTopicParts() // A non-existent managed session id must not crash the page - it falls back to a fresh session. page.navigate(s"/tenants/$t/namespaces/$ns/topics/persistent/$topic/consumer-session?id=does-not-exist-xyz") @@ -33,6 +36,49 @@ class ResilienceSpec extends DekafSuite: assertThat(cs.playButton).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(15000)) } + // A `?id=` pointing at a persisted library item whose stored content is NOT a consumer-session + // config (here a message-filter, saved through the app's own Library) used to take the WHOLE app + // down: the editor reached for `spec.targets` on the foreign spec and threw "TypeError: Cannot + // read properties of undefined (reading 'map')" during render, and with no error boundary React + // unmounted everything - document.body rendered EMPTY. The session configuration editor now checks + // the stored shape and reports it, and the session subtree sits behind an error boundary, so this + // degrades like the missing-id case above (untagged from KnownBug per e2e/README.md §6). + test("RES-2: opening a consumer session whose persisted config is MALFORMED degrades gracefully") { + val (t, ns, topic) = fixtures.freshTopicParts() + val overviewUrl = s"/tenants/$t/namespaces/$ns/topics/persistent/$topic/overview" + page.navigate(overviewUrl) + + // Arrange through the app's own persistence path: save a library item whose content is + // structurally invalid FOR THIS ROUTE, then read its real id out of the app's item editor. + val lib = LibrarySidebar(page) + lib.openLibraryTab() + lib.createItemNamed("message-filter", "malformed-session") + page.navigate(overviewUrl) // reload so the library search re-fetches (see LIB-8/12) + lib.openLibraryTab() + lib.browseType("message-filter").editItem("malformed-session") + val itemId = eventually() { + val editorText = page.getByTestId("lib-save-dialog").innerText() + // The editor renders `ID: `, and a non-breaking space is not `\s` - skip any + // non-hex separator instead. + raw"ID:[^0-9a-fA-F]*([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})".r + .findFirstMatchIn(editorText) + .map(_.group(1)) + .getOrElse(throw new AssertionError(s"no item id in the library item editor: $editorText")) + } + + page.navigate(s"/tenants/$t/namespaces/$ns/topics/persistent/$topic/consumer-session?id=$itemId") + + // The app must survive a malformed persisted config: its chrome still renders (the document is + // not blank) ... + assertThat(page.getByTestId("breadcrumbs")).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(15000)) + // ... and the route shows either a usable session or a visible error - never an empty shell. + assertThat( + ConsumerSessionPage(page).playButton + .or(page.getByText(java.util.regex.Pattern.compile("Unable to fetch item", java.util.regex.Pattern.CASE_INSENSITIVE))) + .first() + ).isVisible(new LocatorAssertions.IsVisibleOptions().setTimeout(15000)) + } + test("RES-3: the unguarded non-persistent /details route renders without a blank crash") { val t = fixtures.createTenant() val ns = fixtures.createNamespace(t) diff --git a/e2e/src/test/scala/routes/SubscriptionActionSpec.scala b/e2e/src/test/scala/routes/SubscriptionActionSpec.scala index 9c1fa5f39..efe8b4144 100644 --- a/e2e/src/test/scala/routes/SubscriptionActionSpec.scala +++ b/e2e/src/test/scala/routes/SubscriptionActionSpec.scala @@ -1,6 +1,7 @@ package routes import harness.DekafSuite +import harness.Eventually.eventually import ui.ConfirmationDialog import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.options.SelectOption @@ -60,23 +61,31 @@ class SubscriptionActionSpec extends DekafSuite: assert(awaitBacklog(fqn, sub, _ == 0L) == 0L) } - test("SUB-3: expire messages older than a duration runs without error (non-partitioned)") { + test("SUB-3: expire messages older than a duration clears the backlog (non-partitioned)") { val (t, ns, topic) = fixtures.freshTopicParts() val fqn = s"persistent://$t/$ns/$topic" produceCapturing(fqn, 3) + val producedAt = System.currentTimeMillis() val sub = fixtures.unique("sub") admin.topics().createSubscription(fqn, sub, MessageId.earliest) + assert(awaitBacklog(fqn, sub, _ == 3L) == 3L) // a REAL backlog to expire page.navigate(overviewUrl(t, ns, topic, sub)) page.getByTestId("expire-subscription-messages-button").click() page.getByTestId("expire-target-select").selectOption(new SelectOption().setValue("expire-time-in-seconds")) - page.getByTestId("expire-duration").locator("input").first().fill("5") // any >0 enables Confirm + page.getByTestId("expire-duration").locator("input").first().fill("1") // seconds (DurationInput default unit) + // Expiry compares publish time against `now - 1s`, so "older than 1s" is a real precondition of + // the assertion below - poll for it (page load usually covers it) instead of sleeping blind. + eventually(timeoutMs = 10000, intervalMs = 200) { + assert(System.currentTimeMillis() - producedAt > 2000) + } ConfirmationDialog(page).confirm(guard = Some("CONFIRM")) - // Oracle for the time-based path: the action runs without error. - // (Deterministic backlog effect is asserted on the by-ID leg above - see NOTES.) assertThat(page.getByText("Messages were successfully expired")).isVisible() - assert(admin.topics().getSubscriptions(fqn).asScala.contains(sub)) // admin cross-check: sub intact + // The oracle - a toast only proves a request was fired; this proves the backlog actually drained + // while the subscription itself survived. + assert(awaitBacklog(fqn, sub, _ == 0L) == 0L) + assert(admin.topics().getSubscriptions(fqn).asScala.contains(sub)) } test("SUB-3: expire by message ID is disabled for a partitioned topic") { diff --git a/e2e/src/test/scala/routes/TableSpec.scala b/e2e/src/test/scala/routes/TableSpec.scala index a6b5ba9fa..0acd6d854 100644 --- a/e2e/src/test/scala/routes/TableSpec.scala +++ b/e2e/src/test/scala/routes/TableSpec.scala @@ -118,3 +118,33 @@ class TableSpec extends DekafSuite: assert(math.abs(domAfterReload - domAfterResize) <= 3, s"restored column rendered at ${domAfterReload}px, expected ~${domAfterResize}px (persisted width not re-applied)") } + + test("NAV-6: a column dragged onto another lands BEFORE it, persists, and survives a reload") { + openTenantsTable() + + def columnKeys(): List[String] = + page.locator("[data-testid='table-th']").all().asScala.toList.map(_.getAttribute("data-column-key")) + + val before = columnKeys() + assert(before.indexOf("allowedClusters") > before.indexOf("namespacesCount"), s"unexpected default order: $before") + + // Drag 'allowedClusters' onto 'namespacesCount': it must land immediately before it. The + // sticky first column (tenantName) is not draggable and must stay first throughout. + page.locator("[data-testid='table-th'][data-column-key='allowedClusters']") + .dragTo(page.locator("[data-testid='table-th'][data-column-key='namespacesCount']")) + + val after = columnKeys() + assert(after.head == "tenantName", s"the sticky column must stay first, got $after") + assert( + after.indexOf("allowedClusters") == after.indexOf("namespacesCount") - 1, + s"dragged column should sit immediately before its target, got $after" + ) + + // Persisted like the widths: the order is in localStorage and survives a reload. + val stored = page.evaluate("() => localStorage.getItem('table:tenants-table:column-order') || ''").toString + assert(stored.contains("allowedClusters"), s"expected a persisted column order, got '$stored'") + page.reload() + assertThat(page.getByTestId("table-counter")).isVisible(vis(15000)) + val reloaded = columnKeys() + assert(reloaded == after, s"the order must survive a reload: before=$after after=$reloaded") + } diff --git a/e2e/src/test/scala/routes/TopicActionsSpec.scala b/e2e/src/test/scala/routes/TopicActionsSpec.scala index 6fdd6a037..ba7af6870 100644 --- a/e2e/src/test/scala/routes/TopicActionsSpec.scala +++ b/e2e/src/test/scala/routes/TopicActionsSpec.scala @@ -1,12 +1,13 @@ package routes import harness.DekafSuite +import harness.Eventually import ui.ConfirmationDialog import com.microsoft.playwright.Page.GetByRoleOptions import com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat import com.microsoft.playwright.assertions.LocatorAssertions import com.microsoft.playwright.options.AriaRole -import org.apache.pulsar.client.api.Schema +import org.apache.pulsar.client.api.{MessageId, Schema} import java.util.regex.Pattern import scala.jdk.CollectionConverters.* @@ -25,6 +26,26 @@ class TopicActionsSpec extends DekafSuite: ok } + /** Produce `n` NON-batched messages (1 message == 1 entry => deterministic backlog/entry counts). */ + private def produce(fqn: String, n: Int): Unit = { + val p = client.newProducer(Schema.STRING).topic(fqn).enableBatching(false).create() + try (1 to n).foreach(i => p.send(s"msg-$i")) + finally p.close() + } + + private def backlogOf(fqn: String, sub: String): Long = { + val subs = admin.topics().getStats(fqn).getSubscriptions + if subs.containsKey(sub) then subs.get(sub).getMsgBacklog else -1L + } + + /** Expiry compares each message's publish time against `now - expireTimeInSeconds`, so + * "the messages are older than the threshold" is a real precondition of the expire assertion. + * Poll for it (page load usually covers it already) instead of sleeping blind. */ + private def awaitOlderThan(producedAt: Long, ageMs: Long): Unit = + Eventually.eventually(timeoutMs = 10000, intervalMs = 200) { + assert(System.currentTimeMillis() - producedAt > ageMs) + } + private def partitionedTopic(count: Int): (String, String, String, String) = { val t = fixtures.createTenant() val ns = fixtures.createNamespace(t) @@ -77,31 +98,67 @@ class TopicActionsSpec extends DekafSuite: }) } - test("TOP-3: expire messages on all subscriptions") { + test("TOP-3: expire messages on all subscriptions clears every subscription's backlog") { val (t, ns, topic) = fixtures.freshTopicParts() + val fqn = s"persistent://$t/$ns/$topic" + + // Arrange a REAL backlog on two subscriptions - "all subscriptions" is only proven by more + // than one - then act through the UI and poll the admin oracle for the state change. + val subA = fixtures.unique("suba") + val subB = fixtures.unique("subb") + admin.topics().createSubscription(fqn, subA, MessageId.earliest) + admin.topics().createSubscription(fqn, subB, MessageId.earliest) + produce(fqn, 3) + val producedAt = System.currentTimeMillis() + Eventually.eventually() { + assert(backlogOf(fqn, subA) == 3L) + assert(backlogOf(fqn, subB) == 3L) + } + page.navigate(s"/tenants/$t/namespaces/$ns/topics/persistent/$topic/overview") page.getByTestId("expire-topic-messages-button").click() - // Confirm stays disabled until duration > 0 (ExpireAllSubscriptions.tsx:93). + // Confirm stays disabled until duration > 0 (ExpireAllSubscriptions.tsx:93). DurationInput's + // default unit is seconds, so "1" == expireTimeInSeconds 1. page.getByRole(AriaRole.SPINBUTTON).fill("1") + awaitOlderThan(producedAt, 2000) // No force checkbox on this dialog - force must stay false. ConfirmationDialog(page).confirm(guard = Some("CONFIRM")) - // Empty-backlog expire is a server no-op → assert the success toast + no error toast. assertThat(page.getByText("Messages were successfully expired")).isVisible(visible(15000)) + // The oracle - a toast only proves a request was fired; this proves the backlog actually drained. + Eventually.eventually() { + assert(backlogOf(fqn, subA) == 0L) + assert(backlogOf(fqn, subB) == 0L) + } } - test("TOP-4: unload a topic") { + test("TOP-4: unload a topic reloads its managed ledger (data preserved)") { val (t, ns, topic) = fixtures.freshTopicParts() val fqn = s"persistent://$t/$ns/$topic" + + // Arrange state that only survives ONE load: entriesAddedCounter lives on the ManagedLedger + // INSTANCE, so it counts this load's 3 writes and restarts at 0 once the topic is closed and + // re-opened (the next admin read re-loads it). Re-opening also appends a fresh ledger. + produce(fqn, 3) + Eventually.eventually() { assert(admin.topics().getInternalStats(fqn).entriesAddedCounter == 3L) } + val ledgersBefore = admin.topics().getInternalStats(fqn).ledgers.size() + page.navigate(s"/tenants/$t/namespaces/$ns/topics/persistent/$topic/overview") page.getByTestId("topic-page-unload-button").click() ConfirmationDialog(page).confirm(guard = Some(fqn)) // guard = topic FQN, no force - // Unload is transient: assert success toast + topic still present. assertThat(page.getByText(Pattern.compile("has been successfully unloaded"))).isVisible(visible(15000)) + // The oracle: the managed ledger really was closed and re-opened. + Eventually.eventually() { + val stats = admin.topics().getInternalStats(fqn) + assert(stats.entriesAddedCounter == 0L) // per-load counter restarted + assert(stats.ledgers.size() > ledgersBefore) // re-open created a new ledger + } + // Unload, not delete: the topic and its 3 entries survive. + assert(admin.topics().getInternalStats(fqn).numberOfEntries == 3L) assert(admin.topics().getList(s"$t/$ns").asScala.exists(_.contains(topic))) } diff --git a/flake.lock b/flake.lock index 636a4d734..ac4035583 100644 --- a/flake.lock +++ b/flake.lock @@ -49,6 +49,22 @@ "type": "indirect" } }, + "nixpkgs-buf": { + "locked": { + "lastModified": 1784796856, + "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e2587caef70cea85dd97d7daab492899902dbf5d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e2587caef70cea85dd97d7daab492899902dbf5d", + "type": "github" + } + }, "nixpkgs-playwright": { "locked": { "lastModified": 1735160951, @@ -70,6 +86,7 @@ "flake-compat": "flake-compat", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", + "nixpkgs-buf": "nixpkgs-buf", "nixpkgs-playwright": "nixpkgs-playwright" } }, diff --git a/flake.nix b/flake.nix index 0205b84d6..2077b3f4a 100644 --- a/flake.nix +++ b/flake.nix @@ -10,6 +10,14 @@ nixpkgs-playwright = { url = "github:NixOS/nixpkgs/c792c60b8a97daa7efe41a6e4954497ae410e0c1"; }; + # Codegen: the buf in the main nixpkgs lock is 1.30.0, whose darwin binary has no + # LC_UUID load command and so cannot launch at all on macOS 15+ ("dyld: missing + # LC_UUID load command"), breaking `cd proto && make build` on every recent Mac. + # Pinned separately rather than bumping the main lock, which would churn the whole + # toolchain (JVM, sbt, node, envoy) for a single tool. + nixpkgs-buf = { + url = "github:NixOS/nixpkgs/e2587caef70cea85dd97d7daab492899902dbf5d"; + }; flake-compat = { url = "github:edolstra/flake-compat"; flake = false; @@ -22,6 +30,7 @@ { self , nixpkgs , nixpkgs-playwright + , nixpkgs-buf , flake-compat , flake-utils , @@ -59,6 +68,9 @@ playwrightBrowsers = (import nixpkgs-playwright { inherit system; }).playwright-driver.browsers; + # See the nixpkgs-buf input: the main lock's buf cannot launch on modern macOS. + buf = (import nixpkgs-buf { inherit system; }).buf; + runtimeLibraryPath = lib.makeLibraryPath ([ pkgs.zlib ]); pulsar-ui-dev = pkgs.mkShell { @@ -90,7 +102,7 @@ pkgs.maven pkgs.protobuf3_20 - pkgs.buf + buf protoc-gen-grpc-web protoc-gen-scala diff --git a/proto/Makefile b/proto/Makefile index 7a525b3c3..72368b6fe 100644 --- a/proto/Makefile +++ b/proto/Makefile @@ -6,7 +6,8 @@ clean: rm -rf ./gen rm -rf ../ui/grpc-web rm -rf ../server/src/main/scala/pb - rm -rf ../demoapp/src/main/scala/pb + rm -rf ../demoapp/src/main/java/com/tools/teal/pulsar/ui + rm -rf ../demoapp/src/main/java/com/google .PHONY: build build: diff --git a/proto/buf.lock b/proto/buf.lock deleted file mode 100644 index c91b5810c..000000000 --- a/proto/buf.lock +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by buf. DO NOT EDIT. -version: v1 diff --git a/proto/buf.yaml b/proto/buf.yaml deleted file mode 100644 index 1a5194568..000000000 --- a/proto/buf.yaml +++ /dev/null @@ -1,7 +0,0 @@ -version: v1 -breaking: - use: - - FILE -lint: - use: - - DEFAULT diff --git a/proto/proto/tools/teal/pulsar/ui/api/v1/consumer.proto b/proto/proto/tools/teal/pulsar/ui/api/v1/consumer.proto index 528be6a7a..3d71ed07b 100644 --- a/proto/proto/tools/teal/pulsar/ui/api/v1/consumer.proto +++ b/proto/proto/tools/teal/pulsar/ui/api/v1/consumer.proto @@ -33,6 +33,20 @@ message Message { int64 num_message_processed = 59; int64 num_message_sent = 60; + // ORDERED delivery (Guaranteed or Best effort): this message's order key is LOWER than the + // highest key already emitted by this session - it is on screen out of order, loudly, never + // silently. Under GUARANTEED that is possible only across a resume seam (the boundary + // extension lets a delta message carry an older producer-stamped key than something an earlier + // chunk already showed) or when the source log itself stores an inversion; under BEST EFFORT + // it means the message arrived after its reorder window and its peers were already shown + // (renamed from replay_seam_violation on 2026-08-11, same tag, when Best effort's late + // deliveries started carrying the flag too). The key compared is the session's SELECTED order + // key (publish / broker publish / event time) against previously emitted keys of the same + // kind - never against wall clock, so ordinary old event times are not violations. Session + // counts: ConsumerStats.replay_seam_violations (Guaranteed) and the late-deliveries counter + // (Best effort). + bool delivered_out_of_order = 61; + // Evaluated message filter code can produce logs or errors. // We need to store them in the message to be able to show them in the UI. // Both, stdout and stderr are stored in the same field. @@ -119,6 +133,29 @@ message RelativeDateTime { bool is_rounded_to_unit_start = 3; } +// Start approximately this far through the entries a physical topic still retains. +// Each topic and each partition is positioned independently. Interior fractions +// leave floor(fraction * retained_entries) entries behind; batching therefore +// makes this an entry position, not an exact message-count position. +message ApproximateEntryPosition { + // 0.0 = earliest retained entry. 1.0 = MessageId.latest, past retained entries, + // so only messages published after the seek is applied are shown. Other values + // must be within [0.0, 1.0]. + double fraction = 1; +} + +// Start approximately this far between the session's observed first- and final-entry +// publish times. The boundary is pooled across EVERY selected physical topic and every +// topic seeks to the same instant, so a multi-topic session has ONE cutoff rather than +// one per topic - without that, 50% meant a different moment on each topic. +message ApproximatePublishTimePosition { + // 0.0 = earliest retained message. 1.0 = the greatest final-entry publish time + // observed across the session's selected topics; every message sharing that timestamp + // can be included. Other values must be within [0.0, 1.0]. Producer clock skew + // makes this boundary estimate approximate. + double fraction = 1; +} + message ConsumerSessionStartFrom { oneof start_from { EarliestMessage start_from_earliest_message = 1; @@ -128,6 +165,8 @@ message ConsumerSessionStartFrom { MessageId start_from_message_id = 3; DateTime start_from_date_time = 4; RelativeDateTime start_from_relative_date_time = 5; + ApproximateEntryPosition start_from_approximate_entry_position = 8; + ApproximatePublishTimePosition start_from_approximate_publish_time_position = 9; } } @@ -277,6 +316,64 @@ message ConsumerSessionTarget { ValueProjectionList value_projection_list = 4; } +// How a session that reads MORE THAN ONE delivery stream (several topics, or one partitioned +// topic) interleaves MESSAGE delivery across them. +enum MessageDeliveryOrder { + // Uses GUARANTEED, the product default (owner decision 2026-08-11, superseding the 2026-08-09 + // Best effort default, which itself superseded an earlier Guaranteed one - the plan file's + // decision log is the record). Zero is also proto3 absence, so every older client and every + // config saved before this field existed lands here: a session that never named an order + // replays recorded history exactly and auto-pauses when caught up. + // BEST_EFFORT_BY_PUBLISH_TIME is the explicit choice for live following; the caught-up + // announcement and the one-click switch (SetDeliveryOrder) are the ways between them. + MESSAGE_DELIVERY_ORDER_UNSPECIFIED = 0; + // Every stream delivers independently and messages interleave as they arrive. Fastest. + MESSAGE_DELIVERY_ORDER_AS_RECEIVED = 1; + // One merged stream ordered by the selected time, BEST EFFORT (bounded lateness). THE PRODUCT + // DEFAULT, and what UNSPECIFIED resolves to: delivery is held for a short reorder window so + // quieter streams get their say. Timestamps are stamped outside the session's control, so skew + // or a slow stream can still deliver late - late messages are delivered out of order, never + // dropped - and delivery lags by up to the window plus one sweep period (~0.5-0.75s in total) + // whenever some stream is quiet. + // + // THE NAME IS HISTORICAL. This mode predates delivery_order_key, so its constant still says + // "by publish time" while the timestamp it actually compares is whichever DeliveryOrderKey the + // session selected - publish time, broker publish time or event time. The constant and its + // value 2 are frozen: renaming or aliasing them would break every saved item and every older + // client for a cosmetic gain. Internally the mode is called plain "best effort" + // (Scala MessageDeliveryOrder.BestEffort, TS 'best-effort'). + MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME = 2; + // AN EXACT REPLAY (owner decision 2026-08-09). One merged stream ordered by the selected + // time: the session delivers everything recorded up to the moment Play was pressed (of what + // retention still holds - a trimmed message is gone for any reader), in strict key order, and + // then AUTO-PAUSES, announcing it on ConsumerStats.replay_caught_up. No live phase: a message + // recorded past the boundary is held for the NEXT chunk (its consumer paused) rather than + // delivered. Resume extends the boundary to now and replays the delta - itself immutable by + // then - exactly; ordering violations are impossible within a chunk, and across a resume seam + // they require producer clock skew and are delivered loudly flagged + // (Message.replay_seam_violation), never silently. An explicit choice, never what absence + // resolves to. Source append order is preserved, so timestamp inversions already stored within + // one source pass through and are counted. Failed sends retry in place. A stream whose + // recorded range retention trimmed away can still hold a replay (the wait is disclosed via + // delivery_order_waiting_streams); the live remedy remains the one-click switch to Best + // effort (SetDeliveryOrder). + MESSAGE_DELIVERY_ORDER_GUARANTEED = 3; +} + +// Which per-message timestamp the delivery order compares. This is not Pulsar's message +// `ordering_key` field. +enum DeliveryOrderKey { + DELIVERY_ORDER_KEY_UNSPECIFIED = 0; + // Added automatically by the producer. Always present. + DELIVERY_ORDER_KEY_PUBLISH_TIME = 1; + // Broker entry metadata recording when an entry arrived. Present only when the broker runs the + // AppendBrokerTimestampMetadataInterceptor and exposes entry metadata to clients. + DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME = 2; + // An optional timestamp set by the application. A message without one (0) uses publish time + // and is counted as a fallback. + DELIVERY_ORDER_KEY_EVENT_TIME = 3; +} + message ConsumerSessionConfig { ConsumerSessionStartFrom start_from = 1; repeated ConsumerSessionTarget targets = 2; @@ -285,6 +382,10 @@ message ConsumerSessionConfig { ColoringRuleChain coloring_rule_chain = 5; ValueProjectionList value_projection_list = 6; google.protobuf.Int64Value num_display_items = 7; + MessageDeliveryOrder message_delivery_order = 8; + // Absent (UNSPECIFIED) means publish time - the timestamp every ordered session used before this + // field existed. Meaningful only when message_delivery_order is not as-received. + DeliveryOrderKey delivery_order_key = 9; } message CreateConsumerRequest { @@ -543,9 +644,110 @@ message ResumeRequest { string consumer_name = 1; bool include_consumer_stats = 2; bool is_debug = 3; -} -message ConsumerStats {} + // Cap on how many messages per second this session DELIVERS to the client, applied from this + // resume onward. 0 means unlimited; negative is refused outright. + // + // PER REQUEST, NOT PER SESSION CONFIG, like the two flags above it, and deliberately so: the + // value belongs to the BROWSER doing the watching (it lives in localStorage), so it must never + // travel into a saved library item and follow the session to another person's screen. + // + // The limit shapes steady-state delivery only. Start-from positioning - the counted skip of a + // "skip first n" or the retained-history walk of a "latest n" - is never slowed by it: those + // messages were never going to be shown, and slowing the seek would only delay the first visible + // message. The count is per SESSION, not per partition, so the number means what it says + // regardless of how many partitions the selector matched. + int64 max_messages_per_second = 4; + + // Deliver AT MOST this many messages on this stream, then deliver nothing more until the next + // resume. 0 means no budget; negative is refused. + // + // The count is of messages LOADED - the ones that passed every filter and went on the wire, the + // number the toolbar's "loaded" counter shows - not of messages processed. A session whose + // filters drop most of what they read may well process hundreds to load ten; the budget lets it, + // and stops the STREAM at exactly ten. + // + // Enforced at the delivery drain, EXACTLY: the message that spends the last unit is the last one + // sent, whatever was mid-batch behind it stays queued - unacknowledged, undelivered - for the + // next resume. A client-side "pause after n" can only ever be approximate (a whole chunk lands + // before the client can react, and the first chunk under a rate limit is the full one-second + // burst); this is the server-side half that makes the number mean itself. + int64 max_messages_to_deliver = 5; +} + +// Progress of a start-from position that has to be resolved by counting messages +// rather than by seeking. Only NthMessageAfterEarliest needs this: skipping N +// messages exactly costs O(N) because Pulsar stores no message-ordinal index, so +// a large N takes real time and the UI must be able to say so. +message StartFromProgress { + int64 messages_skipped = 1; + int64 messages_to_skip = 2; + // True once the requested position has been reached and normal delivery began. + bool complete = 3; + // True once the position was resolved BEST-EFFORT rather than exactly: a stream the merge was + // waiting on stayed silent past the stall window and was abandoned (named below). The COUNT is + // still exact; WHICH messages were dropped may differ from the exact answer. Sticky for the + // session - a degradation that happened does not un-happen. + bool degraded = 4; + // The streams ("consumer@topic") the resolution gave up waiting for. + repeated string abandoned_streams = 5; +} + +message ConsumerStats { + // Absent unless the session's start-from is still being resolved. + StartFromProgress start_from_progress = 1; + // How many messages the delivery-order layer resolved out of the selected timestamp order so + // far, measured before the network hop. Timestamp ties are tie-breaks, never counted. In + // Guaranteed mode this counts disorder already stored within a source (and redeliveries); in + // Best effort it can also count messages arriving after the reorder window. Only meaningful when + // delivery_order_active is true. + int64 ordering_late_deliveries = 2; + // Whether an ordering layer (best-effort OR guaranteed) is ACTUALLY RUNNING for this session. + // False when a BEST-EFFORT configuration resolved to a single delivery stream - one log is + // already in order, so no layer is built and no latency is paid; the client must not claim + // otherwise. GUARANTEED builds its layer at ANY stream count since the exact-replay redesign + // (owner decision 2026-08-09): the replay boundary, the auto-pause and the caught-up signal + // live in that layer, and one stream needs them exactly as much as fifty. + bool delivery_order_active = 3; + // How many messages used publish time because the selected timestamp was + // absent on the message: broker publish time without the broker-side interceptor, or an + // event-time session meeting messages that carry none. The client surfaces the first + // occurrence as a notification with remediation. + int64 order_key_fallbacks = 4; + // Number of topic/partition streams a delivery-order merge has waited on for at least the + // stall-warning interval. Zero while the merge is making normal progress. + int32 delivery_order_waiting_streams = 5; + + // GUARANTEED (exact-replay) delivery: true once the session has delivered everything recorded + // up to the replay boundary (the moment Play or the last Resume was pressed, of what retention + // still held) and has AUTO-PAUSED itself. The server cannot flip the browser's play state - + // this signal is how the client learns; Resume extends the boundary to now and replays the + // delta. Cleared (absent) once a resume starts the next chunk. + bool replay_caught_up = 6; + + // The wall-clock instant the current replay boundary was captured at - "caught up to ". + // Meaningful whenever replay_caught_up is true. + int64 replay_boundary_at_ms = 7; + + // APPROXIMATELY how many newer ENTRIES exist past the replay boundary. The broker's backlog + // and entry arithmetic count ENTRIES, not messages - a batching producer packs many messages + // into one entry - so this can only ever be a "~N" and the client must present it that way + // ("new messages waiting" / "~N newer"), never as an exact count. 0 means none are known, + // which is not proof none exist. + int64 replay_newer_entries_approx = 8; + + // GUARANTEED delivery: how many messages were emitted with an order key LOWER than a key + // already emitted (see Message.replay_seam_violation). Monotonic per session; possible only + // across a resume seam under producer clock skew, or when a source log stores an inversion. + int64 replay_seam_violations = 9; + + // Topics that currently match this session's topic selectors but are NOT part of the running + // session - typically regex-matched topics created after Play. They are excluded from the + // replay by design (no late-joiner classifier); restarting the session includes them. Names + // are capped at a handful; replay_excluded_topic_count carries the full count. + repeated string replay_excluded_topics = 10; + int32 replay_excluded_topic_count = 11; +} message ResumeResponse { google.rpc.Status status = 1; @@ -561,6 +763,55 @@ message PauseResponse { google.rpc.Status status = 1; } +// Change the delivery order of a session that is ALREADY RUNNING, without recreating it. +// +// Two situations need it, both Guaranteed's. Mid-replay, a stream whose recorded range +// retention trimmed away can hold the replay indefinitely; the wait is disclosed +// (ConsumerStats.delivery_order_waiting_streams) and this call is what makes it ACTIONABLE - +// one call switches the live session to Best effort and the already-held messages are +// released, in the new order, exactly once. And at the replay boundary +// (ConsumerStats.replay_caught_up), this is the designed "continue live with Best effort" +// transition: the switch releases the boundary pause and the session resumes live delivery +// under Best effort - unless the user had ALSO paused it manually, in which case their pause +// stands and the next Resume goes live. +// +// WHY NOT JUST RECREATE THE SESSION. A recreate would lose the held set (received, unacknowledged, +// and nowhere else), RE-RESOLVE the start-from against a log that has moved - which for Latest-n +// and both approximate modes selects a DIFFERENT set of messages, silently changing what the user +// is looking at - and redeliver everything already on screen as duplicates. +// +// THIS CHANGES THE LIVE SESSION ONLY. It does not touch the saved session configuration; a client +// that wants the next Play to use the new order writes it into the configuration itself. +// +// ONLY A RELAXATION IS ACCEPTED, and only this one: Guaranteed -> Best effort. See +// SetDeliveryOrderResponse for what the other directions answer and why. +message SetDeliveryOrderRequest { + string consumer_name = 1; + // The order to switch the live session to. UNSPECIFIED names nothing and is refused with + // INVALID_ARGUMENT rather than resolved to the product default - a mutation must say what it + // wants. + MessageDeliveryOrder message_delivery_order = 2; +} + +// OK when the live session now delivers in the requested order - including when it already did, +// so a repeated click is harmless. +// +// FAILED_PRECONDITION, with the reason in the message, when the switch cannot be honestly +// performed: +// +// - the session does not exist, or its play stream has ended; +// - the request asks for a STRONGER order (anything -> Guaranteed). Guaranteed promises that +// Dekaf introduced no cross-stream disorder; a session that has already emitted past a silent +// stream cannot un-emit it, so no future behavior could make that promise true. Restart the +// session to get it; +// - the request asks for AS_RECEIVED. That is not a weaker order of the merged stream, it is +// the absence of the merge - and the same merge also resolves a counted start-from cut, so +// dismantling it mid-cut would silently change which messages the cut selects. Fastest stays +// a configuration choice that takes effect on the next Play. +message SetDeliveryOrderResponse { + google.rpc.Status status = 1; +} + message RunCodeRequest { string consumer_name = 1; string code = 2; @@ -579,11 +830,105 @@ message ResolveTopicSelectorResponse { repeated string topic_fqns = 2; } +// One physical topic's endpoints and how far this session has read through them. +// +// A DEBUG VIEW, polled on demand - never pushed with the message stream. Filling one +// row costs three admin round trips (first entry, last entry, internal stats), so a +// session over a wide selector costs three per PARTITION every refresh. That is why +// the client asks for this rather than receiving it, and why the asking is off by +// default. +// +// EVERY FIELD IS OPTIONAL BECAUSE EVERY LOOKUP CAN DECLINE. A topic Pulsar refuses to +// examine at all (non-persistent: 405) answers with `unavailable_reason` set and the +// endpoints absent - which is different from an EMPTY topic, where the lookups +// succeed and there is genuinely nothing to report. Absent means "not known", never +// "zero". +message TopicPosition { + // The physical topic - a partition of a partitioned topic, not the parent, since + // `examineMessage` refuses the parent outright. + string topic_fqn = 1; + + // The oldest message the topic still holds. Absent on an empty topic, and on one + // whose first entry aged out between the two lookups. + google.protobuf.BytesValue first_message_id = 2; + google.protobuf.Int64Value first_publish_time = 3; + + // The newest message the topic holds. Absent on an empty topic: `examineMessage` + // THROWS for "latest" there rather than answering, where "earliest" clamps. + google.protobuf.BytesValue last_message_id = 4; + google.protobuf.Int64Value last_publish_time = 5; + + // The furthest message-ID position this session has PROCESSED from this topic. This + // is session-local progress, not a Pulsar subscription cursor or the last row on screen. + // A session filter can drop almost everything it reads, so the last DISPLAYED message + // can lag this by an arbitrary amount and would make both progress figures below read + // far too low. + // + // Absent until the session has processed something from this topic. + google.protobuf.BytesValue cursor_message_id = 6; + google.protobuf.Int64Value cursor_publish_time = 7; + + // Where the cursor sits in the topic's TIME range: 0.0 at the first message's + // publish time, 1.0 at the last's. Absent when there is no cursor yet, and on a + // topic that occupies a single instant (first == last), which has no interior to + // place anything in. + google.protobuf.DoubleValue cursor_time_fraction = 8; + + // The session's entry position among the entries currently stored: position ordinal + // over stored count. Sitting ON the first of N entries is 1/N, and on the last it is + // 1.0; a topic storing a single entry is therefore 1.0 when that entry is processed. + // There is no 0.0 with a position present. + // + // ENTRIES, NOT MESSAGES. A batched entry holds many messages, so this tracks + // message count only as closely as batch sizes stayed uniform - the same + // approximation ApproximateEntryPosition documents. It is named for what it measures. + google.protobuf.DoubleValue cursor_entry_fraction = 9; + + // Entries currently stored in the topic, as `getInternalStats` reports them. + google.protobuf.Int64Value retained_entries = 10; + + // The session's 1-based position among the stored entries, which is the numerator of + // `cursor_entry_fraction`. Shown so a reader can see the arithmetic rather than + // trust a percentage. + google.protobuf.Int64Value cursor_entry_ordinal = 11; + + // Why this row is blank, when it is. Set for a topic the broker refuses to examine + // (a non-persistent topic cannot answer either lookup) and for a lookup that failed + // outright. An EMPTY topic is NOT a reason - it answers, with nothing in it. + google.protobuf.StringValue unavailable_reason = 12; + + // The earliest message-ID position this session has PROCESSED from this topic. Like the furthest position, + // this is session-local bookkeeping: it is tracked from the first consumed message + // even if this debug tab has never been opened, and remains known if retention later + // removes the message or the broker cannot expose retained-log endpoints. + google.protobuf.BytesValue first_consumed_message_id = 13; + google.protobuf.Int64Value first_consumed_publish_time = 14; +} + +message GetTopicPositionsRequest { + string consumer_name = 1; +} + +message GetTopicPositionsResponse { + google.rpc.Status status = 1; + repeated TopicPosition positions = 2; +} + service ConsumerService { rpc CreateConsumer(CreateConsumerRequest) returns (CreateConsumerResponse); rpc DeleteConsumer(DeleteConsumerRequest) returns (DeleteConsumerResponse); rpc Resume(ResumeRequest) returns (stream ResumeResponse); rpc Pause(PauseRequest) returns (PauseResponse); + + // Change a LIVE session's delivery order. A dedicated RPC rather than a field on Resume: this + // mutates a running session, and a session-mutating operation has to be explicit rather than a + // side effect of pressing Play. + rpc SetDeliveryOrder(SetDeliveryOrderRequest) returns (SetDeliveryOrderResponse); + rpc RunCode(RunCodeRequest) returns (RunCodeResponse); rpc ResolveTopicSelector(ResolveTopicSelectorRequest) returns (ResolveTopicSelectorResponse); + + // Poll the per-topic debug view. See [[TopicPosition]] for why this is polled rather + // than pushed. + rpc GetTopicPositions(GetTopicPositionsRequest) returns (GetTopicPositionsResponse); } diff --git a/proto/proto/tools/teal/pulsar/ui/library/v1/managed_items.proto b/proto/proto/tools/teal/pulsar/ui/library/v1/managed_items.proto index c431f3483..e939d7e8b 100644 --- a/proto/proto/tools/teal/pulsar/ui/library/v1/managed_items.proto +++ b/proto/proto/tools/teal/pulsar/ui/library/v1/managed_items.proto @@ -95,6 +95,8 @@ message ManagedConsumerSessionStartFromSpec { ManagedMessageIdValOrRef start_from_message_id = 3; ManagedDateTimeValOrRef start_from_date_time = 4; ManagedRelativeDateTimeValOrRef start_from_relative_date_time = 5; + tools.teal.pulsar.ui.api.v1.ApproximateEntryPosition start_from_approximate_entry_position = 8; + tools.teal.pulsar.ui.api.v1.ApproximatePublishTimePosition start_from_approximate_publish_time_position = 9; } } @@ -352,6 +354,13 @@ message ManagedConsumerSessionConfigSpec { ManagedColoringRuleChainValOrRef coloring_rule_chain = 5; ManagedValueProjectionListValOrRef value_projection_list = 6; google.protobuf.Int64Value num_display_items = 7; + // Absent (UNSPECIFIED) means GUARANTEED, the default (owner decision 2026-08-11, superseding + // the 2026-08-09 Best effort default) - including every item saved before this field existed. + // Explicit BEST_EFFORT_BY_PUBLISH_TIME and AS_RECEIVED (Fastest) are choices and are + // preserved as written. + tools.teal.pulsar.ui.api.v1.MessageDeliveryOrder message_delivery_order = 8; + // Absent (UNSPECIFIED) means publish time, exactly as every ordered spec saved before the field. + tools.teal.pulsar.ui.api.v1.DeliveryOrderKey delivery_order_key = 9; } message ManagedConsumerSessionConfig { diff --git a/proto/proto/tools/teal/pulsar/ui/library/v1/resource_matchers.proto b/proto/proto/tools/teal/pulsar/ui/library/v1/resource_matchers.proto index 60b436f06..95c4b2fa2 100644 --- a/proto/proto/tools/teal/pulsar/ui/library/v1/resource_matchers.proto +++ b/proto/proto/tools/teal/pulsar/ui/library/v1/resource_matchers.proto @@ -24,6 +24,15 @@ message ExactNamespaceMatcher { message AllNamespaceMatcher { TenantMatcher tenant = 1; + + // NOT IMPLEMENTED - setting this field is REJECTED with INVALID_ARGUMENT. It is not ignored: + // ignoring it returned a matcher covering EVERY namespace of the matching tenant to a caller that + // had asked for a subset, which is a silent widening of access scope. + // + // Unimplemented deliberately, not by oversight: matchers are tested against other matchers rather + // than against a concrete namespace, so an AllNamespaceMatcher tested against another + // AllNamespaceMatcher would have to decide whether one regex subsumes another, which is + // undecidable in general. Leave it unset; narrow with ExactNamespaceMatcher instead. string namespace_regex = 2; } diff --git a/server/src/main/scala/brokers/BrokersServiceImpl.scala b/server/src/main/scala/brokers/BrokersServiceImpl.scala index c443cb7bc..a6609ec11 100644 --- a/server/src/main/scala/brokers/BrokersServiceImpl.scala +++ b/server/src/main/scala/brokers/BrokersServiceImpl.scala @@ -25,13 +25,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { val config = adminClient.brokers.getAllDynamicConfigurations.asScala.toMap Future.successful( GetAllDynamicConfigurationsResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), config ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetAllDynamicConfigurationsResponse(status = Some(status))) } @@ -42,13 +42,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { val names = adminClient.brokers.getDynamicConfigurationNames.asScala.toList Future.successful( GetDynamicConfigurationNamesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), names ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetDynamicConfigurationNamesResponse(status = Some(status))) } @@ -66,13 +66,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { Future.successful( GetInternalConfigurationDataResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), config = Some(config) ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetInternalConfigurationDataResponse(status = Some(status))) } @@ -83,13 +83,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { val config = adminClient.brokers.getRuntimeConfigurations.asScala.toMap Future.successful( GetRuntimeConfigurationsResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), config ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetRuntimeConfigurationsResponse(status = Some(status))) } @@ -100,11 +100,11 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { try { adminClient.brokers.updateDynamicConfiguration(request.name, request.value) Future.successful( - UpdateDynamicConfigurationResponse(status = Some(Status(code = Code.OK.index))) + UpdateDynamicConfigurationResponse(status = Some(Status(code = Code.OK.value))) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(UpdateDynamicConfigurationResponse(status = Some(status))) } @@ -115,11 +115,11 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { try { adminClient.brokers.deleteDynamicConfiguration(request.name) Future.successful( - DeleteDynamicConfigurationResponse(status = Some(Status(code = Code.OK.index))) + DeleteDynamicConfigurationResponse(status = Some(Status(code = Code.OK.value))) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteDynamicConfigurationResponse(status = Some(status))) } @@ -130,11 +130,11 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { try { adminClient.brokers.healthcheck(TopicVersion.V2) Future.successful( - HealthCheckResponse(status = Some(Status(code = Code.OK.index)), isOk = true) + HealthCheckResponse(status = Some(Status(code = Code.OK.value)), isOk = true) ) } catch { case err => - val status = Status(code = Code.OK.index, message = err.getMessage) + val status = Status(code = Code.OK.value, message = err.getMessage) Future.successful(HealthCheckResponse(status = Some(status), isOk = false)) } @@ -146,7 +146,7 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { val resourceFqn = request.resourceFqn def failWithMessage(message: String): Future[CheckResourceExistsResponse] = - val status = Status(code = Code.FAILED_PRECONDITION.index, message = message) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = message) Future.successful(CheckResourceExistsResponse(status = Some(status), isExists = false)) val isResourceExists = request.resource match @@ -166,7 +166,7 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { return failWithMessage("Resource type should be specified") Future.successful( - CheckResourceExistsResponse(status = Some(Status(code = Code.OK.index)), isExists = isResourceExists) + CheckResourceExistsResponse(status = Some(Status(code = Code.OK.value)), isExists = isResourceExists) ) override def backlogQuotaCheck(request: BacklogQuotaCheckRequest): Future[BacklogQuotaCheckResponse] = @@ -176,11 +176,11 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { try { adminClient.brokers.backlogQuotaCheck() Future.successful( - BacklogQuotaCheckResponse(status = Some(Status(code = Code.OK.index)), isOk = true) + BacklogQuotaCheckResponse(status = Some(Status(code = Code.OK.value)), isOk = true) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(BacklogQuotaCheckResponse(status = Some(status), isOk = false)) } override def getResourceGroupsList(request: GetResourceGroupsListRequest): Future[GetResourceGroupsListResponse] = @@ -191,13 +191,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { Future.successful( GetResourceGroupsListResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), resourceGroups ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetResourceGroupsListResponse(status = Some(status))) } override def getResourceGroups(request: GetResourceGroupsRequest): Future[GetResourceGroupsResponse] = @@ -220,13 +220,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { Future.successful( GetResourceGroupsResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), resourceGroups ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetResourceGroupsResponse(status = Some(status))) } @@ -246,13 +246,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { Future.successful( pb.GetResourceGroupResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), resourceGroup = Some(resourceGroup) ) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetResourceGroupResponse(status = Some(status))) } @@ -271,13 +271,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { rg.publishRateInMsgs.foreach(n => resourceGroup.setPublishRateInMsgs(n)) adminClient.resourcegroups.createResourceGroup(rg.name, resourceGroup) - Future.successful(CreateResourceGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(CreateResourceGroupResponse(status = Some(Status(code = Code.OK.value)))) case None => - val status = Status(code = Code.INVALID_ARGUMENT.index, message = "Resource group should be specified") + val status = Status(code = Code.INVALID_ARGUMENT.value, message = "Resource group should be specified") Future.successful(CreateResourceGroupResponse(status = Some(status))) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateResourceGroupResponse(status = Some(status))) } @@ -288,11 +288,11 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { try { adminClient.resourcegroups.deleteResourceGroup(request.name) Future.successful( - DeleteResourceGroupResponse(status = Some(Status(code = Code.OK.index))) + DeleteResourceGroupResponse(status = Some(Status(code = Code.OK.value))) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteResourceGroupResponse(status = Some(status))) } @@ -311,13 +311,13 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { rg.publishRateInMsgs.foreach(n => resourceGroup.setPublishRateInMsgs(n)) adminClient.resourcegroups.updateResourceGroup(rg.name, resourceGroup) - Future.successful(pb.UpdateResourceGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.UpdateResourceGroupResponse(status = Some(Status(code = Code.OK.value)))) case None => - val status = Status(code = Code.INVALID_ARGUMENT.index, message = "Resource group should be specified") + val status = Status(code = Code.INVALID_ARGUMENT.value, message = "Resource group should be specified") Future.successful(pb.UpdateResourceGroupResponse(status = Some(status))) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.UpdateResourceGroupResponse(status = Some(status))) } @@ -329,15 +329,15 @@ class BrokersServiceImpl extends pb.BrokersServiceGrpc.BrokersService { Option(adminClient.brokers.getVersion) match case Some(version) => Future.successful(pb.GetVersionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), version )) case None => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = "Something went wrong.") + val status = Status(code = Code.FAILED_PRECONDITION.value, message = "Something went wrong.") Future.successful(pb.GetVersionResponse(status = Some(status))) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetVersionResponse(status = Some(status))) } } diff --git a/server/src/main/scala/brokerstats/BrokerStatsServiceImpl.scala b/server/src/main/scala/brokerstats/BrokerStatsServiceImpl.scala index 82fd86271..00f0bec6e 100644 --- a/server/src/main/scala/brokerstats/BrokerStatsServiceImpl.scala +++ b/server/src/main/scala/brokerstats/BrokerStatsServiceImpl.scala @@ -20,10 +20,10 @@ class BrokerStatsServiceImpl extends pb.BrokerStatsServiceGrpc.BrokerStatsServic try { val statsJson = adminClient.brokerStats.getMetrics Future.successful( - pb.GetBrokerStatsJsonResponse(status = Some(Status(code = Code.OK.index)), statsJson) + pb.GetBrokerStatsJsonResponse(status = Some(Status(code = Code.OK.value)), statsJson) ) } catch { case err => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetBrokerStatsJsonResponse(status = Some(status))) } diff --git a/server/src/main/scala/childrencount/ChildrencountServiceImpl.scala b/server/src/main/scala/childrencount/ChildrencountServiceImpl.scala index 5e68de5ff..a3217fdb3 100644 --- a/server/src/main/scala/childrencount/ChildrencountServiceImpl.scala +++ b/server/src/main/scala/childrencount/ChildrencountServiceImpl.scala @@ -46,7 +46,7 @@ class TenantServiceImpl extends pb.ChildrenCountServiceGrpc.ChildrenCountService given ExecutionContext = ExecutionContext.global val allResults = Await.result(Future.sequence(allFutures), Duration(1, TimeUnit.MINUTES)) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful( pb.GetChildrenCountResponse( status = Some(status), @@ -59,6 +59,6 @@ class TenantServiceImpl extends pb.ChildrenCountServiceGrpc.ChildrenCountService ) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetChildrenCountResponse(status = Some(status))) } diff --git a/server/src/main/scala/clusters/ClustersServiceImpl.scala b/server/src/main/scala/clusters/ClustersServiceImpl.scala index 52b820d4c..0796a08ec 100644 --- a/server/src/main/scala/clusters/ClustersServiceImpl.scala +++ b/server/src/main/scala/clusters/ClustersServiceImpl.scala @@ -21,11 +21,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try val clusters = adminClient.clusters.getClusters - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetClustersResponse(status = Some(status), clusters = clusters.asScala.toList)) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetClustersResponse(status = Some(status))) override def getCluster(request: GetClusterRequest): Future[GetClusterResponse] = @@ -35,12 +35,12 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try adminClient.clusters.getCluster(request.cluster) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(GetClusterResponse(status = Some(status))) val clusterDataPb = conversions.clusterDataToPb(clusterData) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetClusterResponse(status = Some(status), clusterData = Some(clusterDataPb))) override def createCluster(request: CreateClusterRequest): Future[CreateClusterResponse] = @@ -49,17 +49,17 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try val clusterData = request.clusterData match case None => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = "Cluster data is empty") + val status = Status(code = Code.FAILED_PRECONDITION.value, message = "Cluster data is empty") return Future.successful(CreateClusterResponse(status = Some(status))) case Some(cd) => conversions.clusterDataFromPb(cd) adminClient.clusters.createCluster(request.cluster, clusterData) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(CreateClusterResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateClusterResponse(status = Some(status))) override def deleteCluster(request: DeleteClusterRequest): Future[DeleteClusterResponse] = @@ -68,11 +68,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try adminClient.clusters.deleteCluster(request.cluster) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(DeleteClusterResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteClusterResponse(status = Some(status))) override def getFailureDomains(request: GetFailureDomainsRequest): Future[GetFailureDomainsResponse] = @@ -81,11 +81,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try val failureDomains = adminClient.clusters.getFailureDomains(request.cluster) val failureDomainsPb = failureDomains.asScala.view.mapValues(conversions.failureDomainToPb).toMap - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetFailureDomainsResponse(status = Some(status), domains = failureDomainsPb)) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetFailureDomainsResponse(status = Some(status))) override def createFailureDomain(request: CreateFailureDomainRequest): Future[CreateFailureDomainResponse] = @@ -101,11 +101,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: request.domainName, failureDomain ) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(CreateFailureDomainResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateFailureDomainResponse(status = Some(status))) override def deleteFailureDomain(request: DeleteFailureDomainRequest): Future[DeleteFailureDomainResponse] = @@ -113,11 +113,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try adminClient.clusters.deleteFailureDomain(request.cluster, request.domainName) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(DeleteFailureDomainResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteFailureDomainResponse(status = Some(status))) override def updateFailureDomain(request: UpdateFailureDomainRequest): Future[UpdateFailureDomainResponse] = @@ -132,11 +132,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: request.domainName, failureDomain ) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(UpdateFailureDomainResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(UpdateFailureDomainResponse(status = Some(status))) override def createNamespaceIsolationPolicy(request: CreateNamespaceIsolationPolicyRequest): Future[CreateNamespaceIsolationPolicyResponse] = @@ -151,11 +151,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: request.policyName, policy ) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(CreateNamespaceIsolationPolicyResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateNamespaceIsolationPolicyResponse(status = Some(status))) override def deleteNamespaceIsolationPolicy(request: DeleteNamespaceIsolationPolicyRequest): Future[DeleteNamespaceIsolationPolicyResponse] = @@ -163,11 +163,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try adminClient.clusters.deleteNamespaceIsolationPolicy(request.cluster, request.policyName) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(DeleteNamespaceIsolationPolicyResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteNamespaceIsolationPolicyResponse(status = Some(status))) override def getNamespaceIsolationPolicy(request: GetNamespaceIsolationPolicyRequest): Future[GetNamespaceIsolationPolicyResponse] = @@ -176,11 +176,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try val policy = adminClient.clusters.getNamespaceIsolationPolicy(request.cluster, request.policyName) val namespaceIsolationDataPb = conversions.namespaceIsolationDataToPb(policy) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetNamespaceIsolationPolicyResponse(status = Some(status), namespaceIsolationData = Some(namespaceIsolationDataPb))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetNamespaceIsolationPolicyResponse(status = Some(status))) override def updateNamespaceIsolationPolicy(request: UpdateNamespaceIsolationPolicyRequest): Future[UpdateNamespaceIsolationPolicyResponse] = @@ -195,11 +195,11 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: request.policyName, policy ) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(UpdateNamespaceIsolationPolicyResponse(status = Some(status))) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(UpdateNamespaceIsolationPolicyResponse(status = Some(status))) override def getBrokersWithNamespaceIsolationPolicy( @@ -210,9 +210,9 @@ class ClustersServiceImpl extends ClustersServiceGrpc.ClustersService: try val brokers = adminClient.clusters.getBrokersWithNamespaceIsolationPolicy(request.cluster) val brokersPb = brokers.asScala.toList.map(conversions.brokerNamespaceIsolationDataToPb) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetBrokersWithNamespaceIsolationPolicyResponse(status = Some(status), brokers = brokersPb)) catch case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetBrokersWithNamespaceIsolationPolicyResponse(status = Some(status))) diff --git a/server/src/main/scala/config/mergeConfigs.scala b/server/src/main/scala/config/mergeConfigs.scala index e6c408bf9..f17ed4e3e 100644 --- a/server/src/main/scala/config/mergeConfigs.scala +++ b/server/src/main/scala/config/mergeConfigs.scala @@ -10,6 +10,8 @@ def mergeConfigs(lowPriority: Config, highPriority: Config): Config = protocol = highPriority.protocol.orElse(lowPriority.protocol), tlsCertificateFilePath = highPriority.tlsCertificateFilePath.orElse(lowPriority.tlsCertificateFilePath), tlsKeyFilePath = highPriority.tlsKeyFilePath.orElse(lowPriority.tlsKeyFilePath), + cookieSecure = highPriority.cookieSecure.orElse(lowPriority.cookieSecure), + cookieSameSite = highPriority.cookieSameSite.orElse(lowPriority.cookieSameSite), pulsarName = highPriority.pulsarName.orElse(lowPriority.pulsarName), pulsarColor = highPriority.pulsarColor.orElse(lowPriority.pulsarColor), pulsarListenerName = highPriority.pulsarListenerName.orElse(lowPriority.pulsarListenerName), diff --git a/server/src/main/scala/consumer/ConsumerServiceImpl.scala b/server/src/main/scala/consumer/ConsumerServiceImpl.scala index 5c89fe8d2..ec0be1f79 100644 --- a/server/src/main/scala/consumer/ConsumerServiceImpl.scala +++ b/server/src/main/scala/consumer/ConsumerServiceImpl.scala @@ -1,6 +1,6 @@ package consumer -import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} import _root_.pulsar_auth.RequestContext import com.google.rpc.code.Code import com.google.rpc.status.Status @@ -12,6 +12,8 @@ import com.tools.teal.pulsar.ui.api.v1.consumer.{ CreateConsumerResponse, DeleteConsumerRequest, DeleteConsumerResponse, + GetTopicPositionsRequest, + GetTopicPositionsResponse, PauseRequest, PauseResponse, ResolveTopicSelectorRequest, @@ -19,135 +21,714 @@ import com.tools.teal.pulsar.ui.api.v1.consumer.{ ResumeRequest, ResumeResponse, RunCodeRequest, - RunCodeResponse + RunCodeResponse, + SetDeliveryOrderRequest, + SetDeliveryOrderResponse } import com.typesafe.scalalogging.Logger import _root_.consumer.session_target.topic_selector.TopicSelector -import consumer.session_runner.ConsumerSessionRunner +import consumer.session_runner.{ + ConsumerSessionRunner, + LedgerSpan, + LogEndpoint, + TopicConsumedBounds, + TopicPositionInputs, + TopicPositionRow, + brokerAnswer, + buildTopicPositionRow, + entryIdOf, + mergeConsumedBounds, + storeConsumerSession, + topicPositionToPb +} +import org.apache.pulsar.client.admin.PulsarAdmin import java.util.concurrent.ConcurrentHashMap import scala.concurrent.Future +import scala.jdk.CollectionConverters.* import scala.jdk.OptionConverters.* import scala.util.{Failure, Success, Try} type ConsumerSessionName = String -class ConsumerServiceImpl extends ConsumerServiceGrpc.ConsumerService: +object ConsumerServiceImpl: + /** The production session builder. The Pulsar clients come from the gRPC request context, which + * is why this is a function rather than a direct call: a test can drive the lifecycle through + * the real RPC surface without a broker or a request context. */ + def makeFromRequestContext(sessionName: ConsumerSessionName, sessionConfig: ConsumerSessionConfig): ConsumerSessionRunner = + ConsumerSessionRunner.make( + sessionName = sessionName, + pulsarClient = RequestContext.pulsarClient.get(), + adminClient = RequestContext.pulsarAdmin.get(), + sessionConfig = sessionConfig + ) + + /** How long a session may sit without a live play stream before it is reaped. Generous on + * purpose: a paused tab is a session with no stream, and a user coming back from a meeting + * should find their session where they left it. What this bounds is the FOREVER case - a + * closed tab whose `beforeunload` Delete never delivered used to leave consumers connected + * and reconnect-looping for the life of the process. */ + private val idleSessionTtlNanos: Long = java.util.concurrent.TimeUnit.HOURS.toNanos(1) + + /** How often the janitor looks - precision is irrelevant at a 1-hour TTL. */ + private val janitorPeriodSeconds: Long = 60 + + /** How long a session may sit with no live play stream AND every one of its consumers + * disconnected before it is reaped - the short leash beside [[idleSessionTtlNanos]]. + * + * Disconnected + unwatched is dead weight. If the disconnect is a transient broker blip and + * a user is actually watching, their live play stream fails the first condition and protects + * the session; if the tab is gone AND the broker connection is gone, nothing of value is + * lost by reaping in minutes rather than an hour. The case this exists for can never recover + * at all: a consumer whose topic or namespace was force-deleted reconnect-loops on a + * subscription with nothing left to reconnect to, at full retry volume, for however long the + * session survives - an hour of that spam serves nobody. + * + * Two janitor periods, so one sweep's sighting never reaps on its own: the state must be + * seen to HOLD across sweeps before it is believed. + */ + private val disconnectedSessionGraceNanos: Long = + java.util.concurrent.TimeUnit.SECONDS.toNanos(janitorPeriodSeconds * 2) + + /** The most sessions this server runs at once. Each session holds consumers, receiver queues + * and JS contexts; without a bound, abandoned tabs alone could grow that without limit (the + * idle janitor trims them, but only after the TTL). + * + * AN ADMISSION BOUND, not a post-hoc count. It used to be read off the map under a lock + * scoped to ONE session name, and the expensive build then ran before the insertion - so + * concurrent creates of DIFFERENT names all observed the same pre-build size and all passed. + * At an empty map an arbitrary number of builds could pass a 100-session check, each free to + * subscribe up to 2,000 streams. See [[ConsumerServiceImpl.reserveAdmission]]. */ + val maxActiveSessions: Int = 100 + + +/** @param consumerSessions + * the live sessions this service owns. A constructor parameter (with the production default) + * only so a test can put a real session behind an RPC without a broker - the resume/delete paths + * touch nothing but the runner and the observer. + * @param makeSession + * how a session is built. Injectable for the same reason: the lifecycle ORDERING - that a + * predecessor is stopped before its replacement subscribes, and that two creates under one name + * never overlap - is a property of this class and has to be testable without a broker. + * @param idleSessionTtlNanos + * how long a session may sit with NO live play stream before the janitor stops and removes it. + * A parameter only so a test can expire a session without waiting an hour. + * @param disconnectedSessionGraceNanos + * how long a session with no live play stream AND every consumer disconnected may persist + * before the janitor reaps it early - see the companion default for why that state gets the + * short leash. A parameter only so a test can cross the window without waiting minutes. + * + * DEFERRED, KNOWINGLY: sessions are addressed by name alone. Any caller that knows (or guesses) a + * session's name can resume it, run code against its JS contexts, pause it or delete it - there + * is no binding to the creator's identity or connection. Fixing that is an API-surface design + * (an opaque capability handed back by create, or identity plumbed from auth), not a patch here. + */ +class ConsumerServiceImpl( + private val consumerSessions: ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner] = + new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner](), + private val makeSession: (ConsumerSessionName, ConsumerSessionConfig) => ConsumerSessionRunner = + ConsumerServiceImpl.makeFromRequestContext, + private val idleSessionTtlNanos: Long = ConsumerServiceImpl.idleSessionTtlNanos, + private val disconnectedSessionGraceNanos: Long = ConsumerServiceImpl.disconnectedSessionGraceNanos, + private val maxActiveSessions: Int = ConsumerServiceImpl.maxActiveSessions +) extends ConsumerServiceGrpc.ConsumerService: private val logger: Logger = Logger(getClass.getName) - private val consumerSessions: ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner] = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + + /** Lifecycle operations on ONE session name run one at a time. + * + * Every target subscribes as `${sessionName}-${targetIndex}`, non-durable and EXCLUSIVE, so two + * runners under one name cannot coexist on the broker at all - the second to subscribe is + * refused. Create and delete therefore have to be single operations rather than a read, some + * broker work, and a write - and resume joins them, because it must not wire a fresh observer + * into a runner a racing delete is stopping. + * + * ONE LOCK PER NAME, not a striped array. The lock is held across the predecessor's stop and + * the replacement's ENTIRE build - subscribing every consumer, seeking it, and for a Latest-N + * start-from a backward walk costing one admin lookup per entry - which is unbounded broker + * work, not one round trip. Striping made two UNRELATED names serialize whenever their hashes + * collided (1 stripe in 64), so one slow create could stall another session's create, delete + * and resume for as long as that broker work took. A map entry is a bare Object of a few dozen + * bytes against names the client chooses; the entries are REF-COUNTED away by + * [[withLifecycleLock]] below, so the map holds only the names something is actively locking + * right now - not every name any client has ever sent. + */ + private val lifecycleLocks = new ConcurrentHashMap[ConsumerSessionName, (Object, java.util.concurrent.atomic.AtomicInteger)]() + + /** How many distinct names currently hold or wait for a lifecycle lock - a test's window into + * the map's lifetime, and nothing else's. */ + private[consumer] def lifecycleLockCount: Int = lifecycleLocks.size + + /** Builds that have taken an admission permit and have not yet installed (or abandoned) their + * runner. The permit has to cover the BUILD, not merely the installed runner: subscribing a + * session's consumers is where the threads, connections and receiver queues the cap exists to + * bound are actually taken, and it happens long before anything reaches the map. + * + * Counted separately from the map rather than replacing it, so the installed half stays + * authoritative: delete, the janitor's reap and a replacement all drop the map entry, and + * none of them needs to remember to hand a permit back for the arithmetic to stay right. + */ + private val buildsInFlight = java.util.concurrent.atomic.AtomicInteger(0) + + /** Sessions this server is running or building right now - what [[maxActiveSessions]] bounds, + * and a test's window onto it. */ + private[consumer] def admittedSessionCount: Int = consumerSessions.size + buildsInFlight.get + + /** At most one Topic Positions scan per session, off the gRPC threads and inside the + * server-wide broker-admin budget. See [[TopicPositionScanner]]. */ + private val topicPositionScans = consumer.session_runner.TopicPositionScanner() + + /** Take one GLOBAL admission permit for a build about to start, or answer false. + * + * `allowance` is how many of the currently installed sessions this build is about to remove: + * 1 for a REPLACEMENT, whose predecessor is stopped a few lines below and therefore does not + * count against the total the replacement will produce. That is what keeps the browser's + * ordinary flow - it re-creates the session on every configuration change - admissible at the + * cap, without the replacement first dropping its predecessor's slot and racing another name + * for it. + * + * A CAS loop rather than a new lock: the count of installed sessions is read fresh on every + * attempt, so two creates that both see room compete for the same permit and exactly one of + * them gets it. No lock is introduced and no lock ordering changes - this is called while the + * caller holds its per-name lifecycle lock and takes nothing else. + */ + private def reserveAdmission(allowance: Int): Boolean = + var reserved = false + var settled = false + while !settled do + val inFlight = buildsInFlight.get + if consumerSessions.size + inFlight - allowance >= maxActiveSessions then settled = true + else if buildsInFlight.compareAndSet(inFlight, inFlight + 1) then + reserved = true + settled = true + reserved + + /** Run `body` holding this name's lifecycle lock. REF-COUNTED: the entry exists only while + * someone holds or waits for it, and the last one out removes it. The names arrive on RPC + * input, so the previous keep-forever map let any client grow the heap without bound by + * naming sessions that never existed; plain eviction was no answer either, because evicting + * an entry a thread was WAITING on hands the next caller a different lock for the same name + * and the exclusion silently vanishes. The count is what makes removal safe: it only happens + * when nobody is inside and nobody is queued. + */ + private def withLifecycleLock[T](sessionName: ConsumerSessionName)(body: => T): T = + val entry = lifecycleLocks.compute( + sessionName, + (_, existing) => + if existing == null then (Object(), java.util.concurrent.atomic.AtomicInteger(1)) + else + existing._2.incrementAndGet() + existing + ) + try entry._1.synchronized(body) + finally + lifecycleLocks.compute( + sessionName, + (_, current) => + if current == null then null + else if current._2.decrementAndGet() == 0 then null + else current + ) + + /** The transport says a play stream's call died - the tab closed, the network dropped, or the + * client simply cancelled after a pause. Intake is PAUSED, never deleted: the ordinary UI + * pause cancels its stream too, and the user expects that session to still be there on the + * next Play. Deletion is the idle janitor's job, an hour later. + * + * Under the lifecycle lock like every other lifecycle step, so it cannot interleave + * target-by-target with a resume in flight; the compare-and-clear inside the runner then + * makes a LATE cancellation of a replaced stream a no-op instead of a stomp on the + * successor's play. + */ + private def onPlayStreamCancelled( + sessionName: ConsumerSessionName, + observer: io.grpc.stub.StreamObserver[ResumeResponse] + ): Unit = + withLifecycleLock(sessionName) { + Option(consumerSessions.get(sessionName)).foreach { consumerSession => + if consumerSession.releaseCancelledObserver(observer) && !consumerSession.isStreamCompleted then + logger.info(s"Play stream for consumer session $sessionName was cancelled by the transport. Pausing its intake.") + Try(consumerSession.pause()).failed.foreach(err => + logger.warn(s"Consumer session $sessionName could not be fully paused after its stream was cancelled. ${err.getMessage}") + ) + } + } + + /** When the janitor FIRST saw each still-installed runner with no live play stream AND every + * consumer disconnected - the early-reap rule's memory between sweeps. An entry is cleared + * the moment a sweep sees the state break (a consumer reconnected, a stream re-wired), so + * the grace window measures a state that HELD, not one that flickered. Keyed by name but + * bound to the runner instance: a replacement under the same name starts its own window. + * Only [[reapIdleSessions]] touches this map - one janitor thread, or a test driving it + * directly - and every sweep prunes the entries of sessions that are gone, so client-chosen + * names cannot accumulate here. + */ + private val disconnectedIdleFirstSeenNanos = new ConcurrentHashMap[ConsumerSessionName, (ConsumerSessionRunner, Long)]() + + /** Stop and remove every session no client is coming back for. TWO rules say when: + * + * - THE TTL: no live play stream for [[idleSessionTtlNanos]]. Generous on purpose - a + * paused tab is a session with no stream, and its user may just be in a meeting. + * - THE SHORT LEASH: no live play stream AND every consumer of every target answering + * `isConnected == false` (a cheap in-memory flag, not a broker round trip), both seen to + * hold across sweeps for [[disconnectedSessionGraceNanos]]. Disconnected + unwatched is + * dead weight: a watching user's live stream protects a session through any broker blip, + * and in the worst case - the session's topics force-deleted under it - the consumers + * reconnect-loop on topics that can never come back, so the remaining TTL buys nothing + * but an hour of retry spam. + * + * This is the backstop for clients that vanish without a Delete RPC: the browser sends it + * from `beforeunload`, where delivery is best-effort, so a closed tab could leave its + * consumers connected - and, once their namespace was deleted, reconnect-looping - for the + * life of the process. Each expired name is re-checked under its lifecycle lock, so a session + * the user resumes between the scan and the reap is left exactly as their resume wired it. + * + * Package-private and time-injected so a test can expire sessions deterministically; the + * scheduled task below is the only production caller. + */ + private[consumer] def reapIdleSessions(nowNanos: Long): Vector[ConsumerSessionName] = + def noLiveStreamFor(runner: ConsumerSessionRunner, windowNanos: Long): Boolean = + runner.reapableSinceNanos.exists(since => nowNanos - since >= windowNanos) + + // "Every consumer of every target says it is not connected" - and there IS at least one. + // A session holding no consumers at all holds nothing on the broker, so the short leash + // has no business with it; the TTL remains its only bound. + def fullyDisconnected(runner: ConsumerSessionRunner): Boolean = + val consumers = runner.targets.values.flatMap(_.consumers.values) + consumers.nonEmpty && consumers.forall(!_.isConnected) + + // Entries of sessions that no longer exist (deleted, replaced, reaped) go first, then the + // installed sessions' observations are brought up to date inside the sweep below. + disconnectedIdleFirstSeenNanos.entrySet.removeIf(entry => consumerSessions.get(entry.getKey) ne entry.getValue._1) + + // The short leash's whole verdict. The first-seen clause is the SUSTAINED requirement: a + // single sweep's sighting records the state but never reaps it - it must still hold a + // full grace window later, with `reapableSinceNanos` (which any resume in between would + // have reset) confirming nobody watched for that window either. + def earlyExpired(sessionName: ConsumerSessionName, runner: ConsumerSessionRunner): Boolean = + noLiveStreamFor(runner, disconnectedSessionGraceNanos) + && Option(disconnectedIdleFirstSeenNanos.get(sessionName)) + .exists((seen, firstSeen) => (seen eq runner) && nowNanos - firstSeen >= disconnectedSessionGraceNanos) + && fullyDisconnected(runner) + + def expired(sessionName: ConsumerSessionName, runner: ConsumerSessionRunner): Boolean = + noLiveStreamFor(runner, idleSessionTtlNanos) || earlyExpired(sessionName, runner) + + consumerSessions.asScala.toVector.flatMap { (sessionName, candidate) => + // This sweep's observation first, so the sustained clock is fed by the same sweeps + // that read it: note a first sighting, keep an existing one, forget a broken one. + if candidate.reapableSinceNanos.isDefined && fullyDisconnected(candidate) then + disconnectedIdleFirstSeenNanos.compute( + sessionName, + (_, existing) => if existing != null && (existing._1 eq candidate) then existing else (candidate, nowNanos) + ) + else disconnectedIdleFirstSeenNanos.remove(sessionName) + + Option.when(expired(sessionName, candidate)) { + withLifecycleLock(sessionName) { + // Same runner, still expired, now that nothing else can be mid-lifecycle. + Option(consumerSessions.get(sessionName)).filter(r => (r eq candidate) && expired(sessionName, r)) match + case Some(runner) => + val reason = + if noLiveStreamFor(runner, idleSessionTtlNanos) then + s"no live play stream for over ${idleSessionTtlNanos / 1_000_000_000L}s" + else + s"no live play stream and every consumer disconnected for over ${disconnectedSessionGraceNanos / 1_000_000_000L}s" + logger.info(s"Reaping consumer session $sessionName: $reason.") + Try(runner.stop()).failed.foreach(err => + logger.warn(s"Idle consumer session $sessionName could not be fully released. ${err.getMessage}") + ) + consumerSessions.remove(sessionName, runner) + disconnectedIdleFirstSeenNanos.remove(sessionName) + Some(sessionName) + case None => None + } + }.flatten + } + + // The janitor itself: one shared maintenance thread, one scan a minute. Never cancelled - the + // service lives for the process. It also drives the cleanup quarantine's retry: a consumer that + // refused to unsubscribe or close is retained rather than forgotten, and this is the only thing + // that ever calls it again - see [[consumer.session_runner.CleanupQuarantine]]. + ConsumerSessionRunner.maintenanceScheduler.scheduleWithFixedDelay( + () => + Try(reapIdleSessions(System.nanoTime())).failed.foreach(err => logger.warn(s"Idle-session sweep failed. ${err.getMessage}")) + Try(consumer.session_runner.consumerCleanupQuarantine.retryPending()).failed + .foreach(err => logger.warn(s"Retrying quarantined consumer cleanups failed. ${err.getMessage}")) + , + ConsumerServiceImpl.janitorPeriodSeconds, + ConsumerServiceImpl.janitorPeriodSeconds, + java.util.concurrent.TimeUnit.SECONDS + ) + + /** Whether the transport has already reported this call dead. Only a real server-streaming + * observer can say; a plain one (a test's, or a unary path) never claims to be cancelled. */ + private def isCancelled(observer: io.grpc.stub.StreamObserver[ResumeResponse]): Boolean = + observer match + case sco: io.grpc.stub.ServerCallStreamObserver[ResumeResponse @unchecked] => sco.isCancelled + case _ => false override def resume(request: ResumeRequest, responseObserver: io.grpc.stub.StreamObserver[ResumeResponse]): Unit = val sessionName = request.consumerName logger.info(s"Resuming consumer session: $sessionName") - val consumerSession = Option(consumerSessions.get(sessionName)) match - case Some(consumerSession) => consumerSession - case _ => - val msg = s"No such consumer consumer session: $sessionName" - logger.warn(msg) - - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = msg) - val res = ResumeResponse(status = Some(status)) - responseObserver.onNext(res) - responseObserver.onCompleted() - return - - try { - consumerSession.resume(grpcResponseObserver = responseObserver, isDebug = request.isDebug) - } catch { - case err: Throwable => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) - val res = ResumeResponse(status = Some(status)) - responseObserver.onNext(res) - responseObserver.onCompleted() - return - } + // FIRST, while gRPC still allows handler registration: hear about the call dying. Without + // this, a closed tab or dropped connection leaves the session consuming and acknowledging + // into a dead stream until something else stops it. Installing the handler also switches + // grpc-java from THROWING on sends into a cancelled call to silently dropping them, which + // is what every Pulsar listener thread pushing into this observer wants. Plain observers + // (tests) simply do without. + responseObserver match + case observer: io.grpc.stub.ServerCallStreamObserver[ResumeResponse @unchecked] => + observer.setOnCancelHandler(() => onPlayStreamCancelled(sessionName, responseObserver)) + case _ => () + + // For the cases where the observer was never wired into anything: nothing else can be + // writing to it, so answering it directly is safe here and only here. + def answerAndClose(message: String, code: Code = Code.FAILED_PRECONDITION): Unit = + logger.warn(message) + responseObserver.onNext(ResumeResponse(status = Some(Status(code = code.value, message = message)))) + responseObserver.onCompleted() - val status: Status = Status(code = Code.OK.index) - Future.successful(ResumeResponse(status = Some(status))) + // The same trust boundary the start-from counts cross: a nonsense number is refused loudly + // here, never clamped into a guess about what the client meant. 0 is the documented + // "unlimited" / "no budget", so only genuinely negative values are nonsense. + if request.maxMessagesPerSecond < 0 then + answerAndClose( + s"max_messages_per_second must not be negative, got ${request.maxMessagesPerSecond}. 0 means unlimited.", + Code.INVALID_ARGUMENT + ) + return + if request.maxMessagesToDeliver < 0 then + answerAndClose( + s"max_messages_to_deliver must not be negative, got ${request.maxMessagesToDeliver}. 0 means no delivery budget.", + Code.INVALID_ARGUMENT + ) + return + + // Serialized with create and delete under the same name. Resume used to take no lock at + // all, so it could read the runner while a delete was stopping it and wire the fresh + // observer into a stream the stop was about to complete - the play stream then hung + // silently, every later send swallowed by the terminal gate, with nothing telling the + // client why. + withLifecycleLock(sessionName) { + Option(consumerSessions.get(sessionName)) match + case None => + answerAndClose(s"No such consumer consumer session: $sessionName") + case Some(consumerSession) if consumerSession.isStreamCompleted => + // Stopped - deleted, replaced, or failed - but still reachable. The terminal + // gate is sticky, so wiring the observer in would swallow every response; + // the client's remedy is to create the session again. + answerAndClose(s"Consumer session $sessionName is closed and cannot be resumed. Create the session again.") + // THE CANCELLATION THAT ARRIVED FIRST. The cancel callback above has to be + // installed before this lock is taken - gRPC only allows handler registration + // early - and the observer is wired into the runner only inside the resume below. + // A call that dies in that window (or while another lifecycle operation holds this + // lock) runs `onPlayStreamCancelled` against a runner this observer was never + // wired into: the compare-and-clear finds no match, answers false, and does + // nothing. Resuming anyway would then wire an ALREADY-DEAD stream, open every + // target's intake and clear the idle clock - so messages fail into a stream nobody + // holds and are nacked forever, while the janitor can no longer see the session as + // abandoned and its subscription pins broker resources for the life of the process. + // + // Rechecked HERE, under the lifecycle lock, which is what makes the window empty: + // a cancellation landing after this read must queue behind us for the lock and + // will then find the observer wired. Nothing is answered on the observer - the + // call is gone - and the session is left exactly as it was, so a predecessor play + // stream (a second tab) keeps running and an idle session stays reapable. + case Some(consumerSession) if isCancelled(responseObserver) => + logger.info( + s"The play stream for consumer session $sessionName was cancelled before it could be wired; " + + "leaving the session as it was." + ) + case Some(consumerSession) => + try + // BOTH request flags, not just the debug one: `include_consumer_stats` is + // the client saying whether it can handle consumer stats at all - including + // the message-less progress frames a skip in flight pushes - and it used to + // be read and then ignored. + consumerSession.resume( + grpcResponseObserver = responseObserver, + isDebug = request.isDebug, + includeConsumerStats = request.includeConsumerStats, + maxMessagesPerSecond = request.maxMessagesPerSecond, + maxMessagesToDeliver = request.maxMessagesToDeliver + ) + catch + case err: Throwable => + // THROUGH THE RUNNER, never straight to the observer: a late target's + // throw leaves the earlier targets' listeners live and pushing into + // this same observer, so the status frame and the completion must go + // through the runner's send lock and set its terminal flag. Written + // directly they interleaved with a push, and every push after the + // onCompleted landed in a completed stream. + consumerSession.failAndComplete(Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage)) + // And then STOP THE INTAKE: the targets resumed before the throw have + // open gates and running consumers, and with the stream terminal their + // output is suppressed - they were consuming and ACKNOWLEDGING messages + // nobody would ever see, until the session was recreated. Pausing + // closes the gates (buffered messages are handed back, not swallowed) + // and stops the consumers; best-effort, because the failed target may + // be in any state. + Try(consumerSession.pause()) + () + } override def pause(request: PauseRequest): Future[PauseResponse] = val sessionName = request.consumerName logger.info(s"Pausing consumer session $sessionName") - val consumerSession = Option(consumerSessions.get(sessionName)) match - case Some(consumerSession) => consumerSession - case _ => - val msg = s"No such consumer consumer session: $sessionName" - logger.warn(msg) - - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = msg) - return Future.successful(PauseResponse(status = Some(status))) - - try { - consumerSession.pause() - } catch { - case err: Throwable => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) - return Future.successful(PauseResponse(status = Some(status))) - } + // ADVISORY FIRST, before the lifecycle lock: close the intake the moment the user asks. + // The serialized pause below can queue behind a resume still arming a wide session (or the + // idle janitor holding this name), and for that whole wait the session kept delivering into + // a browser whose user had already clicked Pause. Pausing is idempotent and every step is + // safe against a runner in any state (gates are flags, arbiter holds are per-reason, a + // consumer mid-close just fails its hold) - so this is best-effort and SILENT: the + // serialized phase below remains the authoritative answer, re-pausing whatever a concurrent + // resume may have re-opened in between, and ITS verdict is what the client hears. Nothing + // is lost in the window either way: a closed gate refuses and hands back for redelivery, + // and the limiter's backlog waits unacknowledged. + Option(consumerSessions.get(sessionName)).foreach(session => Try(session.pause())) - val status: Status = Status(code = Code.OK.index) + // Serialized with create, delete and resume under the same name. Pause used to take no + // lock, so on a multi-target session it could interleave TARGET BY TARGET with a resume - + // both answering OK while half the targets finished paused and half running, with the + // limiter believing whichever call it heard last. The browser makes that race ordinary: a + // quick hidden-then-visible tab abandons its pending pause and immediately resumes. The + // advisory above narrows that race to a transient (final state is decided HERE, last in + // the lock queue); it does not replace the serialization. + val status: Status = withLifecycleLock(sessionName) { + Option(consumerSessions.get(sessionName)) match + case None => + val msg = s"No such consumer consumer session: $sessionName" + logger.warn(msg) + Status(code = Code.FAILED_PRECONDITION.value, message = msg) + case Some(consumerSession) => + Try(consumerSession.pause()) match + case Success(_) => Status(code = Code.OK.value) + case Failure(err) => Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) + } Future.successful(PauseResponse(status = Some(status))) + /** Change the delivery order of a session that is ALREADY RUNNING - the actionable half of the + * Guaranteed stall disclosure. + * + * A DEDICATED RPC rather than a flag on Resume: this mutates a live session, and a + * session-mutating operation must be asked for explicitly instead of riding along with + * pressing Play. It changes the LIVE session only; the saved configuration is the client's to + * update if it wants the next Play to start this way. + * + * Serialized under the same per-name lifecycle lock as create, delete, pause and resume: the + * switch releases held messages down the ordinary delivery path, so it must not interleave + * with a resume rewiring the observers those messages are about to be written to, or with a + * delete stopping the runner underneath it. Inside the runner it then takes the session's + * ORDERING lock, which is the same order `resume` already establishes (lifecycle lock, then + * ordering lock, then the merge's own monitor) - no new lock ordering is introduced. + */ + override def setDeliveryOrder(request: SetDeliveryOrderRequest): Future[SetDeliveryOrderResponse] = + val sessionName = request.consumerName + + def answer(code: Code, message: String = ""): Future[SetDeliveryOrderResponse] = + if code != Code.OK then logger.warn(message) + Future.successful(SetDeliveryOrderResponse(status = Some(Status(code = code.value, message = message)))) + + // THE WIRE VALUE IS CHECKED BEFORE THE CONVERSION, and that is load-bearing. + // `MessageDeliveryOrder.fromPb` resolves both UNSPECIFIED and an unrecognized future value + // to the product default (Guaranteed) - right for a session CONFIGURATION, where absence + // means "the product default", and exactly wrong for a MUTATION, where it would turn + // "the client said nothing" into "the client asked to switch to Best effort". + request.messageDeliveryOrder match + case consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED => + answer(Code.INVALID_ARGUMENT, s"message_delivery_order names no order. Say which order session $sessionName should switch to.") + case unknown: consumerPb.MessageDeliveryOrder.Unrecognized => + answer( + Code.INVALID_ARGUMENT, + s"message_delivery_order ${unknown.unrecognizedValue} is not a delivery order this server knows. Upgrade the server, or choose a known order." + ) + case wire => + val requested = MessageDeliveryOrder.fromPb(wire) + logger.info(s"Setting the delivery order of consumer session $sessionName to $requested") + + val status: Status = withLifecycleLock(sessionName) { + Option(consumerSessions.get(sessionName)) match + case None => + val msg = s"No such consumer session: $sessionName" + logger.warn(msg) + Status(code = Code.FAILED_PRECONDITION.value, message = msg) + case Some(consumerSession) if consumerSession.isStreamCompleted => + val msg = s"Consumer session $sessionName is closed; its delivery order cannot be changed. Create the session again." + logger.warn(msg) + Status(code = Code.FAILED_PRECONDITION.value, message = msg) + case Some(consumerSession) => + Try(consumerSession.setDeliveryOrder(requested)) match + case Success(_) => Status(code = Code.OK.value) + case Failure(err) => + logger.warn(s"Refused to change the delivery order of consumer session $sessionName. ${err.getMessage}") + Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) + } + Future.successful(SetDeliveryOrderResponse(status = Some(status))) + + /** Why this CREATE REQUEST is malformed, or `None` - everything decidable from the request + * ALONE, before a single broker call. + * + * These same rules are also enforced inside `ConsumerSessionRunner.make`, and deliberately so: + * there they guard the build, here they let the refusal be classified honestly. A request that + * asks for a negative skip, a fraction of NaN, more targets than a session may run, or a start + * position that cannot describe a compacted target is WRONG - retrying it unchanged can never + * work - and answering FAILED_PRECONDITION told the client the opposite. Deciding it here also + * means a malformed request never reaches the broker at all. + */ + private def malformedRequestReason(sessionName: ConsumerSessionName, sessionConfig: ConsumerSessionConfig): Option[String] = + val enabledTargets = sessionConfig.targets.filter(_.isEnabled) + // Indexed over ENABLED targets, exactly as `make` counts them - a disabled target is not + // part of this session and its consumption mode is not this session's contradiction. + val readCompactedTargetIndexes = enabledTargets.zipWithIndex.collect { + case (targetConfig, targetIndex) if targetConfig.consumptionMode.mode + .isInstanceOf[_root_.consumer.session_target.consumption_mode.modes.ReadCompactedConsumptionMode] => + targetIndex + } + ConsumerSessionRunner + .enabledTargetCountRejectionReason(sessionName, enabledTargets.size) + .orElse(consumer.session_runner.startFromCountRejectionReason(sessionConfig.startFrom)) + .orElse(consumer.session_runner.startFromFractionRejectionReason(sessionConfig.startFrom)) + .orElse(consumer.session_runner.readCompactedStartFromRejectionReason(sessionConfig.startFrom, readCompactedTargetIndexes)) + override def createConsumer(request: CreateConsumerRequest): Future[CreateConsumerResponse] = + val sessionName = request.consumerName + + def invalidArgument(message: String): Future[CreateConsumerResponse] = + logger.warn(s"Refusing to create consumer session $sessionName: $message") + Future.successful(CreateConsumerResponse(status = Some(Status(code = Code.INVALID_ARGUMENT.value, message = message)))) + + // THE REQUEST IS JUDGED BEFORE THE WORLD IS. Everything below this point can fail because + // of the broker or of what this server is already running, and that is FAILED_PRECONDITION; + // everything above it is the request itself, and that is INVALID_ARGUMENT. They used to be + // one status, so a client could not tell "fix your request" from "try again later". + request.consumerSessionConfig match + case None => invalidArgument("consumer_session_config is required, but the request carried none.") + case Some(configPb) => + Try(ConsumerSessionConfig.fromPb(configPb)) match + case Failure(err) => + invalidArgument(s"consumer_session_config could not be read. ${Option(err.getMessage).getOrElse(err.getClass.getSimpleName)}") + case Success(sessionConfig) => + malformedRequestReason(sessionName, sessionConfig) match + case Some(reason) => invalidArgument(reason) + case None => buildConsumerSession(sessionName, sessionConfig) + + /** The half that can fail because of the broker, the topology or this server's own load. Every + * failure here is FAILED_PRECONDITION: the request was fine, the world was not. */ + private def buildConsumerSession( + sessionName: ConsumerSessionName, + sessionConfig: ConsumerSessionConfig + ): Future[CreateConsumerResponse] = Try { - val sessionName = request.consumerName logger.info(s"Creating consumer session. $sessionName") - val pulsarClient = RequestContext.pulsarClient.get() - val adminClient = RequestContext.pulsarAdmin.get() + withLifecycleLock(sessionName) { + // ADMISSION BEFORE THE BUILD, and the permit is held across ALL of it. The check + // used to sit between the predecessor's removal and the build, reading a map size + // that every concurrent create of a different name read identically - so the cap + // bounded only what had already finished building, which is nothing anybody is + // protected by. `allowance = 1` for a replacement: its predecessor is stopped just + // below and does not count against the total this build will produce. + val replacing = consumerSessions.containsKey(sessionName) + if !reserveAdmission(allowance = if replacing then 1 else 0) then + throw new RuntimeException( + s"This server is already running $admittedSessionCount consumer sessions, its limit. " + + "Delete sessions you no longer need (abandoned ones expire on their own) and try again." + ) - val consumerSession = ConsumerSessionRunner.make( - sessionName = sessionName, - pulsarClient = pulsarClient, - adminClient = adminClient, - sessionConfig = ConsumerSessionConfig.fromPb(request.consumerSessionConfig.get) - ) + try + // STOP THE PREDECESSOR FIRST. The replacement subscribes under the very same + // exclusive, non-durable subscription, so on the same topic a still-live + // predecessor refuses it - and the failure arrived long before the atomic map + // swap that was supposed to release it. The browser re-creates a session + // whenever its configuration changes, so this was the ordinary path, not a + // corner. + // + // Removing before building also means a build that FAILS leaves the name empty + // rather than leaving the old session running behind a client that has moved + // on: the user asked for this name to hold something else, and is told it does + // not. + Option(consumerSessions.remove(sessionName)).foreach(previous => + Try(previous.stop()).failed.foreach(err => + logger.warn(s"The consumer session being replaced under $sessionName could not be fully released. ${err.getMessage}") + ) + ) - consumerSessions.put(sessionName, consumerSession) + // Still not a bare `put`: nothing but this lock stands between two creates, and + // a predecessor that somehow survives one must be stopped rather than + // abandoned. + storeConsumerSession(consumerSessions, sessionName, makeSession(sessionName, sessionConfig)) + finally + // The permit goes back LAST, after the runner is in the map - the map entry is + // what counts from here. Releasing it first would open a window in which the + // session is admitted by nothing at all. On the failure path nothing was + // installed, so this is the release that keeps a refused build from + // permanently costing the server a session slot. + buildsInFlight.decrementAndGet() + () + } } match case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(CreateConsumerResponse(status = Some(status))) case Failure(err) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateConsumerResponse(status = Some(status))) override def deleteConsumer(request: DeleteConsumerRequest): Future[DeleteConsumerResponse] = val sessionName = request.consumerName logger.info(s"Deleting consumer session: $sessionName") - val consumerSession = Option(consumerSessions.get(sessionName)) match - case Some(consumerSession) => consumerSession - case _ => - val msg = s"No such consumer session: $sessionName" - logger.warn(msg) - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = msg) - return Future.successful(DeleteConsumerResponse(status = Some(status))) - - try { - consumerSession.stop() - consumerSessions.remove(sessionName) - } catch { - case err: Throwable => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) - return Future.successful(DeleteConsumerResponse(status = Some(status))) + // Serialized against creates under the same name: reading the runner, stopping it (a broker + // round trip) and removing the entry have to be one operation. + val stopped = withLifecycleLock(sessionName) { + Option(consumerSessions.get(sessionName)) match + case None => + val msg = s"No such consumer session: $sessionName" + logger.warn(msg) + Left(msg) + case Some(consumerSession) => + val outcome = Try(consumerSession.stop()) + + // The handle goes WHATEVER stopping did. `stop` releases everything it can + // before it reports, so keeping the entry after a partial failure would leave a + // session that nothing can reach and nothing can retry - and the previous order + // (remove only on success) meant a broker that refused one unsubscribe made the + // session name permanently undeletable. + // + // COMPARE-AND-REMOVE, not a bare remove: only the session this call actually + // stopped may be unhooked. An unconditional remove would silently drop a + // replacement installed meanwhile, leaving it running and unreachable. + consumerSessions.remove(sessionName, consumerSession) + Right(outcome) } - val status: Status = Status(code = Code.OK.index) - Future.successful(DeleteConsumerResponse(status = Some(status))) + val outcome = stopped match + case Left(msg) => + return Future.successful(DeleteConsumerResponse(status = Some(Status(code = Code.FAILED_PRECONDITION.value, message = msg)))) + case Right(outcome) => outcome + + outcome match + case Success(_) => + Future.successful(DeleteConsumerResponse(status = Some(Status(code = Code.OK.value)))) + case Failure(err) => + logger.warn(s"Consumer session $sessionName was removed but could not be fully released. ${err.getMessage}") + Future.successful(DeleteConsumerResponse(status = Some(Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage)))) override def runCode(request: RunCodeRequest): Future[RunCodeResponse] = val consumerSession = Option(consumerSessions.get(request.consumerName)) match case Some(consumerSession) => consumerSession case _ => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = s"Consumer isn't found: ${request.consumerName}") + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = s"Consumer isn't found: ${request.consumerName}") return Future.successful(RunCodeResponse(status = Some(status))) - val result = consumerSession.sessionContextPool.getContext(0).runCode(request.code) + // Leased, not grabbed: this runs on a gRPC thread while the session's listener threads are + // using the SAME context, so a raw handle here answered the user's expression with + // "[ERROR] Multi threaded access requested by thread ..." whenever the two overlapped. + val result = consumerSession.sessionContextPool.withContext(0)(_.runCode(request.code)) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) val response = RunCodeResponse(status = Some(status), result = Some(result)) Future.successful(response) @@ -159,9 +740,118 @@ class ConsumerServiceImpl extends ConsumerServiceGrpc.ConsumerService: topicSelector.getNonPartitionedTopics(adminClient) } match case Success(topicFqns) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) val response = ResolveTopicSelectorResponse(status = Some(status), topicFqns = topicFqns) Future.successful(response) case Failure(err) => - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = err.getMessage) + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = err.getMessage) Future.successful(ResolveTopicSelectorResponse(status = Some(status))) + + /** Read one physical topic's endpoints and entry count, and pair them with how far the session + * has read. + * + * THREE ADMIN ROUND TRIPS, each O(1) in the size of the topic: the first entry, the last entry, + * and the internal stats. That per-topic cost is the whole reason this is polled on demand + * behind a client-side switch instead of riding along with the message stream - a session over + * a hundred partitions pays it a hundred times per refresh. + * + * EVERY LOOKUP IS ALLOWED TO DECLINE, and the two ways it can decline mean different things: + * + * - `brokerAnswer` returns `None` for "the broker says this log is empty" - `examineMessage` + * THROWS for "latest" on an empty topic where "earliest" answers 412 - and that is not an + * error. The row simply has no endpoints. + * - A topic Pulsar refuses to examine at all answers with a REASON. A non-persistent topic is + * the ordinary case (405): it retains nothing to examine. Distinguishing the two is why + * `unavailable_reason` exists rather than leaving the client to guess from blank cells. + */ + private def gatherTopicPosition( + adminClient: PulsarAdmin, + topicFqn: String, + consumedBounds: Option[TopicConsumedBounds] + ): TopicPositionRow = + Try { + val first = brokerAnswer("examining the first entry", topicFqn)( + adminClient.topics.examineMessage(topicFqn, "earliest", 1) + ).map(message => LogEndpoint(entryIdOf(message.getMessageId), message.getPublishTime)) + + val last = brokerAnswer("examining the last entry", topicFqn)( + adminClient.topics.examineMessage(topicFqn, "latest", 1) + ).map(message => LogEndpoint(entryIdOf(message.getMessageId), message.getPublishTime)) + + val stats = adminClient.topics.getInternalStats(topicFqn) + val ledgers = stats.ledgers.asScala.toVector.map(info => LedgerSpan(info.ledgerId, info.entries)) + + TopicPositionInputs( + topicFqn = topicFqn, + first = first, + last = last, + firstConsumed = consumedBounds.map(_.first), + cursor = consumedBounds.map(_.last), + ledgers = ledgers, + currentLedgerEntries = stats.currentLedgerEntries, + retainedEntries = stats.numberOfEntries, + unavailableReason = None + ) + } match + case Success(inputs) => buildTopicPositionRow(inputs) + case Failure(err) => + // Report the topic with a reason rather than failing the whole call: one + // non-persistent topic in a multi-topic session must not blank the other rows. + logger.debug(s"Topic positions unavailable for $topicFqn. ${err.getMessage}") + buildTopicPositionRow( + TopicPositionInputs( + topicFqn = topicFqn, + first = None, + last = None, + firstConsumed = consumedBounds.map(_.first), + cursor = consumedBounds.map(_.last), + ledgers = Vector.empty, + currentLedgerEntries = 0, + retainedEntries = 0, + unavailableReason = Some(Option(err.getMessage).getOrElse(err.getClass.getSimpleName)) + ) + ) + + override def getTopicPositions(request: GetTopicPositionsRequest): Future[GetTopicPositionsResponse] = + val sessionName = request.consumerName + + // The session lookup comes FIRST, and the admin client is fetched only once there is + // something to ask about. A client polling this tab before the session has been started is + // the ordinary case, not an error path, and resolving the request context up here made even + // that answer impossible to produce - or to test - without a broker behind it. + // + // A TERMINAL RUNNER IS "GONE" TOO, and answering that is what lets a polling client stop. + // The tab polls once a second for as long as it believes the session exists, and a runner + // whose stream has ended - stopped, replaced, or failed out of a resume - is unreachable + // for good (the terminal gate is sticky; the client's remedy is to create the session + // again). It stays in the map until delete or the janitor removes it, and while it did the + // poll answered OK and went on scanning the broker on its behalf indefinitely. + Option(consumerSessions.get(sessionName)).filterNot(_.isStreamCompleted) match + case None => + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = s"No such consumer session: $sessionName") + Future.successful(GetTopicPositionsResponse(status = Some(status))) + case Some(consumerSession) => + // RESOLVED HERE, ON THE gRPC THREAD, and passed into the scan: the request context + // is a thread-local, so it exists only for as long as this call is on this thread. + val adminClient = RequestContext.pulsarAdmin.get() + + // One row per PHYSICAL topic, which is what the session actually subscribes to and + // the only thing `examineMessage` will answer for - it refuses a partitioned parent + // outright (405). + val consumedBounds = mergeConsumedBounds(consumerSession.targets.values.map(_.consumerListener.consumedBounds)) + val topicFqns = consumerSession.targets.values.flatMap(_.consumers.keys).toVector.distinct.sorted + + // OFF THIS THREAD, BOUNDED, AND COALESCED - see [[TopicPositionScanner]]. The rows + // used to be gathered serially, right here, before an already-completed Future was + // returned: a 2,000-topic session held a service thread for about 6,000 admin calls + // that no client deadline could cancel, and the tab's once-a-second poll started + // another sweep on top of every sweep still running. + topicPositionScans + .scan(sessionName, topicFqns, topicFqn => gatherTopicPosition(adminClient, topicFqn, consumedBounds.get(topicFqn))) + .transform { + case Success(rows) => + Success(GetTopicPositionsResponse(status = Some(Status(code = Code.OK.value)), positions = rows.map(topicPositionToPb))) + case Failure(err) => + logger.warn(s"Could not read topic positions for $sessionName. ${err.getMessage}") + Success(GetTopicPositionsResponse(status = Some(Status(code = Code.UNKNOWN.value, message = err.getMessage)))) + }(scala.concurrent.ExecutionContext.parasitic) diff --git a/server/src/main/scala/consumer/atProtoField.scala b/server/src/main/scala/consumer/atProtoField.scala new file mode 100644 index 000000000..8e2e5a85d --- /dev/null +++ b/server/src/main/scala/consumer/atProtoField.scala @@ -0,0 +1,19 @@ +package consumer + +/** Convert one protobuf field, NAMING THE PATH when it fails. + * + * A consumer session config is a tree - targets, each with a selector, a consumption mode, a + * deserializer and three chains - and every one of those conversions refuses an absent or unknown + * oneof by throwing. What reached the client was the innermost message alone: "Invalid + * ConsumerSessionTargetConsumptionMode mode." over a session with nine targets, with nothing + * saying which. Wrapping each conversion in its field's name composes into the path + * (`targets[3]: consumption_mode: ...`), which is the difference between a session the user can + * repair and one they can only rebuild. + * + * The original is kept as the cause, so nothing about the failure is lost. + */ +def atProtoField[A](path: String)(convert: => A): A = + try convert + catch + case err: Throwable => + throw new IllegalArgumentException(s"$path: ${Option(err.getMessage).getOrElse(err.getClass.getSimpleName)}", err) diff --git a/server/src/main/scala/consumer/deserializer/deserializers/UseLatestTopicSchema.scala b/server/src/main/scala/consumer/deserializer/deserializers/UseLatestTopicSchema.scala index ce5f63725..22b0268a4 100644 --- a/server/src/main/scala/consumer/deserializer/deserializers/UseLatestTopicSchema.scala +++ b/server/src/main/scala/consumer/deserializer/deserializers/UseLatestTopicSchema.scala @@ -5,7 +5,7 @@ import io.circe.generic.auto.* import _root_.schema.{avro, protobufnative} import com.tools.teal.pulsar.ui.api.v1.consumer as pb import _root_.consumer.deserializer.Deserializer -import _root_.consumer.session_runner.{MessageAsJsonOmittingValue, MessageValueAsJson, SchemasByTopic} +import _root_.consumer.session_runner.{MessageAsJsonOmittingValue, MessageValueAsJson, SchemasByTopic, SchemasByVersion} import org.apache.pulsar.common.schema.SchemaType import org.apache.pulsar.client.api.Message import _root_.conversions.primitiveConv.* @@ -42,14 +42,20 @@ object UseLatestTopicSchema: case Some(v) => bytesToInt64(v).toOption case None => None - val schemaInfo = - if msgSchemaVersion.isEmpty - then return Right(bytesToJsonString(msgData)) - else schemasByVersion.get.get(msgSchemaVersion.get) match - case Some(si) => si - case None => return Right(bytesToJsonString(msgData)) + // The schema the MESSAGE names, when the registry holds that version: always the most + // faithful way to read that particular message, and unchanged behaviour. + val exactSchema = msgSchemaVersion.flatMap(schemasByVersion.get.get) - schemaInfo.getType match + // Otherwise the topic's LATEST registered schema - the case this deserializer is NAMED + // for. A producer using Schema.BYTES (or an older client) stamps no schema version, so + // only the registry says what the bytes are; falling straight through to + // `bytesToJsonString` rendered a JSON document as an escaped JSON STRING instead of the + // object. `SchemasByVersion.getLatest` was written for this and had no caller. + val schemaInfo = exactSchema.orElse(SchemasByVersion.getLatest(schemasByVersion.get)) match + case Some(si) => si + case None => return Right(bytesToJsonString(msgData)) + + val decoded = schemaInfo.getType match case SchemaType.AVRO => avro.converters.toJson(schemaInfo.getSchema, msgData).map(String(_, StandardCharsets.UTF_8)) case SchemaType.JSON => msgData match @@ -70,3 +76,11 @@ object UseLatestTopicSchema: // case SchemaType.BYTES => the message schema version is empty in this case and is handled in the code above // case SchemaType.NONE => the message schema version is empty in this case and is handled in the code above case _ => Left(new Exception("Can't convert bytes to json")) + + // A schema the message did not name is an INFERENCE, so it must never turn a readable + // payload into an error: if decoding under the latest schema fails, keep the raw-string + // answer this path used to give unconditionally. A schema the message DID name is the + // message's own truth, and its failures stay visible. + if exactSchema.isEmpty && decoded.isLeft + then Right(bytesToJsonString(msgData)) + else decoded diff --git a/server/src/main/scala/consumer/session_config/ConsumerSessionConfig.scala b/server/src/main/scala/consumer/session_config/ConsumerSessionConfig.scala index e94c95171..741b45c09 100644 --- a/server/src/main/scala/consumer/session_config/ConsumerSessionConfig.scala +++ b/server/src/main/scala/consumer/session_config/ConsumerSessionConfig.scala @@ -7,6 +7,97 @@ import _root_.consumer.message_filter.MessageFilterChain import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain import _root_.consumer.coloring_rules.ColoringRuleChain import _root_.consumer.value_projections.ValueProjectionList +import _root_.consumer.atProtoField + +/** How a session that reads more than one DELIVERY STREAM interleaves processing across them - + * several topics, one partitioned topic, or overlapping enabled targets on a single topic all + * count, because each resolves to its own consumer. */ +enum MessageDeliveryOrder: + /** Every topic streams independently; messages interleave as they arrive. Fastest. */ + case AsReceived + + /** One merged stream ordered by the selected timestamp, BEST EFFORT: a bounded reorder + * grace, worn openly - see `mergeTopicsGraceMs`. Never stalls on a silent stream, at the + * price of admitting late messages out of order rather than dropping them. THE PRODUCT + * DEFAULT: what UNSPECIFIED resolves to, and what a session that never named an order runs + * under (owner decision 2026-08-09, superseding the earlier Guaranteed default). + * + * NOT publish-time-specific, whatever the wire constant is called: the timestamp compared is + * whichever [[DeliveryOrderKey]] the session selected. The protobuf symbol + * `MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME` (value 2) predates the selectable key + * and is kept exactly as it is for wire and source compatibility - it is a name, not a + * behavior, and renumbering or aliasing it would buy nothing and cost every saved item. */ + case BestEffort + + /** One merged stream ordered by the selected timestamp: the exact k-way merge + * of the source logs, with NO liveness escape - a silent stream holds delivery until it + * speaks, failed sends retry in place, nothing is ever abandoned. The session introduces + * zero disorder of its own; only inversions already stored inside a single source log pass + * through, in append order, counted. AN EXPLICIT CHOICE, never a fallback: an empty or idle + * partition holds delivery indefinitely, which is the mode's whole promise - so the wait is + * DISCLOSED (the waiting-stream count rides the stats channel and surfaces on the toolbar + * chip) and the chip offers a one-click switch to Best effort. */ + case Guaranteed + +object MessageDeliveryOrder: + def fromPb(v: pb.MessageDeliveryOrder): MessageDeliveryOrder = v match + case pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED => MessageDeliveryOrder.AsReceived + case pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME => MessageDeliveryOrder.BestEffort + case pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED => MessageDeliveryOrder.Guaranteed + // Zero is both proto3 absence and UNSPECIFIED, including requests from older clients and + // every config saved before the field existed. A session that never NAMED an order gets + // the product default - Guaranteed: an exact replay of recorded history that auto-pauses + // when caught up, with the caught-up panel offering the ways on. Owner decision + // (2026-08-11, direct instruction), superseding the 2026-08-09 Best effort default - + // this default has now moved twice, so the decision log in the plan file is the record. + case pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED => MessageDeliveryOrder.Guaranteed + // An old server cannot implement a future mode; use its current product default instead + // of accidentally selecting Fastest. The explicit match keeps this distinct from zero. + case _: pb.MessageDeliveryOrder.Unrecognized => MessageDeliveryOrder.Guaranteed + + def toPb(v: MessageDeliveryOrder): pb.MessageDeliveryOrder = v match + case MessageDeliveryOrder.AsReceived => pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED + case MessageDeliveryOrder.BestEffort => pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + case MessageDeliveryOrder.Guaranteed => pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + +/** Which per-message timestamp the delivery order compares. Not Pulsar's message ordering key. */ +enum DeliveryOrderKey: + /** Added automatically by the producer; always present. The default. */ + case PublishTime + + /** Broker entry metadata recording when an entry arrived. Present only when the broker stamps + * and exposes that metadata; an unstamped message uses publish time and is counted. */ + case BrokerPublishTime + + /** An optional timestamp set by the application; a message carrying none (0) uses publish + * time and is counted as a fallback. */ + case EventTime + +object DeliveryOrderKey: + def fromPb(v: pb.DeliveryOrderKey): DeliveryOrderKey = v match + case pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME => DeliveryOrderKey.BrokerPublishTime + case pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME => DeliveryOrderKey.EventTime + // Zero is proto3 absence: every older client, and every config saved before the field + // existed. Publish time is the documented default and is always present on a message. + case pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME | pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_UNSPECIFIED => + DeliveryOrderKey.PublishTime + // A VALUE THIS SERVER DOES NOT KNOW IS NOT THE DEFAULT. The three keys order a merged + // session by three different clocks, so quietly answering "publish time" to a client that + // asked for a fourth delivers a session ordered by something it never chose, with nothing + // saying so. Unlike MessageDeliveryOrder - where the product decided an unknown value + // resolves to the product default rather than accidentally selecting Fastest - there is no + // safe substitute for a timestamp: it is refused. Callers at a request boundary turn this + // into INVALID_ARGUMENT. + case unknown: pb.DeliveryOrderKey.Unrecognized => + throw new IllegalArgumentException( + s"delivery_order_key ${unknown.unrecognizedValue} is not a timestamp this server knows. " + + "Upgrade the server, or choose publish time, broker publish time or event time." + ) + + def toPb(v: DeliveryOrderKey): pb.DeliveryOrderKey = v match + case DeliveryOrderKey.PublishTime => pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME + case DeliveryOrderKey.BrokerPublishTime => pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME + case DeliveryOrderKey.EventTime => pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME case class ConsumerSessionConfig( startFrom: ConsumerSessionStartFrom, @@ -14,24 +105,38 @@ case class ConsumerSessionConfig( messageFilterChain: MessageFilterChain, coloringRuleChain: ColoringRuleChain, pauseTriggerChain: ConsumerSessionPauseTriggerChain, - valueProjectionList: ValueProjectionList + valueProjectionList: ValueProjectionList, + messageDeliveryOrder: MessageDeliveryOrder = MessageDeliveryOrder.Guaranteed, + deliveryOrderKey: DeliveryOrderKey = DeliveryOrderKey.PublishTime ) object ConsumerSessionConfig: + /** EVERY FIELD IS CONVERTED UNDER ITS OWN NAME, so a malformed one is reported as a path rather + * than as whatever the innermost conversion happened to say - see [[atProtoField]]. A session + * with nine targets used to fail with a bare "Invalid ConsumerSessionTargetConsumptionMode + * mode." and no way to tell which target was at fault. */ def fromPb(v: pb.ConsumerSessionConfig): ConsumerSessionConfig = ConsumerSessionConfig( - startFrom = v.startFrom.map(ConsumerSessionStartFrom.fromPb).getOrElse(EarliestMessage()), - targets = v.targets.map(ConsumerSessionTarget.fromPb).toVector, - messageFilterChain = v.messageFilterChain - .map(MessageFilterChain.fromPb) - .getOrElse(MessageFilterChain.empty), - coloringRuleChain = v.coloringRuleChain - .map(ColoringRuleChain.fromPb) - .getOrElse(ColoringRuleChain.empty), - pauseTriggerChain = v.pauseTriggerChain - .map(ConsumerSessionPauseTriggerChain.fromPb) - .getOrElse(ConsumerSessionPauseTriggerChain.empty), - valueProjectionList = ValueProjectionList.fromPb(v.getValueProjectionList) + startFrom = atProtoField("start_from")(v.startFrom.map(ConsumerSessionStartFrom.fromPb).getOrElse(EarliestMessage())), + targets = v.targets.zipWithIndex.map((target, i) => atProtoField(s"targets[$i]")(ConsumerSessionTarget.fromPb(target))).toVector, + messageFilterChain = atProtoField("message_filter_chain")( + v.messageFilterChain + .map(MessageFilterChain.fromPb) + .getOrElse(MessageFilterChain.empty) + ), + coloringRuleChain = atProtoField("coloring_rule_chain")( + v.coloringRuleChain + .map(ColoringRuleChain.fromPb) + .getOrElse(ColoringRuleChain.empty) + ), + pauseTriggerChain = atProtoField("pause_trigger_chain")( + v.pauseTriggerChain + .map(ConsumerSessionPauseTriggerChain.fromPb) + .getOrElse(ConsumerSessionPauseTriggerChain.empty) + ), + valueProjectionList = atProtoField("value_projection_list")(ValueProjectionList.fromPb(v.getValueProjectionList)), + messageDeliveryOrder = atProtoField("message_delivery_order")(MessageDeliveryOrder.fromPb(v.messageDeliveryOrder)), + deliveryOrderKey = atProtoField("delivery_order_key")(DeliveryOrderKey.fromPb(v.deliveryOrderKey)) ) def toPb(v: ConsumerSessionConfig): pb.ConsumerSessionConfig = @@ -45,5 +150,7 @@ object ConsumerSessionConfig: mode = pb.ConsumerSessionPauseTriggerChainMode.CONSUMER_SESSION_PAUSE_TRIGGER_CHAIN_MODE_ALL ) ), - valueProjectionList = Some(ValueProjectionList.toPb(v.valueProjectionList)) + valueProjectionList = Some(ValueProjectionList.toPb(v.valueProjectionList)), + messageDeliveryOrder = MessageDeliveryOrder.toPb(v.messageDeliveryOrder), + deliveryOrderKey = DeliveryOrderKey.toPb(v.deliveryOrderKey) ) diff --git a/server/src/main/scala/consumer/session_runner/ConsumerListener.scala b/server/src/main/scala/consumer/session_runner/ConsumerListener.scala index 8d9a403cf..1f77d560e 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerListener.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerListener.scala @@ -1,27 +1,804 @@ package consumer.session_runner import com.typesafe.scalalogging.Logger -import org.apache.pulsar.client.api.MessageListener +import org.apache.pulsar.client.api.{MessageListener, MessageId as PulsarMessageId} + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import scala.jdk.CollectionConverters.* +import scala.util.{Failure, Success, Try} + +/** What this session has already decided about ONE broker entry's batch members, and whether the + * broker can still redeliver that entry. + * + * `decided` is the standing decision - delivered or dropped - that makes a redelivered copy pure + * paperwork. `unfinished` is the lifecycle half: members whose broker answer is not final yet, + * either because their acknowledgment is still in flight or because they were handed BACK. The + * broker retires an entry only once every one of its members has been acknowledged on one consumer + * incarnation, so an entry is redeliverable - and its record therefore load-bearing - until every + * member has been decided AND every one of those decisions has been acknowledged. + */ +private final class BatchEntryState: + val decided: java.util.BitSet = java.util.BitSet() + val unfinished: java.util.BitSet = java.util.BitSet() + + /** How many members this entry holds, as the broker's own id reports it, or -1 when the id does + * not carry it. Only ids built by hand lack it; every id a Pulsar consumer delivers has it. */ + var batchSize: Int = -1 + + /** This entry can no longer be redelivered - every member has been decided and answered for - + * so its record is dead weight. Without a batch size, completeness is unknowable and the + * answer falls back to "nothing outstanding", which is no weaker than having no record at all. + */ + def isSettled: Boolean = + unfinished.isEmpty && (batchSize <= 0 || decided.cardinality >= batchSize) + +object ConsumerListener: + /** How many SETTLED entry records to keep - entries nothing is owed for, kept only against a + * broker unload redelivering what it has not yet retired. Entries still owed a redelivery are + * exempt (see [[BatchEntryState]]), which is what makes this a cache size rather than the + * correctness bound it was mistaken for. */ + private[session_runner] val settledBatchEntryCap: Int = 512 + + /** How far one insertion may walk looking for settled records to reclaim. Keeps the per-message + * cost constant when the oldest records are all still owed; the next insertion continues. */ + private[session_runner] val settledBatchEntryEvictionWalk: Int = 64 + + /** What to do with a message the broker just delivered. */ + enum Action: + /** The session is paused - hand it back so it is redelivered on resume. */ + case Reject + + /** Consumed by the start-from discard - acknowledge it, but show it to nobody. */ + case Drop + + case Deliver class ConsumerListener(val targetMessageHandler: ConsumerSessionTargetMessageHandler) extends MessageListener[Array[Byte]] { val logger: Logger = Logger(getClass.getName) // https://levelup.gitconnected.com/graceful-shutdown-of-pulsar-queue-consumers-in-java-and-spring-boot-f93645a92b2b - private var isAcceptingNewMessages: Boolean = true + // + // STARTS CLOSED, and that is load-bearing. `handleStartFrom` RESUMES every consumer before it + // seeks them, so the broker starts delivering while the session is still being built - before + // the start-from counters and the global ordering layer are armed, and while the target's + // message handler is still the no-op it was constructed with. Accepting there consumed those + // messages and ACKNOWLEDGED them into nothing, so a session whose set-up did any broker round + // trip (the backward entry walk, or reading each topic's last message id) could swallow its + // whole backlog and then deliver nothing at all. Rejecting instead hands them straight back. + // + // ATOMIC, and that is load-bearing too. This gate is READ from one Pulsar listener thread per + // physical topic and WRITTEN from the gRPC threads that pause and resume the session. As a + // plain `var Boolean` there was no happens-before between the two at all, so a listener thread + // was entitled to go on seeing "accepting" indefinitely after the user had paused - delivering + // into a paused session, and spending its start-from budget while doing so. Nothing here needs + // compare-and-set; what it needs is the visibility. + private val acceptingNewMessages = AtomicBoolean(false) + + /** Armed ONCE by `ConsumerSessionRunner.make`, after the start-from seek and before anything is + * resumed. Nothing on the pause/resume path may reassign it: re-arming would skip a fresh + * batch of messages every time the user hits play. + */ + var startFromDiscard: StartFromDiscard = StartFromDiscard.none + + /** The session-wide layer that puts the delivered streams into their GLOBAL order, for the two + * start-from modes that are defined over the merged stream rather than over one log. + * + * Armed ONCE alongside [[startFromDiscard]], and the SAME instance on every target of the + * session - the counting is over the whole session, so a per-target layer would count each + * target separately. Nothing on the pause/resume path may reassign it, for the same reason. + */ + var startFromOrdering: StartFromOrdering[HeldMessage] = StartFromOrdering.passThrough + + /** The counter start-from progress is read off - the USER'S skip, and nothing else. + * + * A global skip merge owns its own budget - it has to, since it decides the drops itself - and + * that budget is what the client should be shown. Otherwise it is this listener's own counter, + * but ONLY when that counter is a user skip: a "latest n" arms a per-topic counter to drop the + * over-fetch its seek could not avoid, and showing that as progress told a client that asked + * for the last 5 messages it was "skipping 95". See [[StartFromDiscard.reportsProgress]]. + */ + def progressDiscard: StartFromDiscard = + if effectiveDiscard.reportsProgress then effectiveDiscard else StartFromDiscard.none + + /** The counter actually doing the dropping, whoever owns it: the merge when it decides the drops + * itself, this listener otherwise. + * + * Deliberately WIDER than [[progressDiscard]]. This is the mechanism - what diagnostics and + * tests ask "how many messages are still to be dropped" - and it must answer for a latest-n + * seek correction too. What the CLIENT is shown is the narrower one. + * + * `filter(_.total > 0)`: an ordering-ONLY layer carries a shared ZERO budget (it orders, it + * drops nothing), and letting that zero shadow the listener's own counter made a latest-n's + * per-topic overshoot report 0 remaining - so `startFromResolutionActive` went false, and the + * rate limiter's permit hold was un-suppressed in exactly the window the suppression exists + * to protect. A merge that was ARMED with a budget keeps answering even once it is spent - + * its zero then genuinely means "nothing left to drop". + */ + def effectiveDiscard: StartFromDiscard = startFromOrdering.progressDiscard.filter(_.total > 0).getOrElse(startFromDiscard) + + /** The consumed range for each physical topic this listener consumes, for the "Topic Positions" + * debug view. Read out of band by [[consumedBounds]] on a gRPC thread while listener threads + * write it, hence concurrent. + * + * RECORDED WHERE MESSAGES ARE ACKNOWLEDGED - on a Drop as well as a Deliver - because both mean + * the session consumed the message. The alternative, letting the client derive it from the rows + * on screen, is wrong by however much the session filters out: a filter passing 1% of a topic + * would show a session that had read to the end as barely started. + * + * OUTWARD-ONLY BOUNDS. `negativeAcknowledge` and the merge's cap both put messages back for + * redelivery, so an OLDER message can legitimately arrive after a newer one has been counted. + * That older message may widen `first`, but can never walk `last` backwards. One immutable + * value per topic also means an RPC cannot observe a new first paired with an old last. + */ + private val consumedBoundsByTopic = ConcurrentHashMap[String, TopicConsumedBounds]() + + /** Record one message that `topicFqn` has consumed. The range only widens - see + * [[consumedBoundsByTopic]]. + */ + def recordCursor(topicFqn: String, messageId: PulsarMessageId, publishTime: Long): Unit = + val incoming = TopicCursor(messageId, publishTime) + // `compute` avoids constructing and immediately discarding a one-element bounds object + // for every message after the first; the only steady-state allocations are the incoming + // cursor and the immutable replacement the atomic map publishes. + consumedBoundsByTopic.compute( + topicFqn, + (_, existing) => + if existing == null then TopicConsumedBounds(incoming, incoming) + else widenConsumedBounds(existing, incoming) + ) + () + + /** An atomic-per-topic snapshot of the consumed ranges. */ + def consumedBounds: Map[String, TopicConsumedBounds] = consumedBoundsByTopic.asScala.toMap + + /** Whether the pause gate is open. The permit-hold arbitration reads this so the delivery + * pacer's backpressure can never resume a consumer the USER paused. */ + def isAcceptingNewMessages: Boolean = acceptingNewMessages.get + + /** The session's delivery rate limiter, or None until a resume installs one. SHARED - every + * listener of the session holds the same instance, because the limit is per session, not per + * topic. Rewired on resume BEFORE the gate opens, like the message handler: the gate's + * volatile write is what publishes it to the listener threads. + */ + @volatile var deliveryRateLimiter: Option[DeliveryRateLimiter[HeldMessage]] = None + + /** Deliver one resolved message NOW, on the calling thread: hand it to the target's message + * handler, acknowledge it, and advance the read position. + * + * SELF-CONTAINED FAILURE HANDLING, because it has two callers with different surroundings: the + * resolved-batch loop (whose own catch would also cover it) and the rate limiter's drain + * thread (which has nothing above it). A failed delivery is handed back for redelivery - the + * same contract the loop pinned in round 3 - and costs only itself. + */ + def deliverNow(held: HeldMessage): Unit = + try + // The merge decided at emission whether this delivery undercuts an already-emitted + // key (Best effort's late emission); the stamp pairs with THIS call stack only. + held.listener.outOfOrderFlagPending.set(held.deliveredOutOfOrder) + try held.listener.targetMessageHandler.onNext(held.message) + finally held.listener.outOfOrderFlagPending.set(false) + acknowledge(held.consumer, held.message) + held.listener.recordCursor( + held.consumer.getTopic, + held.message.getMessageId, + held.message.getPublishTime + ) + catch + case err: Throwable => + logger.warn(s"Handing a message back for redelivery: delivering it failed. ${err.getMessage}") + // THE EXPLICIT LIFECYCLE TRANSITION the merge's duplicate arm consults: only a + // failure recorded here licenses delivering a watermarked copy as the retry. + failedDeliveries.add((held.consumer.getTopic, held.message.getMessageId)) + held.listener.noteBatchMemberHandedBack(held.consumer.getTopic, held.message.getMessageId) + Try(held.consumer.negativeAcknowledge(held.message)) + () + + /** Route one Deliver outcome: through the session's rate limiter when one is installed, + * straight through otherwise. + * + * WHILE THE ORDERING LOCK IS LIVE the offer is FORCED to be an enqueue. The unlimited + * (rate 0) path used to answer processNow and run the whole delivery - deserialization, the + * JS lease, the blocking gRPC write - inline on the offering thread with the session's + * ordering lock held, so one backpressured client parked every listener thread and the + * sweep with them. Forcing the queue moves the write to the limiter's single drainer, + * OUTSIDE the lock, and order is preserved by construction: the enqueue happens under the + * same lock that decided the order, the queue is FIFO, and the limiter refuses inline + * processing whenever a backlog or an in-flight drain exists - so a later message can never + * overtake a queued one. Once the merge settles (`serializesDeliveries` false), the + * unlimited inline fast path is back, exactly as before. + */ + def deliver(held: HeldMessage): Unit = deliveryRateLimiter match + case Some(limiter) => limiter.offer(held, forceQueueOnce = startFromOrdering.serializesDeliveries) + case None => deliverNow(held) + + /** Called once per message the discard swallows, so a skip in flight can be reported to the + * client - a dropped message reaches nothing else, so this is the only signal there is. + * + * A no-op until a client resumes the session, and rewired by + * `ConsumerSessionTargetRunner.resume` on every play. Unlike [[startFromDiscard]] there is + * nothing to preserve across a pause: it carries no state. + */ + var onStartFromDiscardProgress: () => Unit = () => () + + /** GUARANTEED replay: the chunk completed - every stream delivered its recorded range and + * the heap is drained. The runner wires this per play (it carries the play generation and + * a once-per-chunk claim) to the AUTO-PAUSE and the caught-up signal; a no-op until then. + * Invoked from [[pumpGuaranteedDelivery]] under the session's ordering lock. */ + var onReplayCaughtUp: () => Unit = () => () + + /** Whether the delivery currently running through this listener's target pipeline was emitted + * OUT OF ORDER - its order key undercuts one already emitted. Under GUARANTEED that is a + * seam violation (resume-seam clock skew, or a source-log inversion), set and cleared around + * each barrier send by [[pumpGuaranteedDelivery]]. Under BEST EFFORT (since 2026-08-11) it + * is a late emission - the message outlived its reorder window - and rides the HeldMessage + * from the merge through [[deliverNow]]. + * + * A THREAD LOCAL, not a plain flag: guaranteed sends are serialized by the ordering lock, + * but best-effort sends can run concurrently on different listener threads for the SAME + * target, and the stamp must pair with the delivery on ITS OWN call stack - the target + * pipeline reads it synchronously inside onNext. The target pipeline stamps it onto the + * outgoing row (Message.delivered_out_of_order). */ + val outOfOrderFlagPending: ThreadLocal[java.lang.Boolean] = ThreadLocal.withInitial(() => java.lang.Boolean.FALSE) + + /** Report a drop to the client, and NEVER let that reporting cost the message. + * + * The push ends in `StreamObserver.onNext`, which throws whenever the client's call has been + * cancelled. It used to be called between claiming the discard budget and acknowledging the + * message: the throw propagated out of `received`, so the budget had been spent on a message + * that was never acknowledged - the broker redelivered it, the spent budget let it through, + * and the session showed a message the user had asked to skip. Fewer than n unique messages + * were dropped, and nothing said so. + * + * Progress is a UI nicety. Losing it costs a progress bar; losing a message costs correctness. + */ + private def reportDiscardProgress(): Unit = + try onStartFromDiscardProgress() + catch + case err: Throwable => + logger.warn(s"Failed to report start-from progress; the skip itself is unaffected. ${err.getMessage}") def stopAcceptingNewMessages(): Unit = - this.isAcceptingNewMessages = false + acceptingNewMessages.set(false) def startAcceptingNewMessages(): Unit = - this.isAcceptingNewMessages = true + acceptingNewMessages.set(true) + + /** The whole decision, split out of `received` so it is testable without a broker - a paused + * session must NOT consume the discard (the message is coming back), and the discard must + * survive any number of pause/resume cycles. + * + * `canAcknowledge` is whether the consumer can still answer the broker for this message. A + * message that CANNOT be acknowledged must not be decided at all: claiming the discard for it + * spends a unit of the user's "skip the first n" on a message the broker will simply redeliver, + * and the redelivery then meets an empty budget and is SHOWN - so the session skips fewer than + * n unique messages while reporting that the skip completed. Handing it straight back costs + * one redelivery and nothing else. + * + * Reporting the drop happens HERE rather than in `received`, next to the claim that caused it: + * the claim is already the mutation this method performs, and keeping the two together is what + * makes "a rejected message does not count as skipped" testable without a broker. It goes + * through [[reportDiscardProgress]], so a failing report cannot abort the acknowledgment that + * has to follow the claim. + */ + def decide(topicFqn: NonPartitionedTopicFqn, canAcknowledge: Boolean): ConsumerListener.Action = + if !acceptingNewMessages.get then ConsumerListener.Action.Reject + else if !canAcknowledge then ConsumerListener.Action.Reject + else if startFromDiscard.claim(topicFqn) then + reportDiscardProgress() + ConsumerListener.Action.Drop + else ConsumerListener.Action.Deliver + + /** Messages this session DECIDED - dropped or delivered - whose acknowledgment the broker did + * not take. The decision stands; only the paperwork failed. The redelivery is met at the very + * top of [[received]]: acknowledged again and otherwise ignored, never re-decided. + * + * THIS SET IS WHAT MAKES THE COUNTED MODES EXACT UNDER A FAILED ACK. The alternatives were + * both wrong in a way three review rounds circled: REFUNDING the budget kept the count right + * but let the budget be spent on a DIFFERENT message, so the redelivered original was shown - + * "skip the first n" skipped some other n. NOT refunding kept the set right until the + * redelivery arrived after the budget closed and was shown anyway - n was simply wrong. The + * decision-stands-retry-the-ack rule keeps both: the unit stays spent on exactly the message + * it was claimed for, and that message can never reappear, however late the broker redelivers + * it. A DELIVERED message whose ack failed gets the same treatment, which also stops its + * redelivery from being shown twice. + * + * Bounded by the number of failed acknowledgments (rare - a disconnect race), and each entry + * leaves the moment its redelivery is finalized. + */ + /** streamId -> the consumer's pause arbiter, set by the target runner at build time. The + * merge's flow-control hooks go through these - see [[ConsumerPauseArbiter]]. */ + var pauseArbiters: Map[String, ConsumerPauseArbiter] = Map.empty + + // Keyed by (topic FQN, message id), NEVER by the id alone: this listener serves every topic + // consumer of its target, and a bare id is not an identity - measured against a real broker, + // EVERY non-persistent message arrives as ledger 0, entry 0, so an id-keyed marker from one + // failed ack would swallow the next unrelated message of that topic as "paperwork". + private val awaitingAckRetry = ConcurrentHashMap.newKeySet[(String, PulsarMessageId)]() + + /** Which per-message timestamp the delivery order compares - rewired at session build from the + * configured [[consumer.session_config.DeliveryOrderKey]]. The default is publish time, + * which every message carries. */ + var orderingTimeOf: org.apache.pulsar.client.api.Message[Array[Byte]] => Long = _.getPublishTime + + /** How many messages used publish time because the selected timestamp was absent on + * the message (no broker stamp, no event time). Feeds the client's remediation notice. */ + val orderKeyFallbacks = new java.util.concurrent.atomic.AtomicLong(0) + + /** Deliveries this listener SAW fail downstream - the explicit lifecycle fact that licenses + * the merge to treat a watermarked broker copy as the retry (and nothing else does; see the + * merge's duplicate arm). Entries leave when the retry is claimed. Bounded by the number of + * failed deliveries between redeliveries - rare, and discarded with the listener. */ + private val failedDeliveries = ConcurrentHashMap.newKeySet[(String, PulsarMessageId)]() + + /** Batch members this session already DECIDED (delivered or dropped), by their entry. + * + * A batched producer packs many messages into one broker ENTRY, and without broker-side + * batch-index acknowledgment (a broker setting Dekaf must not require) the per-member acks + * are client-local bits: the broker only retires the entry once EVERY member is acknowledged + * on one consumer incarnation of it. So when the pause gate slices an entry - first half + * delivered, second half refused and handed back - the broker redelivers the WHOLE entry, + * already-shown members included, wearing a fresh ack set. Those copies used to be re-decided + * from scratch: re-shown (the loaded count inflated by half a batch per hot pause), or in the + * counted modes re-dropped against budget already spent on them. + * + * The bits are set inside [[acknowledge]] - the one point every decision funnels through, for + * delivered and dropped alike - and consulted at the top of [[received]], in the same + * "the decision stands, only the paperwork remains" position as [[awaitingAckRetry]]: + * acknowledge the copy (that is what completes the redelivered entry's ack set and retires + * the entry) and show it to nobody. + * + * Non-persistent topics are excluded for the usual identity reason (every id is ledger 0, + * entry 0 - and nothing is ever redelivered). + * + * RETENTION IS A LIFECYCLE, NOT AN AGE, and that is the whole design of the eviction below. + * These records used to be evicted by insertion order at 512 entries while ONE target may hold + * a thousand physical streams: a hot pause slices one outstanding entry PER STREAM, so the + * earliest ~488 of them were discarded before their redelivery ever arrived, and the + * whole-entry redelivery then re-showed members this session had already delivered - or, in + * the counted modes, spent a second unit of the user's budget on them. A record is now exempt + * from the cap for exactly as long as the broker still owes this session the redelivery it + * describes ([[BatchEntryState.isSettled]]), so the structure covers every admitted stream by + * construction and not by a bigger magic number. It still cannot grow without bound: an entry + * leaves the owed set the moment its handed-back members are decided again, so the size is + * bounded by the settled cap plus the sliced entries the session's own streams are holding - + * one per stream, and the streams are admission-capped. + */ + private val decidedBatchMembers = java.util.LinkedHashMap[(String, Long, Long), BatchEntryState]() + + private def batchMemberKey(topicFqn: String, id: PulsarMessageId): Option[((String, Long, Long), Int, Int)] = + id match + case b: org.apache.pulsar.client.impl.BatchMessageIdImpl if b.getBatchIndex >= 0 && !isNonPersistentTopic(topicFqn) => + Some(((topicFqn, b.getLedgerId, b.getEntryId), b.getBatchIndex, b.getBatchSize)) + case _ => None + + private def noteBatchMemberDecided(topicFqn: String, id: PulsarMessageId): Unit = + batchMemberKey(topicFqn, id).foreach { (key, idx, batchSize) => + decidedBatchMembers.synchronized { + val state = decidedBatchMembers.computeIfAbsent(key, _ => BatchEntryState()) + if batchSize > state.batchSize then state.batchSize = batchSize + state.decided.set(idx) + // Decided, but the broker has not answered for it yet: the acknowledgment this + // decision is about to issue is in flight, and until it lands the entry can still + // be redelivered whole. + state.unfinished.set(idx) + evictSettledBatchEntries() + } + } + + /** The broker took this member's acknowledgment. One member closer to the entry being retired, + * and the only transition that can ever make a record reclaimable. */ + private def noteBatchMemberAcknowledged(topicFqn: String, id: PulsarMessageId): Unit = + batchMemberKey(topicFqn, id).foreach { (key, idx, _) => + decidedBatchMembers.synchronized { + Option(decidedBatchMembers.get(key)).foreach(_.unfinished.clear(idx)) + } + } + + /** A member of an entry this session is tracking was handed BACK to the broker, so the whole + * entry is coming again. Recorded only for entries already being tracked: an entry with no + * decided member has nothing to protect - every member of it will simply be decided on the + * redelivery, which is correct. */ + private def noteBatchMemberHandedBack(topicFqn: String, id: PulsarMessageId): Unit = + batchMemberKey(topicFqn, id).foreach { (key, idx, _) => + decidedBatchMembers.synchronized { + Option(decidedBatchMembers.get(key)).foreach(_.unfinished.set(idx)) + } + } + + /** Drop the records the broker can no longer redeliver, oldest first, until the settled ones are + * back inside their cap. An entry still owed a redelivery is skipped WHATEVER its age - that is + * the fix. The walk is budgeted so the per-message cost stays constant even in the pathological + * case where the oldest records are all still owed; anything the budget does not reach is + * reclaimed by the next insertion. */ + private def evictSettledBatchEntries(): Unit = + if decidedBatchMembers.size > ConsumerListener.settledBatchEntryCap then + val entries = decidedBatchMembers.entrySet.iterator + var steps = 0 + while entries.hasNext + && steps < ConsumerListener.settledBatchEntryEvictionWalk + && decidedBatchMembers.size > ConsumerListener.settledBatchEntryCap + do + if entries.next().getValue.isSettled then entries.remove() + steps += 1 + + private def isDecidedBatchCopy(topicFqn: String, id: PulsarMessageId): Boolean = + batchMemberKey(topicFqn, id).exists { (key, idx, _) => + decidedBatchMembers.synchronized { + Option(decidedBatchMembers.get(key)).exists(_.decided.get(idx)) + } + } + + /** Test-only window: how many batch entries currently carry decided-member bits. */ + private[session_runner] def decidedBatchEntryCount: Int = + decidedBatchMembers.synchronized(decidedBatchMembers.size) + + /** Test-only window into the failed-delivery registry. */ + private[session_runner] def failedDeliveryCount: Int = failedDeliveries.size + + /** Test-only window: how many decided messages still owe the broker an acknowledgment. */ + private[session_runner] def awaitingAckRetryCount: Int = awaitingAckRetry.size + + /** Acknowledge a DECIDED message - dropped or delivered, merge-path or local. On failure the + * decision stands: the id is remembered, the message handed back, and the redelivery is + * acknowledged-and-ignored at the top of [[received]]. See [[awaitingAckRetry]] for why this + * replaced both the refund and the log-and-hope paths. + */ + private def acknowledge(consumer: org.apache.pulsar.client.api.Consumer[Array[Byte]], msg: org.apache.pulsar.client.api.Message[Array[Byte]]): Unit = + // The decision this acknowledgment finalizes is recorded FIRST, unconditionally: even if + // the ack itself fails, the decision stands (that is [[awaitingAckRetry]]'s doctrine), and + // a whole-entry redelivery caused by a sliced sibling must not re-decide this member. + noteBatchMemberDecided(consumer.getTopic, msg.getMessageId) + def retryOnRedelivery(reason: String): Unit = + // A NON-PERSISTENT message's failed ack is FINAL, and inconsequential: the broker + // stores nothing, so nothing will ever be redelivered to finalize - and every + // non-persistent id looks identical (ledger 0, entry 0), so a marker here would + // swallow the next unrelated message of the topic as paperwork. + if isNonPersistentTopic(consumer.getTopic) then + logger.warn( + s"A non-persistent message's acknowledgment failed; nothing is stored, so there is nothing to retry. Consumer: ${consumer.getConsumerName}. $reason" + ) + else + logger.warn( + s"A decided message's acknowledgment failed; its decision stands and the redelivery will only be acknowledged. Consumer: ${consumer.getConsumerName}. $reason" + ) + awaitingAckRetry.add((consumer.getTopic, msg.getMessageId)) + noteBatchMemberHandedBack(consumer.getTopic, msg.getMessageId) + Try(consumer.negativeAcknowledge(msg)) + () + + if !consumer.isConnected then retryOnRedelivery(s"Consumer ${consumer.getConsumerName} is not connected.") + else + Try(consumer.acknowledgeAsync(msg)) match + case Success(acknowledged) => + acknowledged.whenComplete((_, err) => + if err != null then retryOnRedelivery(err.getMessage) + // THE BROKER HAS THE ANSWER. Only now is this member's part of its entry + // final; the entry is retired - and its record reclaimable - once every + // member has got this far. + else noteBatchMemberAcknowledged(consumer.getTopic, msg.getMessageId) + ) + () + case Failure(err) => retryOnRedelivery(err.getMessage) + + /** Acknowledge a message the start-from swallowed. The budget unit STAYS SPENT whatever the + * acknowledgment does: it was claimed for exactly this message, and [[awaitingAckRetry]] + * guarantees the message cannot come back as anything but paperwork. Refunding here used to + * let the redelivered original be shown while the refunded unit was spent on a different + * message - an exact COUNT of the wrong SET. + */ + private[session_runner] def acknowledgeDrop( + consumer: org.apache.pulsar.client.api.Consumer[Array[Byte]], + msg: org.apache.pulsar.client.api.Message[Array[Byte]] + ): Unit = acknowledge(consumer, msg) override def received(consumer: org.apache.pulsar.client.api.Consumer[Array[Byte]], msg: org.apache.pulsar.client.api.Message[Array[Byte]]): Unit = - if !isAcceptingNewMessages then - consumer.negativeAcknowledge(msg) - return; + // A redelivery of a message this session already DECIDED - its ack failed, nothing else. + // Finalize the paperwork and show it to nobody: not the gate (a pause cannot un-decide + // it), not the discard (its unit was spent on this very message), not the merge (it was + // already cut). This check is what keeps skip-n and latest-n exact across ack failures. + if awaitingAckRetry.remove((consumer.getTopic, msg.getMessageId)) then + acknowledge(consumer, msg) + // A COPY OF A DECIDED BATCH MEMBER: a sibling's refusal (the pause gate slicing an entry) + // made the broker redeliver the whole entry, this member included. Its decision stands - + // it was shown or dropped already - so acknowledge the copy (completing the redelivered + // entry's fresh ack set is what finally retires the entry) and show it to nobody: not the + // gate, not the discard, not the merge. Without this, every hot pause re-showed the + // already-shown half of the entry it sliced. + else if isDecidedBatchCopy(consumer.getTopic, msg.getMessageId) then + acknowledge(consumer, msg) + else decide(consumer.getTopic, canAcknowledge = consumer.isConnected) match + case ConsumerListener.Action.Reject => + // THE PAUSE SLICE. This member is going back, so the broker will redeliver its + // whole entry - already-decided siblings included - and their records have to + // outlive the cap until it does. See [[decidedBatchMembers]]. + noteBatchMemberHandedBack(consumer.getTopic, msg.getMessageId) + consumer.negativeAcknowledge(msg) + + case ConsumerListener.Action.Drop => + logger.debug(s"Listener discarded a message for the start-from position. Consumer: ${consumer.getConsumerName}") + acknowledgeDrop(consumer, msg) + // The read-position contract counts DROPS as read - the merge path records them, + // and this local path used to skip it, so a single-stream skip-n that consumed a + // whole backlog showed Topic Positions with no cursor at all. + recordCursor(consumer.getTopic, msg.getMessageId, msg.getPublishTime) + + case ConsumerListener.Action.Deliver => + // The offer AND everything it resolved run under the session's ordering lock. The + // lock used to be released with `offer`, and what `offer` answers with is a BATCH + // that still has to be handled - so another listener thread could resolve a later + // batch and process it first, and the session's stateful filters, coloring rules + // and value projections then saw the messages in a different order than the merge + // had just decided on. A pass-through session takes no lock at all. + startFromOrdering.inOrder { + // How the merge's flow control reaches THIS stream's consumer: through its + // pause ARBITER, holding and releasing only the Merge reason - so releasing a + // hot stream at the cut can never wake a consumer the user or the delivery + // pacer still holds. Registered on first delivery; a no-op for pass-through. + val streamId = startFromStreamId(consumer.getConsumerName, consumer.getTopic) + pauseArbiters.get(streamId).foreach { arbiter => + startFromOrdering.registerStreamPauseHooks( + streamId, + // The arbiter answers whether the client call took; the reconcile + // records only successful transitions and retries the rest. + pause = () => arbiter.hold(PauseReason.Merge), + resume = () => arbiter.release(PauseReason.Merge) + ) + } + // What comes back is what the OFFER resolved, which is often not the message + // just offered and may belong to another topic - and therefore to another + // target's listener, which is why each held message carries its own. + val resolved = startFromOrdering.offer( + consumerName = consumer.getConsumerName, + topicFqn = consumer.getTopic, + orderTime = orderingTimeOf(msg), + messageId = msg.getMessageId, + payload = HeldMessage(consumer, msg, this), + // Claimed exactly once: this arrival IS the retry of a delivery this + // listener saw fail, and the merge may deliver it past its watermark. + knownFailedRetry = failedDeliveries.remove((consumer.getTopic, msg.getMessageId)) + ) + handleResolved(resolved) + // The batch that spends the budget's last unit is handled just above, still + // under the lock; only AFTER it may later messages take the lock-free path. + startFromOrdering.settleIfDone() + startFromOrdering.reconcileFlowControl() + // Guaranteed ordering delivers through its barrier, not through the offer's + // resolutions - this offer may have completed the head set it was waiting on. + pumpGuaranteedDelivery() + } + + /** The stall watchdog's entry: give up on a stream silent past the window even when NO further + * offer will ever arrive to trigger the offer-driven check - the last held backlog message has + * nobody behind it to speak. Same lock, same handling as an offer's resolutions. + */ + def sweepStartFromStall(): Unit = + startFromOrdering.inOrder { + val abandonedBefore = startFromOrdering.abandonedStreams.size + val warningsBefore = startFromOrdering.stallWarningCount + val resolved = startFromOrdering.sweepStalled() + if resolved.nonEmpty then handleResolved(resolved) + // A sweep's give-up can spend the budget's last unit too (drain on abandonment), and + // its drains change what flow control wants paused. + startFromOrdering.settleIfDone() + startFromOrdering.reconcileFlowControl() + // A give-up changed the ANSWER (degraded, abandoned stream names) even when it + // resolved no messages - and it may never be followed by another drop. A stall WARNING + // changed what the user needs to see too: without a frame now, a stalled positioning + // sits behind "awaiting new messages" until the give-up half a minute later. Push the + // disclosure itself; the runner's gate lets the first degraded frame and each stall + // warning through whatever the reporting interval says. + if startFromOrdering.abandonedStreams.size > abandonedBefore + || startFromOrdering.stallWarningCount > warningsBefore + then reportDiscardProgress() + // The tick is also the guaranteed barrier's RETRY timer: a failed send left its head + // in place, and nothing else may ever come along to re-attempt it. + pumpGuaranteedDelivery() + } + + /** The streams the start-from resolution gave up waiting for - the session's degradation + * record, surfaced through the progress API. */ + def startFromAbandonedStreams: Vector[String] = startFromOrdering.abandonedStreams + + /** How often the resolution has newly stalled - see [[StartFromMerge.stallWarningCount]]. The + * runner's report gate reads this to let the disclosing frame through its interval. */ + def startFromStallWarnings: Long = startFromOrdering.stallWarningCount + + /** GUARANTEED-MODE delivery pump: deliver the single safe head, commit only when the send + * SUCCEEDED, retry the same head in place otherwise. This is the no-compromise barrier: a + * message never leaves the merge until a client actually received it, so a failed or + * superseded send costs latency, never order and never the message. Paced by the delivery + * limiter's token bucket (its queue would break the barrier); stopped by a paused drain + * (user pause, spent budget). Runs under the ordering lock like every delivery decision. */ + def pumpGuaranteedDelivery(): Unit = + if startFromOrdering.isGuaranteedOrdering then + startFromOrdering.inOrder { + var going = true + while going do + // Find a deliverable head BEFORE spending a rate token. A hot topic can call + // this pump repeatedly while another topic is still headless; charging those + // waits depleted the bucket without delivering anything, so the head that + // finally became safe was needlessly delayed. + startFromOrdering.peekGuaranteed() match + case None => going = false + case Some(held) => + val paced = deliveryRateLimiter.forall(l => !l.isDrainingPausedNow && l.tryAcquireDeliveryToken()) + if !paced then + startFromOrdering.abortGuaranteed() + going = false + else + // The seam flag is decided at the peek - between it and the + // commit nothing else can emit, so the row the client sees and + // the counter the commit advances cannot disagree. + if deliverStrict(held, startFromOrdering.peekIsSeamViolation) then + startFromOrdering.commitGuaranteed() + startFromOrdering.reconcileFlowControl() + else + // The send never took: the head stays in the merge AND the + // token goes back - the retry must not pay a second token + // for the same delivery (mirrors requeueFront's refund on + // the queued path). + deliveryRateLimiter.foreach(_.returnDeliveryToken()) + startFromOrdering.abortGuaranteed() + going = false + // THE REPLAY BOUNDARY CHECK, on every pump's way out: the chunk can complete on + // the commit above, on a past-boundary arrival finishing the last waited stream, + // or - for an empty replay - on the resume's very first pump with no message + // ever delivered. The hook auto-pauses and signals; its generation gate and + // once-flag make a repeat call here free. + if startFromOrdering.isReplayCaughtUp then onReplayCaughtUp() + } + + /** Deliver for the guaranteed barrier: true only when the send took. NO negative-ack and NO + * failed-delivery record on failure - the original never left the merge, the broker owes + * nothing, and the pump retries it in place on the next tick. `seamViolation` rides to the + * target pipeline through the listener's flag (set around the send, serialized by the + * ordering lock) and onto the outgoing row; a retry re-sends the memoized response, flag + * and all, without re-reading it. */ + private def deliverStrict(held: HeldMessage, seamViolation: Boolean): Boolean = + try + held.listener.outOfOrderFlagPending.set(seamViolation) + try held.listener.targetMessageHandler.onNext(held.message) + finally held.listener.outOfOrderFlagPending.set(false) + held.listener.finalizeDelivered(held) + true + catch + case err: Throwable => + logger.info(s"Guaranteed delivery attempt failed; the head stays in place and will be retried. ${err.getMessage}") + false + + /** The paperwork of a SUCCESSFUL delivery - acknowledgment and the debug cursor - on the + * listener the message belongs to. */ + private[session_runner] def finalizeDelivered(held: HeldMessage): Unit = + acknowledge(held.consumer, held.message) + recordCursor(held.consumer.getTopic, held.message.getMessageId, held.message.getPublishTime) + + /** Called by the runner on RESUME, before it re-arms the sweep: paused time must not count + * against a silent stream. (The flow-control bookkeeping needs no reset any more - a user + * resume releases only the User reason on each consumer's arbiter, so merge holds survive + * it untouched.) */ + def resetStartFromStallClock(): Unit = startFromOrdering.inOrder { + startFromOrdering.resetStallClock() + } + + /** THE LIVE DELIVERY-ORDER SWITCH: relax this session's guaranteed ordering to best effort and + * hand out whatever that releases, in the new order. + * + * Under [[StartFromOrdering.inOrder]] exactly like an offer and a sweep, which is what makes + * it safe against concurrent delivery: the relaxation, the messages it releases and their + * handling are one serialized step, so no listener thread can interleave a later batch in + * front of it and no guaranteed peek can be outstanding across it (a peek and its + * commit/abort live inside one `inOrder` block in [[pumpGuaranteedDelivery]]). + * + * NOT a re-initialization: the same consumers, the same subscription, the same acknowledgment + * identity and the same start-from budget carry on. The held messages are RELEASED, never + * re-read - they were received and not acknowledged, and a recreate would have lost them. + */ + def relaxDeliveryOrderToBestEffort(): Unit = startFromOrdering.inOrder { + // The replay's per-delivery flag dies with the barrier on THIS thread; best-effort + // deliveries stamp their own (the flag is thread-local precisely so they cannot cross). + outOfOrderFlagPending.set(false) + val released = startFromOrdering.relaxGuaranteedToBestEffort() + if released.nonEmpty then handleResolved(released) + // Same tail as every other decision point: a released batch can spend a counted cut's last + // unit, and what it drained changes what flow control wants held still. There is + // deliberately no `pumpGuaranteedDelivery` here - the barrier is exactly what was given up. + startFromOrdering.settleIfDone() + startFromOrdering.reconcileFlowControl() + } - logger.debug(s"Listener received a message. Consumer: ${consumer.getConsumerName}") - targetMessageHandler.onNext(msg) + /** GUARANTEED replay, on RESUME: extend the boundary to the freshly captured ends, under the + * ordering lock like every boundary decision. The runner calls this AFTER bumping the play + * generation and BEFORE releasing any consumer - the serialization against the offer path + * is what makes a past-end nack racing the re-capture converge (see + * [[StartFromOrdering.extendReplayBoundary]]). */ + def extendReplayBoundary(streams: Vector[StartFromStream]): Unit = startFromOrdering.inOrder { + startFromOrdering.extendReplayBoundary(streams) + } - if consumer.isConnected then consumer.acknowledgeAsync(msg) + /** Act on what the ordering layer resolved - deliver, drop, or hand back each message. Shared + * by the offer path and the stall watchdog, and containment is per message: see the comment + * inside. Callers hold the ordering lock. + */ + private[session_runner] def handleResolved(resolved: Vector[(HeldMessage, StartFromOutcome)]): Unit = + val ordersProgress = startFromOrdering.progressDiscard.isDefined + resolved.foreach { (held, outcome) => + // Each pair was already DEQUEUED from the merge, so it exists nowhere else. A + // throw here - a cancelled client makes the delivery push throw - used to + // escape the whole loop, abandoning every pair after it: neither delivered, + // acknowledged, nor handed back, and with no ackTimeout on a NonDurable + // subscription the broker never redelivered them while the runner lived. + // Contain per message and hand a failed one back so the broker redelivers it; + // the progress push below is separately shielded (see reportDiscardProgress). + try + outcome match + case StartFromOutcome.Drop => + // Acknowledge FIRST: a progress push is best-effort, and a + // message must never be counted as dropped without being + // acknowledged. Through the ORIGIN listener, like every other + // per-message action here: a failed ack is remembered in + // awaitingAckRetry, and the broker redelivers to the listener + // the consumer belongs to - remembering it HERE (the listener + // that happened to process the batch) let the redelivery + // arrive at a listener that had never heard of it, which + // re-decided it and, after the cut, DELIVERED the very message + // the user asked to skip. + held.listener.acknowledgeDrop(held.consumer, held.message) + // A dropped message was still READ - the start-from discard + // consumed it - so the debug view's read position must count it. + // Omitting it would park the cursor at the start of the topic for + // the whole of a skip-n, which is the one time it is interesting. + held.listener.recordCursor( + held.consumer.getTopic, + held.message.getMessageId, + held.message.getPublishTime + ) + // Only when the ordering layer is doing the counting. A latest-n + // retain drops most of what it sees and counts none of it, and + // reporting those would put a frame on the wire per dropped message. + if ordersProgress then held.listener.reportDiscardProgress() + case StartFromOutcome.Requeue => + // A broker copy whose original's fate is still open: decide + // NOTHING - no acknowledgment, no budget, no cursor - and + // hand it back for the broker's backoff to retry. + held.listener.noteBatchMemberHandedBack(held.consumer.getTopic, held.message.getMessageId) + Try(held.consumer.negativeAcknowledge(held.message)) + () + case StartFromOutcome.NextChunk => + // GUARANTEED replay: recorded past the boundary. Hand it back + // and PAUSE its consumer - the arbiter's Boundary reason, so a + // user pause and the merge's flow control are untouched - and + // decide nothing at all: no acknowledgment, no budget, no + // cursor, no memo entry, no batch-member record (its whole + // entry is past the boundary - a batch cannot straddle it, so + // no sibling is tracked). The Resume that extends the boundary + // releases the hold and the redelivery arrives in-chunk. + held.listener.pauseArbiters + .get(startFromStreamId(held.consumer.getConsumerName, held.consumer.getTopic)) + .foreach(_.hold(PauseReason.Boundary)) + Try(held.consumer.negativeAcknowledge(held.message)) + () + case StartFromOutcome.Deliver => + // Through the rate limiter when one is installed. Under a + // live merge the offer is FORCED to be an instant enqueue + // (see [[deliver]]), so holding the ordering lock across it + // costs nanoseconds - the write itself runs on the limiter's + // drainer, outside this lock - and the DROPS around it stay + // unlimited, which is what keeps a counted skip positioning + // at full speed under any limit. + held.listener.deliver(held) + case StartFromOutcome.DeliverOutOfOrder => + // Deliver, with the merge's own verdict stamped where the + // generic merge could not put it: the row will carry the + // out-of-order marker. + held.listener.deliver(held.copy(deliveredOutOfOrder = true)) + catch + case err: Throwable => + logger.warn( + s"Handing a start-from-ordered message back for redelivery: delivering it failed. ${err.getMessage}" + ) + held.listener.noteBatchMemberHandedBack(held.consumer.getTopic, held.message.getMessageId) + Try(held.consumer.negativeAcknowledge(held.message)) + () + } } diff --git a/server/src/main/scala/consumer/session_runner/ConsumerSessionContext.scala b/server/src/main/scala/consumer/session_runner/ConsumerSessionContext.scala index 87feb1764..73c5202a3 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerSessionContext.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerSessionContext.scala @@ -28,6 +28,33 @@ case class ConsumerSessionContextConfig( ) class ConsumerSessionContext(config: ConsumerSessionContextConfig): + /** Nobody enters this context without holding this. + * + * A GraalVM context may MIGRATE between threads but may NOT be entered by two at once - the + * loser gets "Multi threaded access requested by thread ... but is not allowed for + * language(s) js". A consumer session hands this ONE context to every partition of every + * target, and Pulsar delivers each partition on its own listener thread, so they collide. + * + * It guards more than that exception, and the more that it guards is the harder half: + * `setCurrentMessage` writes the message under test into a GLOBAL JS variable, and the filter + * chain, the coloring rules and the value projections all read it back out afterwards - each + * one a SEPARATE entry into the context, with a gap in between. `getStdout` likewise drains + * and RESETS a buffer shared by all of them. Held for the whole of one message those hand-offs + * cannot cross; held per JS call they still could, and that outcome throws nothing at all - it + * just judges one message by another message's contents. + * + * Reentrant, so a lease may nest inside a lease without deadlocking. + */ + private val lock = new java.util.concurrent.locks.ReentrantLock() + + /** Run `use` with this context entered by nobody else. Prefer leasing through + * `ConsumerSessionContextPool.withNextContext`, which keeps the choice of context and the + * exclusion over it together. */ + def exclusively[A](use: => A): A = + lock.lock() + try use + finally lock.unlock() + val context: Context = Context .newBuilder("js") .engine(config.engine) @@ -60,6 +87,32 @@ class ConsumerSessionContext(config: ConsumerSessionContextConfig): """.stripMargin ) + // The expression inspector documents `lastMessage` as the latest message seen by the + // session. Keep the implementation's private current-message slot as the source of truth for + // filters and projections, and expose a read-only live view so the two names cannot drift. + // Before the first message the getter deliberately yields `undefined`. + context.eval( + "js", + s""" + |Object.defineProperty(globalThis, 'lastMessage', { + | configurable: false, + | enumerable: true, + | get: () => $CurrentMessageVarName + |}); + """.stripMargin + ) + + /** Release the JS context this session held. + * + * `close(true)` rather than `close()`: the plain form REFUSES while any thread is inside the + * context, and a session is stopped from a gRPC thread while its listener threads may still be + * mid-message. The cancelling form is the only one that can be relied on to actually free it. + * + * Nothing closed this at all, so every consumer session ever created leaked a Graal context + * (and, through the pool, an engine) for the life of the process. + */ + def close(): Unit = context.close(true) + def getStdout: String = val logs = config.stdout.toString config.stdout match @@ -73,7 +126,6 @@ class ConsumerSessionContext(config: ConsumerSessionContextConfig): | const message = JSON.parse(messageAsJsonOmittingValue); | message.value = JSON.parse(messageValueAsJson); | message.state = $JsonStateVarName; - | console.log(JSON.stringify(message, null, 4)); | $CurrentMessageVarName = message; |}) |""".stripMargin diff --git a/server/src/main/scala/consumer/session_runner/ConsumerSessionContextPool.scala b/server/src/main/scala/consumer/session_runner/ConsumerSessionContextPool.scala index 6145d5103..0672a4166 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerSessionContextPool.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerSessionContextPool.scala @@ -31,3 +31,37 @@ case class ConsumerSessionContextPool(isDebug: Boolean = false): contextPool(key) def getContext(key: Int): ConsumerSessionContext = contextPool(key) + + /** Lease the next context for the WHOLE of `use`, with no other thread inside it meanwhile. + * + * Every caller that runs off a delivery thread - the per-message handler, the browser console - + * must come through here rather than through `getNextContext`/`getContext`: the pool is one + * context shared by every partition listener of the session, and a raw handle carries no + * exclusion. See `ConsumerSessionContext.exclusively` for what that exclusion is protecting. + */ + def withNextContext[A](use: ConsumerSessionContext => A): A = + val sessionContext = getNextContext + sessionContext.exclusively(use(sessionContext)) + + def withContext[A](key: Int)(use: ConsumerSessionContext => A): A = + val sessionContext = getContext(key) + sessionContext.exclusively(use(sessionContext)) + + /** Release every context and the engine behind them, when the session that owns this pool stops, + * and ANSWER WITH WHAT COULD NOT BE RELEASED. + * + * Best-effort per item, so one context that will not close cannot strand the rest or the + * engine. Idempotent: closing an already-closed Graal context or engine is a no-op, and + * stopping a session twice is an ordinary thing for a client to do. + * + * It used to swallow every failure and answer `Unit`, and the caller discarded that too - so a + * context still executing on some thread stayed open, holding its heap, while `deleteConsumer` + * told the client the session had been released. The caller aggregates these into the same + * failure it reports for consumers. + */ + def close(): Vector[String] = + val contextFailures = contextPool.toVector.flatMap { (key, sessionContext) => + scala.util.Try(sessionContext.close()).failed.toOption.map(err => s"JS context $key: ${err.getMessage}") + } + val engineFailure = scala.util.Try(engine.close(true)).failed.toOption.map(err => s"JS engine: ${err.getMessage}") + contextFailures ++ engineFailure.toVector diff --git a/server/src/main/scala/consumer/session_runner/ConsumerSessionRunner.scala b/server/src/main/scala/consumer/session_runner/ConsumerSessionRunner.scala index 9cbae16ff..6e7f51ce8 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerSessionRunner.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerSessionRunner.scala @@ -1,10 +1,16 @@ package consumer.session_runner import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb -import consumer.session_config.ConsumerSessionConfig +import consumer.session_config.{ConsumerSessionConfig, DeliveryOrderKey, MessageDeliveryOrder} +import consumer.session_target.ConsumerSessionTarget import org.apache.pulsar.client.admin.PulsarAdmin import org.apache.pulsar.client.api.PulsarClient +import scala.util.{Failure, Success, Try} +import scala.jdk.CollectionConverters.* + +import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit} +import java.util.concurrent.atomic.AtomicLong import java.io.ByteArrayOutputStream import com.google.rpc.code.Code import com.google.rpc.status.Status @@ -15,6 +21,135 @@ import boundary.break type ConsumerSessionTargetIndex = Int +/** ONE MESSAGE'S IDENTITY for the single-attempt delivery memo: the topic it arrived on and its + * broker message id. Never the id alone - a session consumes many topics, and on a non-persistent + * topic every id is (ledger 0, entry 0), which is why such topics carry no key at all. */ +type DeliveryKey = (NonPartitionedTopicFqn, org.apache.pulsar.client.api.MessageId) + +/** EVERYTHING ONE DELIVERY ATTEMPT PREPARED for the client: the finished response, and whether + * sending it spends a unit of the play's delivery budget. Held from the moment the pipeline + * produced it until a send actually takes, so a retry is a re-SEND rather than a second pass + * through deserialization, the JS context, the user's filters and the counters. + * + * Stamped with the play generation it was prepared under: a response belongs to the play that + * built it - it carries that play's counters and was meant for that play's stream - so a retry + * arriving after a second Play must prepare a fresh one instead. + */ +final case class PreparedDelivery( + playGeneration: Long, + messages: Seq[consumerPb.Message], + errors: Vector[String], + spendsBudgetUnit: Boolean +) + +/** What the SESSION says about a delivery attempt before the target does any work for it. */ +enum DeliveryAdmission: + /** A fresh attempt under the current play: run the pipeline. */ + case Prepare + + /** The play this handler was wired under has been superseded (a second Play, a Stop, or a + * stream ended by a failed resume). Nothing may be mutated for it. */ + case Superseded + + /** A previous attempt's response was re-sent; the pipeline must NOT run again. */ + case Replayed + +/** The single shape every ResumeResponse takes, so a message response and a progress-only push + * cannot drift apart. + * + * `startFromProgress` rides along on EVERY response, which is how the client learns that a skip + * finished: the completing state arrives with the first message that actually gets delivered. + * `None` means the session's start-from needed no counting at all, and the client is told nothing + * rather than told zero. + */ +def resumeResponse( + messages: Seq[consumerPb.Message], + errors: Vector[String], + startFromProgress: Option[consumerPb.StartFromProgress], + orderingLateDeliveries: Long = 0L, + deliveryOrderActive: Boolean = false, + orderKeyFallbacks: Long = 0L, + deliveryOrderWaitingStreams: Int = 0, + replayCaughtUp: Boolean = false, + replayBoundaryAtMs: Long = 0L, + replayNewerEntriesApprox: Long = 0L, + replaySeamViolations: Long = 0L, + replayExcludedTopics: Seq[String] = Seq.empty, + replayExcludedTopicCount: Int = 0 +): consumerPb.ResumeResponse = + val status = errors.size match + case 0 => Status(code = Code.OK.value) + case _ => Status(code = Code.UNKNOWN.value, message = errors.mkString("\n\n")) + + // Stats ride along when ANY part has something to say: a skip in flight, a best-effort + // order confessing late deliveries, the guaranteed replay's caught-up state or seam ledger, + // or the fact that an ordering layer is RUNNING at all - which is what lets the client show + // its chip only for sessions that actually pay the reorder latency (a single-stream + // BEST-EFFORT session builds no layer, whatever the config asked; Guaranteed builds one at + // any width). Absent otherwise, so clients keep treating absence as "nothing to report". + val consumerStats = + if startFromProgress.isEmpty && orderingLateDeliveries == 0 && !deliveryOrderActive && orderKeyFallbacks == 0 + && deliveryOrderWaitingStreams == 0 && !replayCaughtUp && replaySeamViolations == 0 + then None + else + Some(consumerPb.ConsumerStats( + startFromProgress = startFromProgress, + orderingLateDeliveries = orderingLateDeliveries, + deliveryOrderActive = deliveryOrderActive, + orderKeyFallbacks = orderKeyFallbacks, + deliveryOrderWaitingStreams = deliveryOrderWaitingStreams, + replayCaughtUp = replayCaughtUp, + replayBoundaryAtMs = if replayCaughtUp then replayBoundaryAtMs else 0L, + replayNewerEntriesApprox = if replayCaughtUp then replayNewerEntriesApprox else 0L, + replaySeamViolations = replaySeamViolations, + replayExcludedTopics = if replayCaughtUp then replayExcludedTopics else Seq.empty, + replayExcludedTopicCount = if replayCaughtUp then replayExcludedTopicCount else 0 + )) + + consumerPb.ResumeResponse( + messages = messages, + status = Some(status), + consumerStats = consumerStats + ) + +/** One stalled stream's classification for the DEBUG-level stall diagnostic: is the stream the + * delivery order is waiting on verifiably out of messages, or does it hold retained messages the + * session has not received yet (delivery lag - the case the guarantee exists for)? + * + * `endOfTopic` is where the stream's topic ends NOW (None when the broker would not say); + * `consumedThrough` is the newest position this session has processed from it (None before the + * first message). Pure, so the classification is pinned without a broker; the caller does the + * lookups. SERVER-SIDE ONLY: the wire's ConsumerStats carries the waiting-streams count alone - + * telling the CLIENT which case each stream is in would need a per-stream proto field that does + * not exist, so the distinction lives in the debug log until one does. + */ +/** APPROXIMATELY how many entries one stream recorded past its replay boundary, from the boundary + * end and the broker's current end. + * + * Entry arithmetic is exact only within one ledger; across a rollover the older ledgers' entry + * counts are unknowable from the ids alone, so the newest ledger's entries stand in as the + * lower bound. And an ENTRY is not a message - a batching producer packs many messages into + * one - which is why everything downstream of this says "~N" and never promises a count + * (measured 2026-07-25: the broker's own backlog numbers count entries too). PURE, so the + * arithmetic is pinned without a broker. + */ +def replayNewerEntriesApproxOf(boundary: EntryPosition, current: EntryPosition): Long = + if current == EntryPosition.empty then 0L + else if boundary == EntryPosition.empty then (current.entryId + 1) max 1L + else if !isStrictlyPastBacklogEnd(current, boundary) then 0L + else if current.ledgerId == boundary.ledgerId then current.entryId - boundary.entryId + else (current.entryId + 1) max 1L + +def stalledStreamEmptinessNote(endOfTopic: Option[EntryPosition], consumedThrough: Option[EntryPosition]): String = + endOfTopic match + case None => "the topic's end could not be read, so whether it holds unread messages is unknown" + case Some(end) if end == EntryPosition.empty => "verifiably empty - no message has ever been published (or retained)" + case Some(end) => + consumedThrough match + case Some(consumed) if isPastBacklogEnd(consumed, end) => + "verifiably empty beyond the session's position - waiting for a NEW message" + case _ => "holds retained messages the session has not received yet (delivery lag)" + case class ConsumerSessionRunner( sessionName: String, sessionConfig: ConsumerSessionConfig, @@ -22,35 +157,970 @@ case class ConsumerSessionRunner( var grpcResponseObserver: Option[io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]], var schemasByTopic: SchemasByTopic, var targets: Map[ConsumerSessionTargetIndex, ConsumerSessionTargetRunner], - var numMessageProcessed: Long = 0, - var numMessageSent: Long = 0 + // For the GUARANTEED replay's excluded-topics indicator only: re-resolving the topic + // selectors at caught-up needs an admin client the runner otherwise never held. Optional + // with a None default so the offline test fixtures build exactly as before; production + // (`make`) always passes one. + adminClient: Option[PulsarAdmin] = None ) { - def incrementNumMessageProcessed(): Unit = numMessageProcessed = numMessageProcessed + 1 + /** Session-wide counters, stamped onto EVERY `pb.Message` the browser receives. + * + * Atomic, and held in the body rather than as constructor params, because + * `incrementNumMessageProcessed` is called from the target message handler BEFORE the + * per-message context lease - i.e. concurrently, one listener thread per partition. As plain + * `var Long`s these lost read-modify-write updates, so a partitioned session under-reported how + * many messages it had processed and shipped that wrong number to the UI. + * + * `numMessageSent` happens to be incremented inside the lease today and so is already + * serialized; it is atomic too so the guarantee does not silently depend on that call staying + * where it is. + */ + private val numMessageProcessedCounter = AtomicLong(0) + private val numMessageSentCounter = AtomicLong(0) + + /** The per-resume delivery budget: how many more messages may be LOADED before the drain stops. + * Long.MaxValue when no budget is armed - the decrement below runs unconditionally, and from + * MaxValue it cannot reach zero in a session's lifetime. Reset by every resume. + */ + private val remainingToDeliver = AtomicLong(Long.MaxValue) + + /** The PLAY GENERATION: bumped by every resume, by stop, and by [[failAndComplete]] - every + * event that makes the play a handler was wired under no longer the current one. + * + * Every resume's handler closure captures the generation it was wired under and refuses to + * run under any other. It is checked in the TARGET handler, before that handler's first + * mutation, and again in the session callback before the send: the observer-identity check at + * the send alone was far too late, because by the time a superseded handler reached it the + * counters had moved, the message had been deserialized, the JS context had been leased and + * the user's stateful filters had already seen the message. + */ + private val playGeneration = AtomicLong(0) + + /** THE SINGLE-ATTEMPT DELIVERY MEMO: what each in-flight delivery has already prepared for the + * client, keyed by the message it is for and kept only until its send takes. + * + * A delivery's pipeline is NOT repeatable. It advances the processed and sent counters, and it + * runs the user's filters, coloring rules and value projections - which are stateful by + * design: they accumulate into the session's JS state and its stdout. So when a send failed + * (a cancelled call, a backpressured client) and the message came back - by broker redelivery + * on the ordinary path, or by Guaranteed's in-place retry - re-entering that pipeline applied + * every one of those effects a second time. The browser received the message exactly once and + * saw counters, accumulated state and console output as though it had been processed twice. + * + * The retry therefore RE-SENDS what the first attempt built. Entries live only between a + * failed send and its retry, and are dropped by the next resume; the size bound is a last + * resort against a client that fails every write forever - evicting merely returns that + * message to the old re-run behaviour, it never loses one. + */ + private val preparedDeliveries = + new java.util.LinkedHashMap[DeliveryKey, PreparedDelivery](64, 0.75f, false): + override def removeEldestEntry(e: java.util.Map.Entry[DeliveryKey, PreparedDelivery]): Boolean = + size > 512 + + private def rememberPrepared(key: DeliveryKey, prepared: PreparedDelivery): Unit = + preparedDeliveries.synchronized { preparedDeliveries.put(key, prepared) } + () + + private def forgetPrepared(key: DeliveryKey): Unit = + preparedDeliveries.synchronized { preparedDeliveries.remove(key) } + () + + /** What a previous attempt prepared for this message UNDER THIS PLAY, if anything. */ + private def preparedFor(key: DeliveryKey, generation: Long): Option[PreparedDelivery] = + preparedDeliveries.synchronized(Option(preparedDeliveries.get(key)).filter(_.playGeneration == generation)) + + /** Test-only window: how many deliveries are prepared but not yet sent. */ + private[session_runner] def preparedDeliveryCount: Int = preparedDeliveries.synchronized(preparedDeliveries.size) + + /** Test-only window: which play generation is current. */ + private[session_runner] def currentPlayGeneration: Long = playGeneration.get + + def numMessageProcessed: Long = numMessageProcessedCounter.get + def numMessageSent: Long = numMessageSentCounter.get + + def incrementNumMessageProcessed(): Unit = numMessageProcessedCounter.incrementAndGet() + + /** Messages the start-from discard has still to drop before this session shows anything. + * + * Exposed so a test can assert "exactly n were skipped" instead of inferring it from what came + * out. `distinct` is identity-based (StartFromDiscard defines no equals), which is what a + * SharedTotal plan needs: every target holds the SAME counter and it must be counted once. + * + * Reads the EFFECTIVE counter, not the reportable one: this is the mechanism, and a latest-n + * seek correction still has messages to drop even though the client is deliberately not shown + * a progress bar for it. + */ + def remainingStartFromDiscard: Long = + targets.values.map(_.consumerListener.effectiveDiscard).toVector.distinct.map(_.remaining).sum + + /** How far the start-from skip has got, or `None` when there is no skip to do. + * + * Absent - not zero-and-complete - for every mode that seeks exactly (earliest, latest, a + * date/time, a message id, an approximate position): those reach their position with the seek + * itself, and a progress report for them would be an invention. + * + * `distinct` is identity-based, which is what a shared counter needs: every target of a + * "skip first n" session holds the SAME counter, and counting it once per target would report + * n times the number of targets. On a partitioned "skip first n" that counter belongs to the + * global merge rather than to the listener, which is what `progressDiscard` resolves - the + * merge decides the drops, so it is the only thing that knows how many are left. + */ + def startFromProgress: Option[consumerPb.StartFromProgress] = + val discards = targets.values.map(_.consumerListener.progressDiscard).toVector.distinct + val toSkip = discards.map(_.total).sum + + Option.when(toSkip > 0) { + val left = discards.map(_.remaining).sum + // The degradation record rides on every progress frame: a stream the merge abandoned + // means the position is best-effort, and the client keeps saying so for the session's + // life. One listener suffices - the ordering layer is session-wide. + val abandoned = targets.values.headOption.map(_.consumerListener.startFromAbandonedStreams).getOrElse(Vector.empty) + consumerPb.StartFromProgress( + messagesSkipped = toSkip - left, + messagesToSkip = toSkip, + complete = left <= 0, + degraded = abandoned.nonEmpty, + abandonedStreams = abandoned + ) + } + + /** The order this session is delivering in RIGHT NOW. + * + * Starts as the configured one and can be LOWERED by [[setDeliveryOrder]] while the session + * runs. The saved configuration is deliberately untouched: this is a live-session control, and + * a client that wants the next Play to use the new order writes it into the configuration + * itself. + * + * Volatile: written on the gRPC thread that switches, read by whoever asks afterwards. + */ + @volatile private var liveDeliveryOrder: MessageDeliveryOrder = sessionConfig.messageDeliveryOrder + + /** What [[setDeliveryOrder]] would compare a request against - the live order, not the saved + * one. */ + def deliveryOrder: MessageDeliveryOrder = liveDeliveryOrder + + /** CHANGE THE DELIVERY ORDER OF THIS RUNNING SESSION, without recreating it. + * + * Only Guaranteed -> Best effort is honoured; [[deliveryOrderSwitchFor]] carries the whole + * matrix and the reasons the other directions are refused rather than faked. A refusal is + * thrown with the client-facing reason, exactly as `pause` reports its failures. + * + * NOTHING ELSE ABOUT THE SESSION MOVES. The consumers, their subscriptions, the + * acknowledgment identity, the counted start-from budget and the flow-control state all carry + * on; the messages the guaranteed barrier was holding are RELEASED under the new rules rather + * than dropped or re-read. That is the entire reason this is not implemented as "recreate the + * session with a different configuration": the held set is received-but-unacknowledged and + * lives nowhere else, re-resolving the start-from against a log that has moved would select a + * different set of messages for Latest-n and both approximate modes, and everything already on + * screen would arrive again as duplicates. + * + * THE LAYER IS THE SOURCE OF TRUTH. It is relaxed first and the live order is recorded only + * once that returned, so a session can never advertise an order its ordering layer is not + * actually applying. + */ + def setDeliveryOrder(requested: MessageDeliveryOrder): Unit = + deliveryOrderSwitchFor(liveDeliveryOrder, requested) match + case DeliveryOrderSwitch.AlreadyThere => () + case DeliveryOrderSwitch.Refused(reason) => throw new IllegalArgumentException(reason) + case DeliveryOrderSwitch.RelaxToBestEffort => + // ONE listener does it for the whole session: the ordering layer is session-wide - + // the same instance on every target - because the counted modes are defined over + // the merged stream. Relaxing it once relaxes it for every target. + targets.values.headOption.foreach(_.consumerListener.relaxDeliveryOrderToBestEffort()) + // THE REPLAY STATE DIES WITH THE BARRIER. This is the designed "continue live + // with Best effort" transition (owner decision 2026-08-09): the caught-up wire + // state clears, every Boundary hold lifts - the auto-pause's and any past-end + // racer's - and the intake re-opens, so live delivery resumes under Best effort. + // UNLESS the user's own pause is also standing (PauseReason.User held): their + // pause outranks the transition, the gate stays shut, and the next Resume goes + // live - relaxing a session must never un-pause one the user explicitly stopped. + replayCaughtUpNow = false + replayNewerEntriesApprox = 0L + replayExcludedTopics = Vector.empty + replayExcludedTopicCount = 0 + val userPaused = targets.values.exists(_.pauseArbiters.values.exists(_.heldReasons.contains(PauseReason.User))) + targets.values.foreach { target => + if !userPaused then target.consumerListener.startAcceptingNewMessages() + target.pauseArbiters.values.foreach(_.release(PauseReason.Boundary)) + } + liveDeliveryOrder = requested + // RE-ARM AT THE NEW CADENCE. Best effort's sweep is the timer half of its residence + // bound, so a session relaxed out of Guaranteed must not keep Guaranteed's slower + // diagnostics cadence - an all-idle tail would then sit up to a second past its + // grace instead of a quarter of one. + cancelStallSweep() + armStallSweep() + permitLogger.info( + s"Consumer session $sessionName switched from Guaranteed to Best effort delivery order while running; " + + "the messages it was holding were released in the new order." + ) + + /** Whether the client that resumed this session asked for consumer stats + * (`ResumeRequest.include_consumer_stats`). + * + * It used to be read off the request and then dropped, so every client received the stats - + * including the MESSAGE-LESS progress frames a skip in flight pushes, which a client that + * asked for no stats has no reason to expect. Set on every resume, like the debug flag. + * + * Volatile: written on the gRPC thread that resumes, read by listener threads composing + * responses - without it a listener could briefly keep using the previous play's flag. + */ + @volatile var includeConsumerStats: Boolean = true + + /** What may actually go on the wire: nothing at all unless the client asked for it. */ + private def reportableStartFromProgress: Option[consumerPb.StartFromProgress] = + if includeConsumerStats then startFromProgress else None + + /** Deliveries resolved out of selected-timestamp order, for the same wire gate: zero unless + * the client asked for stats and the session actually merges. The ordering layer is + * session-wide, so one listener's count is the session's. */ + private def reportableOrderingLateDeliveries: Long = + if includeConsumerStats && continuousOrderingArmed then + targets.values.headOption.map(_.consumerListener.startFromOrdering.lateDeliveryCount).getOrElse(0L) + else 0L + + /** Messages that used publish time because the selected timestamp was absent - summed across + * listeners (each message counts exactly once, at the listener that received it). */ + private def reportableOrderKeyFallbacks: Long = + if includeConsumerStats && continuousOrderingArmed then + targets.values.map(_.consumerListener.orderKeyFallbacks.get).sum + else 0L + + /** Topic/partition streams that have kept an active delivery-order layer waiting past the + * warning interval. Unlike the warning counter, this clears when progress resumes. */ + private def reportableDeliveryOrderWaitingStreams: Int = + if includeConsumerStats && continuousOrderingArmed then + targets.values.headOption.map(_.consumerListener.startFromOrdering.stalledStreamCount).getOrElse(0) + else 0 + + /** THE GUARANTEED REPLAY'S WIRE STATE, stamped onto every response's ConsumerStats while it + * stands. `replayCaughtUpNow` turns true when the chunk completes (the auto-pause moment) + * and false the moment a resume extends the boundary or a relax dissolves the barrier; the + * boundary instant is re-captured by every resume; the newer-entries indicator starts from + * what the session itself saw handed back past the boundary and may be raised by the + * off-thread broker refinement, together with the excluded-topics (late-joiner) list. + * All volatile: written on the pump's thread and the maintenance thread, read by whichever + * listener thread is composing a response. + */ + @volatile private var replayCaughtUpNow: Boolean = false + @volatile private var replayBoundaryAtMs: Long = 0L + @volatile private var replayNewerEntriesApprox: Long = 0L + @volatile private var replayExcludedTopics: Vector[String] = Vector.empty + @volatile private var replayExcludedTopicCount: Int = 0 + + /** When the CREATE-TIME replay boundary was captured: session build seeks the consumers and + * decides every stream's recorded end moments before this runner exists, so construction + * time is that capture's instant - what the first chunk's caught-up banner reports. */ + private val replayBoundaryCapturedAtMs: Long = System.currentTimeMillis() + + /** Whether any Play has CONSUMED the create-time replay boundary yet. The first Play must use + * the boundaries decided at session build instead of re-reading the broker (see resume); + * every later one re-captures and extends. Volatile: resumes are serialized by the service's + * per-name lock, but successive plays run on different gRPC threads. */ + @volatile private var replayCreateBoundaryConsumed: Boolean = false + + /** Test-only window onto the caught-up state. */ + private[session_runner] def isReplayCaughtUpNow: Boolean = replayCaughtUpNow + + /** The seam-violation counter, from the session-wide ordering layer - one listener's answer + * is the session's. Reported whenever guaranteed ordering is armed, caught up or not: a + * seam violation happens MID-replay, right after a resume. */ + private def reportableReplaySeamViolations: Long = + if includeConsumerStats && guaranteedOrderingArmed then + targets.values.headOption.map(_.consumerListener.startFromOrdering.replaySeamViolationCount).getOrElse(0L) + else 0L + + /** Whether a counted start-from is still resolving: the discard has messages left to drop, or + * the global merge is holding messages while it decides. The rate limiter's permit hold is + * refused during this window - a consumer whose permits a THROTTLE paused looks exactly like + * the silent stream the merge gives up on after 30 seconds, and that give-up silently changes + * a skip's result. Delivers still queue during the window (they are the few at the boundary), + * so the user-visible rate stays exact; only the broker-side backpressure waits. + */ + private def startFromResolutionActive: Boolean = + // For a continuous delivery-order layer, held messages are the steady + // state, not a resolution in flight: counting them here would refuse the rate limiter's + // permit holds for the session's whole life. Its budget half still counts - a skip's cut + // must run at full speed whatever ordering mode carries it. + remainingStartFromDiscard > 0 || targets.values.exists { target => + val ordering = target.consumerListener.startFromOrdering + !ordering.isContinuousOrdering && ordering.heldCount > 0 + } + + /** Every physical stream this session consumes, across all of its targets. */ + private def allConsumedTopics: Set[NonPartitionedTopicFqn] = targets.values.flatMap(_.consumers.keys).toSet + + /** Which of this session's streams the delivery pacer must LEAVE RUNNING right now, because a + * counted start-from is still waiting for the next message from them. + * + * PER STREAM, and that is the whole point of it. [[startFromResolutionActive]] answers for the + * SESSION, and a latest-n seek correction that never finishes on one disconnected or idle + * stream therefore refused every permit hold for the session's life: a hot peer kept feeding + * the limiter's queue while its watermarks were declined on crossing after crossing, so the + * advertised 2,000-message / 128 MiB bounds held nothing at all. Only the streams actually + * still counting are protected now; their peers get the backpressure they earned. + * + * Session-wide in the two cases where ANY stream can supply the next counted message: a + * SHARED "skip first n" budget (one counter over the merged stream - `remainingFor` answers + * for every topic by construction), and a non-continuous merge still holding messages while + * it decides the cut. A continuous delivery-order layer is excluded for the same reason + * [[startFromResolutionActive]] excludes it: held messages are its steady state, not a + * resolution in flight. + */ + private def permitHoldSuppressedTopics: Set[NonPartitionedTopicFqn] = + val mergeStillDeciding = targets.values.exists { target => + val ordering = target.consumerListener.startFromOrdering + !ordering.isContinuousOrdering && ordering.heldCount > 0 + } + if mergeStillDeciding then allConsumedTopics + else + val discards = targets.values.map(_.consumerListener.effectiveDiscard).toVector.distinct + allConsumedTopics.filter(topicFqn => discards.exists(_.remainingFor(topicFqn) > 0)) + + /** Whether any target's ordering layer merges for the session's LIFE - the sweep then stays + * armed forever and ticks at the grace cadence, because an all-idle tail emits nothing until + * a tick notices the reorder grace has passed. */ + private def continuousOrderingArmed: Boolean = + targets.values.exists(_.consumerListener.startFromOrdering.isContinuousOrdering) + + /** Whether that continuous layer is the GUARANTEED one - which decides the sweep cadence, since + * Guaranteed has no residence for a tick to expire. The ordering layer is session-wide (one + * instance across every target), so one target's answer is the session's. */ + private def guaranteedOrderingArmed: Boolean = + targets.values.headOption.exists(target => + val ordering = target.consumerListener.startFromOrdering + ordering.isContinuousOrdering && ordering.isGuaranteedOrdering + ) + + /** The sweep cadence currently armed, or None when no sweep is - a test's window onto the + * policy-specific choice, and nothing else's. */ + private[consumer] def armedSweepPeriodMs: Option[Long] = synchronized(armedSweepPeriod) + + private var armedSweepPeriod: Option[Long] = None + + /** Applies or lifts the rate limiter's permit hold on every target. Each target refuses under + * a closed gate, so a user pause always outranks the limiter's backpressure. + * + * ANSWERS whether every target applied it. `Consumer.pause()` can throw (a consumer mid-close + * during a shutdown race), and swallowing that used to let the limiter latch `permitsHeld` + * with nothing actually paused - the broker kept filling the queue and no retry ever came, + * because the flag said the work was done. Every target is still ATTEMPTED (one broken + * consumer must not shield the others), and the aggregate verdict lets the caller keep the + * flag honest. + */ + private val permitLogger = com.typesafe.scalalogging.Logger(getClass.getName) + + private def setPermitHold(hold: Boolean, appliesTo: NonPartitionedTopicFqn => Boolean = _ => true): Boolean = + targets.values.toVector + .map { target => + Try(target.setPermitHold(hold, appliesTo)) match + // The target's own verdict, not merely "the method did not throw": a target + // answers false when one of its consumers refused the client call, and + // mapping that to true let the limiter latch permitsHeld with consumers + // still running - the exact lie this aggregation exists to prevent. + case Success(applied) => applied + case Failure(err) => + permitLogger.warn( + s"Could not ${if hold then "hold" else "release"} permits on target ${target.targetIndex}: ${err.getMessage}" + ) + false + } + .forall(identity) + + /** The DEBUG-level why behind a waiting-streams disclosure: which streams the delivery order + * is waiting on, and [[stalledStreamEmptinessNote]] for each. Scheduled onto the shared + * maintenance thread, never run inline - classifying a stream reads its topic's end from the + * broker, and the disclosure that triggers this fires under the session's ordering lock. + * Throttled exactly as the disclosure is (once per NEW stall warning), and skipped outright + * unless debug logging is enabled, so no broker round trip is ever spent on a log nobody + * collects. Per-stream and per-episode, hence DEBUG - see the logging guidance. + */ + private def scheduleStalledStreamDiagnostic(): Unit = + if permitLogger.underlying.isDebugEnabled then + Try(ConsumerSessionRunner.maintenanceScheduler.execute(() => logStalledStreamEmptiness())) + () + + /** REFINE THE CAUGHT-UP INDICATORS off-thread: the newer-entries estimate from the broker's + * current ends against the boundary, and the excluded-topics (late-joiner) list from + * re-resolving the topic selectors. Scheduled by the caught-up announcement onto the shared + * maintenance thread - both halves are broker round trips, and the announcement runs under + * the session's ordering lock, exactly the reason the stalled-stream diagnostic is + * scheduled the same way. Best-effort throughout: a broker that will not answer leaves the + * locally-known lower bound standing. + */ + private def scheduleReplayIndicatorRefinement( + generation: Long, + observer: io.grpc.stub.StreamObserver[consumerPb.ResumeResponse] + ): Unit = + Try(ConsumerSessionRunner.maintenanceScheduler.execute(() => refineReplayIndicators(generation, observer))) + () + + private def refineReplayIndicators(generation: Long, observer: io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]): Unit = + Try { + if playGeneration.get == generation && replayCaughtUpNow then + val boundaryEnds = + targets.values.headOption.map(_.consumerListener.startFromOrdering.replayBoundaryEnds).getOrElse(Map.empty) + var newerEntries = 0L + targets.values.foreach { target => + target.consumers.foreach { (topicFqn, consumer) => + if !isNonPersistentTopic(topicFqn) then + val streamId = startFromStreamId(consumer.getConsumerName, topicFqn) + boundaryEnds.get(streamId).foreach { boundary => + Try(consumer.getLastMessageIds.asScala.toVector).foreach { ids => + val current = ids + .map(EntryPosition.of) + .maxOption(Ordering.by[EntryPosition, (Long, Long, Int)](p => (p.ledgerId, p.entryId, p.batchIndex))) + .getOrElse(EntryPosition.empty) + newerEntries += replayNewerEntriesApproxOf(boundary, current) + } + } + } + } + // LATE JOINERS: topics the selectors match NOW that the session is not consuming + // - typically regex matches created after Play. Excluded from the replay by + // design; surfaced so the banner can say "restart to include". + val subscribed = allConsumedTopics + val matchingNow = adminClient.toVector.flatMap(admin => + targets.values.toVector.flatMap(target => + Try(target.targetConfig.topicSelector.getNonPartitionedTopics(adminClient = admin)).getOrElse(Vector.empty) + ) + ) + val excluded = matchingNow.distinct.filterNot(subscribed.contains) + // Still the same chunk and still caught up? Then record and push one refreshed + // frame; a resume that landed meanwhile owns the state now and is left alone. + if playGeneration.get == generation && replayCaughtUpNow then + val changed = newerEntries > replayNewerEntriesApprox || excluded.size != replayExcludedTopicCount + replayNewerEntriesApprox = math.max(replayNewerEntriesApprox, newerEntries) + replayExcludedTopics = excluded.take(ConsumerSessionRunner.replayExcludedTopicNamesCap) + replayExcludedTopicCount = excluded.size + if changed && includeConsumerStats then Try(sendResponse(observer, Seq.empty, Vector.empty)) + }.failed.foreach(err => permitLogger.debug(s"The caught-up indicator refinement failed and was skipped. ${err.getMessage}")) + + private def logStalledStreamEmptiness(): Unit = + Try { + // The ordering layer is session-wide, so the first listener's view is the session's. + val stalled = targets.values.headOption.map(_.consumerListener.startFromOrdering.stalledStreamIds).getOrElse(Vector.empty) + if stalled.nonEmpty then + val consumersByStreamId = targets.values.flatMap { target => + target.consumers.map((topicFqn, consumer) => + startFromStreamId(consumer.getConsumerName, topicFqn) -> (consumer, target.consumerListener) + ) + }.toMap + stalled.foreach { streamId => + val note = consumersByStreamId.get(streamId) match + case None => "no live consumer for this stream" + case Some((consumer, listener)) => + val endOfTopic = Try(consumer.getLastMessageIds.asScala.toVector).toOption.map(ids => + ids + .map(EntryPosition.of) + .maxOption(Ordering.by[EntryPosition, (Long, Long, Int)](p => (p.ledgerId, p.entryId, p.batchIndex))) + .getOrElse(EntryPosition.empty) + ) + val consumedThrough = listener.consumedBounds + .get(consumer.getTopic) + .map(bounds => EntryPosition.of(bounds.last.messageId)) + stalledStreamEmptinessNote(endOfTopic, consumedThrough) + permitLogger.debug(s"Delivery ordering in session $sessionName is waiting on $streamId: $note") + } + }.failed.foreach(err => permitLogger.debug(s"The stalled-stream diagnostic failed and was skipped. ${err.getMessage}")) + + /** The single timer thread behind the session's delivery rate limiter AND, for a + * best-effort-ordered session, its continuous sweep ticks - created on first use, so a + * session that neither throttles nor merges never owns one. Both tenants do real per-message + * work here (JS lease, send lock), which is exactly why this thread is PER SESSION: on the + * shared maintenance thread one slow session's filter delayed every other session's timers. + */ + private var rateLimiterExecutor: Option[ScheduledExecutorService] = None + + private def timerExecutor(): ScheduledExecutorService = synchronized { + rateLimiterExecutor.getOrElse { + val created = Executors.newSingleThreadScheduledExecutor(runnable => { + val thread = Thread(runnable, s"delivery-rate-limit-$sessionName") + thread.setDaemon(true) + thread + }) + rateLimiterExecutor = Some(created) + created + } + } + + private def scheduleRateLimiterTick(delayMs: Long, task: Runnable): Unit = + // A tick scheduled while stop() is shutting the executor down is a delivery that no longer + // matters; dropping it is the correct outcome, not an error. + Try(timerExecutor().schedule(task, delayMs, TimeUnit.MILLISECONDS)) + () + + /** The merge's stall bound is TIME-driven only through this watchdog: the in-band check runs on + * offers, and after the last held backlog message no offer may ever come again - a + * retention-trimmed partition then held the merge forever with the give-up window long + * expired. Armed on resume while a counted start-from is unresolved, disarmed the moment it + * resolves (the task cancels itself), on pause, and on stop. + */ + private var stallSweepTask: Option[java.util.concurrent.ScheduledFuture[?]] = None + + private def armStallSweep(): Unit = synchronized { + val continuous = continuousOrderingArmed + if (startFromResolutionActive || continuous) && stallSweepTask.forall(task => task.isDone || task.isCancelled) then + // The grace cadence for a continuous merge (its ticks are the timer half of the + // residence bound and stay armed for the session's life); the coarse stall cadence + // for a counted skip, whose sweep self-cancels once the resolution settles. + // + // ON THE SESSION'S OWN TIMER THREAD when continuous: a tick can release held + // messages and run the whole downstream path - deserialization, user JavaScript, the + // gRPC send - and on the SHARED maintenance thread one slow session's filter would + // have delayed every other session's grace timer and the idle janitor with it. The + // per-session timer already does exactly this class of work for the rate limiter. + // The skip watchdog stays on the shared thread: its ticks are tiny checks, and its + // rare give-up drain is a bounded one-off. + // POLICY-SPECIFIC when continuous - see [[continuousSweepPeriodMs]]. Guaranteed has no + // residence for a tick to expire and never gives a stream up, so the best-effort grace + // cadence bought it nothing but work on a session that was simply waiting. + val periodMs = if continuous then continuousSweepPeriodMs(guaranteedOrderingArmed) else startFromStallSweepPeriodMs + def tick(): Unit = + if !startFromResolutionActive && !continuous then cancelStallSweep() + else + Try(targets.values.headOption.foreach(_.consumerListener.sweepStartFromStall())).failed + .foreach(err => permitLogger.warn(s"The start-from stall sweep failed; it will run again. ${err.getMessage}")) + stallSweepTask = Try( + if continuous then timerExecutor().scheduleWithFixedDelay(() => tick(), periodMs, periodMs, TimeUnit.MILLISECONDS) + else ConsumerSessionRunner.maintenanceScheduler.scheduleWithFixedDelay(() => tick(), periodMs, periodMs, TimeUnit.MILLISECONDS) + ).toOption + armedSweepPeriod = stallSweepTask.map(_ => periodMs) + } + + private def cancelStallSweep(): Unit = synchronized { + stallSweepTask.foreach(_.cancel(false)) + stallSweepTask = None + armedSweepPeriod = None + } + + /** The session's delivery rate limiter. ONE per session however many targets and partitions the + * selector matched, so the configured number means "per second, total" - the only reading a + * user can act on. Rate 0 (the default) short-circuits to the unlimited path. + */ + val deliveryRateLimiter: DeliveryRateLimiter[HeldMessage] = DeliveryRateLimiter[HeldMessage]( + core = DeliveryRateLimiterCore[HeldMessage]( + nowMs = () => System.nanoTime() / 1_000_000L, + payloadBytesOf = held => scala.util.Try(Option(held.message.getData).map(_.length.toLong).getOrElse(0L)).getOrElse(0L) + ), + schedule = scheduleRateLimiterTick, + process = held => held.listener.deliverNow(held), + holdPermits = () => { + // PER STREAM: hold everything except the streams a counted start-from is still waiting + // on. Nothing eligible at all is the old whole-session refusal, and it hands back + // `false` exactly as a target whose consumer threw does - the limiter keeps its flag + // clear and retries on the next crossing offer. + val eligible = allConsumedTopics -- permitHoldSuppressedTopics + eligible.nonEmpty && setPermitHold(true, eligible.contains) + }, + releasePermits = () => + // Truthful, like the hold: the limiter keeps its flag set and RETRIES on its own timer + // when this answers false. Clearing the flag on a failed release used to strand + // consumers paused forever - the queue was drained, so no crossing would ever come + // to notice. EVERY stream, so a partial hold cannot leave one behind. + setPermitHold(false), + forceHoldPermits = () => + // The ceiling's escalation - see [[deliveryRateLimitForceHoldPermitsAboveQueued]]. + // Every stream, suppression and all: past this point an unbounded queue is the worse + // outcome than a start-from cut the client is already told is degraded. + setPermitHold(true) + ) + + /** Hand ONE response to the client, with wherever the start-from skip has got to attached. + * + * Every response leaves through here - a delivered message, a count-only placeholder, a + * progress push - so the client cannot receive one that forgot the stats. That matters for the + * completing state in particular: the client clears its progress panel when `complete` is true + * or when the field is absent, so a delivered message that dropped the stats would leave a + * "skipping..." panel on screen for the rest of the session. + * + * SERIALIZED, AND THE RESPONSE IS BUILT INSIDE THE SERIALIZED SECTION. `io.grpc.stub + * .StreamObserver` is not thread-safe and this is entered from every Pulsar listener thread of + * the session - one per physical topic - as well as from the gRPC thread that resumed it. + * Progress pushes made that concrete: several partitions claiming a discard at once each called + * `onNext` directly, concurrently, on one observer. + * + * Taking the lock only around `onNext` was not enough. The response - INCLUDING its start-from + * counters - was built first, so two threads could snapshot the counters in one order and send + * in the other: an older, still-incomplete progress frame could be written after a newer, + * complete one. The client clears its progress panel when it sees `complete`, so the stale + * frame behind it reopened a "skipping..." panel that stayed for the rest of the session. The + * snapshot and the write now happen under one lock, which is the only way the two can agree. + * + * NOTHING IS WRITTEN AFTER THE STREAM HAS BEEN COMPLETED. See [[stop]]. + * + * AND A DATA SEND INTO AN ENDED STREAM FAILS rather than reporting success. It used to be a + * silent no-op, which is the one shape a caller cannot survive: `deliverNow` takes a normal + * return as "the client has it" and goes straight on to acknowledge, so a delivery in flight + * across [[stop]] or [[failAndComplete]] - the ordinary consequence of one target's resume + * throwing while another target is mid-message - removed a persistent message from this + * session's subscription with no browser ever having seen it. Throwing routes it down the + * caller's failure path: negative-acknowledge and broker redelivery, so the message survives + * to be shown by whatever session the client creates next. + * + * A MESSAGE-LESS progress frame is still dropped silently: nothing was delivered, so nothing + * can be lost, and its callers treat a failed push as best-effort by design. + */ + def sendResponse( + observer: io.grpc.stub.StreamObserver[consumerPb.ResumeResponse], + messages: Seq[consumerPb.Message], + errors: Vector[String] + ): Unit = + sendLock.synchronized { + if streamCompleted && messages.nonEmpty then + throw new IllegalStateException( + "This play stream has ended; the message is handed back for redelivery." + ) + else if streamCompleted then () + else if grpcResponseObserver.exists(_ eq observer) then + observer match + // grpc-java SILENTLY DISCARDS writes to a cancelled call once a cancel + // handler is installed, and the cancellation handler clears the observer + // only a moment later - a write in that window would vanish while the + // caller acknowledged the message. Cancellation is a failed delivery. + case sco: io.grpc.stub.ServerCallStreamObserver[?] if sco.isCancelled => + throw new IllegalStateException("The play stream was cancelled; the message is handed back for redelivery.") + case _ => () + observer.onNext(resumeResponse( + messages, + errors, + reportableStartFromProgress, + reportableOrderingLateDeliveries, + deliveryOrderActive = includeConsumerStats && continuousOrderingArmed, + orderKeyFallbacks = reportableOrderKeyFallbacks, + deliveryOrderWaitingStreams = reportableDeliveryOrderWaitingStreams, + replayCaughtUp = includeConsumerStats && replayCaughtUpNow, + replayBoundaryAtMs = replayBoundaryAtMs, + replayNewerEntriesApprox = replayNewerEntriesApprox, + replaySeamViolations = reportableReplaySeamViolations, + replayExcludedTopics = replayExcludedTopics, + replayExcludedTopicCount = replayExcludedTopicCount + )) + else + // The observer this send was built for is no longer the wired one: a second Play + // replaced it, or the transport cancelled it. Writing anyway could SILENTLY + // vanish (grpc-java drops writes to a cancelled call once a cancel handler is + // installed) and the caller would then acknowledge a message no client received. + // Throwing routes it down the caller's failure path - negative-acknowledge and + // broker redelivery - so it reaches the CURRENT stream instead, merely late. + throw new IllegalStateException( + "This play stream was replaced or cancelled; the message is handed back for redelivery." + ) + } + + /** The one thing every write to this session's response stream goes through - including the + * terminal `onCompleted`, which is what makes "no response after the end" enforceable. */ + private val sendLock = new Object + + /** Whether the client's stream has been ended. Read and written only under [[sendLock]], so + * every listener thread sees it as soon as the thread that ended the stream let go. + * + * STICKY, deliberately. A session is only stopped on its way out - deleted, or replaced by a + * new session under the same name - and the runner is discarded immediately afterwards. A + * resume that raced the stop would otherwise start writing into a completed gRPC stream, which + * throws; the client's own remedy is to create a session, not to revive this one. + */ + private var streamCompleted: Boolean = false + + /** When this runner stopped having a live play stream (monotonic nanos), `None` while one is + * wired. Starts ticking at construction - a session that is created and never resumed is + * exactly as abandoned as one whose tab was closed. Read and written under [[sendLock]] with + * the observer it describes. This is the idle janitor's whole input: a client that vanishes + * without a Delete RPC (tab closed, network drop, `beforeunload` that never delivered) leaves + * consumers connected and reconnect-looping forever, and the ONLY signal that remains is that + * nobody holds the play stream any more. + */ + private var idleSinceNanos: Option[Long] = if grpcResponseObserver.isEmpty then Some(System.nanoTime()) else None + + /** Whether this runner's response stream has been ENDED - by [[stop]], or by + * [[failAndComplete]]. Sticky, like the flag it reads: a terminal runner can never speak + * again, so `ConsumerServiceImpl.resume` must answer a fresh observer itself rather than wire + * it into one - every send would be swallowed by the gate and the play stream would hang + * silent. + */ + def isStreamCompleted: Boolean = sendLock.synchronized(streamCompleted) + + /** Nanos-timestamp since when this runner has had NO live play stream (or has been terminal), + * `None` while one is wired - the janitor reaps runners whose value is old enough. */ + def reapableSinceNanos: Option[Long] = sendLock.synchronized(idleSinceNanos) + + /** Forget `observer` if it is still the wired one - the transport told us its call died. + * + * Compare-and-clear, not a bare clear: the cancellation of an OLD play stream can arrive + * after a new resume has already wired its successor, and clearing (or pausing) then would + * stomp the play the user just started. Returns whether anything was cleared, so the caller + * knows if the cancellation was current news or stale. + */ + def releaseCancelledObserver(observer: io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]): Boolean = + sendLock.synchronized { + if grpcResponseObserver.exists(_ eq observer) then + grpcResponseObserver = None + idleSinceNanos = Some(System.nanoTime()) + true + else false + } + + /** End the client's stream with a non-OK `status`, through the same lock and terminal flag as + * every other write to it. + * + * This exists for the failure paths that used to write to the observer DIRECTLY: a late + * target throwing out of a resume leaves the earlier targets' listeners live and pushing into + * the same observer, so a status frame written outside [[sendLock]] could interleave with a + * push (`StreamObserver` is not thread-safe), and an `onCompleted` that set no terminal flag + * let every later push land in a completed stream. Both sends are best-effort - the client's + * call may already be dead - but the flag, which is what stops the pushing, is set regardless. + * + * THE PLAY GENERATION DIES FIRST, before the stream is ended, exactly as in [[stop]]: a + * handler still in flight under it can no longer reach any client, so it must fail at the + * generation gate rather than run the whole pipeline - the counters, the JS context, the + * user's stateful filters - for a response that will never be written. + */ + def failAndComplete(status: Status): Unit = + playGeneration.incrementAndGet() + sendLock.synchronized { + if !streamCompleted then + streamCompleted = true + grpcResponseObserver.foreach { observer => + Try(observer.onNext(consumerPb.ResumeResponse(status = Some(status)))) + Try(observer.onCompleted()) + } + grpcResponseObserver = None + idleSinceNanos = idleSinceNanos.orElse(Some(System.nanoTime())) + } def resume( grpcResponseObserver: io.grpc.stub.StreamObserver[consumerPb.ResumeResponse], - isDebug: Boolean + isDebug: Boolean, + includeConsumerStats: Boolean = true, + maxMessagesPerSecond: Long = 0, + maxMessagesToDeliver: Long = 0 ): Unit = - this.grpcResponseObserver = Some(grpcResponseObserver) + // FIRST: this play's generation. Anything still running under an older one dies at the + // generation gate - in the TARGET handler, before it touches state, budget or the JS + // context, and again here before the send. + val myGeneration = playGeneration.incrementAndGet() + // A new play prepares its own responses: anything the previous one left half-sent carries + // that play's counters and was built for a stream that is gone, so it is not replayable. + // Its retry re-runs the pipeline once under this play, like any first attempt. + preparedDeliveries.synchronized(preparedDeliveries.clear()) + // Under the send lock like every other touch of the observer field: it is read by + // [[sendResponse]], [[failAndComplete]] and [[stop]] on other threads, and an unserialized + // write here could interleave with a stop completing the PREVIOUS observer. + // + // ONE RESUME GENERATION AT A TIME: a predecessor still wired here is a play stream some + // client is still holding open (a second tab, a double Play), and silently orphaning it + // left that stream starving forever - nothing would ever write to it again. It is + // COMPLETED, under this same lock, so the swap and the goodbye are one atomic step. + sendLock.synchronized { + if !streamCompleted then + this.grpcResponseObserver.filter(_ ne grpcResponseObserver).foreach(previous => Try(previous.onCompleted())) + this.grpcResponseObserver = Some(grpcResponseObserver) + idleSinceNanos = None + } + this.includeConsumerStats = includeConsumerStats + + // Both delivery controls are per RESUME, like the two flags above them - they belong to + // the browser that pressed Play, not to the session's saved definition. Configured before + // the targets resume below, so the gate's volatile write publishes the limiter to the + // listener threads along with everything else this resume rewired. The user's resume also + // resumed every consumer regardless of any permit hold the limiter had placed, so the + // limiter is told to forget it - its next watermark crossing re-asserts the hold instead + // of believing a stale flag. + // + // The delivery budget counts LOADED messages at the send site below; it needs the queue + // (forceQueue) so the drain can stop the line BETWEEN messages - the budget's whole + // contract is that the message spending the last unit is the last one sent. + remainingToDeliver.set(if maxMessagesToDeliver > 0 then maxMessagesToDeliver else Long.MaxValue) + deliveryRateLimiter.core.setRate(math.max(0, maxMessagesPerSecond)) + deliveryRateLimiter.core.setForceQueue(maxMessagesToDeliver > 0) + targets.values.foreach(_.consumerListener.deliveryRateLimiter = Some(deliveryRateLimiter)) + + // THE GUARANTEED REPLAY BOUNDARY IS THE MOMENT PLAY WAS PRESSED - and CREATE IS THE + // FIRST PLAY. The boundaries decided at session build - the real recorded ends of the + // history seeks, and the deliberately EMPTY ends of the live-edge seeks (Latest, + // latest-n's non-contributors, the approximate-entry endpoint) - are already armed in + // the ordering layer and ARE the first chunk's boundaries, so the first Play CONSUMES + // them. Re-reading the broker here instead OVERWROTE a live-edge stream's empty + // boundary with the backlog's real end - a range the consumer's cursor already sits + // PAST, so nothing could ever drain it: Latest x Guaranteed on a non-empty topic armed + // `waiting` over undeliverable history and the caught-up never fired (e2e CS-DM-R3B). + // Only a LATER resume re-captures fresh ends and EXTENDS the boundary - the one rule + // for a manual pause's resume and the auto-pause's alike, and correct there because the + // cursor genuinely sits at the previous chunk's boundary, at or before any new end: the + // un-emitted remainder and the freshly recorded delta merge in one heap and sort + // exactly. A failed re-capture fails that resume loudly, exactly as a target that will + // not resume does. Sequenced deliberately: the generation was bumped FIRST (top of this + // method), the extension runs under the session's ordering lock HERE, and the consumers + // - the Boundary holds included - are released only in the target walk BELOW; that + // order is what makes a stale auto-pause and a past-end nack racing this re-capture + // converge instead of stomping the play (see announceReplayCaughtUp). + if guaranteedOrderingArmed then + val firstPlay = !replayCreateBoundaryConsumed + val boundaryStreams = + if firstPlay then Vector.empty + else startFromStreamsAt(targets.values.flatMap(_.consumers.values).toVector) + replayCaughtUpNow = false + replayNewerEntriesApprox = 0L + replayExcludedTopics = Vector.empty + replayExcludedTopicCount = 0 + // "Caught up to " reports when the STANDING boundary was captured: session build + // for the first chunk, this resume for an extension. + replayBoundaryAtMs = if firstPlay then replayBoundaryCapturedAtMs else System.currentTimeMillis() + if !firstPlay then targets.values.headOption.foreach(_.consumerListener.extendReplayBoundary(boundaryStreams)) + replayCreateBoundaryConsumed = true + + // THE CAUGHT-UP HOOK, wired per play like the limiter above: the pump invokes it (under + // the session's ordering lock) whenever the replay barrier reports the chunk complete. + // The generation gate plus the once-flag make it one announcement per chunk - and make a + // stale announcement impossible: a Resume bumps the generation BEFORE its boundary + // extension takes the same ordering lock, so a hook running after that sees a foreign + // generation and does nothing. + val replayCaughtUpAnnounced = java.util.concurrent.atomic.AtomicBoolean(false) + def announceReplayCaughtUp(): Unit = + if playGeneration.get == myGeneration && replayCaughtUpAnnounced.compareAndSet(false, true) then + replayNewerEntriesApprox = + targets.values.headOption.map(_.consumerListener.startFromOrdering.replayNewerEntriesSeen).getOrElse(0L) + replayCaughtUpNow = true + // THE RUNNER-SIDE AUTO-PAUSE: close every intake gate and hold every consumer + // still - the arbiter's Boundary reason, so the user's own pause state and the + // merge's flow-control holds stay untouched. Arbiters are leaf locks the + // flow-control reconcile already takes under the ordering lock on every handled + // batch, so this introduces no new lock ordering. The full user-pause path + // (permit lock, limiter quiesce) is deliberately NOT reused here: it takes + // lifecycle-side locks this thread must not acquire under the ordering lock, and + // a caught-up barrier has nothing in flight for it to quiesce anyway. + targets.values.foreach { target => + target.consumerListener.stopAcceptingNewMessages() + target.pauseArbiters.values.foreach(_.hold(PauseReason.Boundary)) + } + permitLogger.info( + s"Consumer session $sessionName reached its replay boundary and auto-paused; " + + "the caught-up signal was pushed to the client." + ) + // The signal frame: message-less, stats-bearing - the client learns the state + // change from it. Best-effort like every progress push; the state itself is + // already recorded above and rides every later response too. + if includeConsumerStats then Try(sendResponse(grpcResponseObserver, Seq.empty, Vector.empty)) + scheduleReplayIndicatorRefinement(myGeneration, grpcResponseObserver) + targets.values.foreach(_.consumerListener.onReplayCaughtUp = () => announceReplayCaughtUp()) + // The drain is re-armed at the BOTTOM of this method, after every target has rewired its + // handlers onto this resume's observer. Re-arming here scheduled a zero-delay tick that + // could run a queued tail through the PREVIOUS play's handler closure - sending messages + // to the cancelled observer, acknowledging them, and charging them to THIS resume's + // delivery budget. The messages were consumed and never seen. + + // A message the start-from discard swallowed reaches nobody: the listener drops it before the + // message handler, so nothing on the path below fires while a skip is in flight. Without + // this push a session skipping millions of messages looks hung - it delivers nothing and + // says nothing. Runs on the Pulsar listener threads, one per consumer, and on the sweep + // timer - hence ATOMIC once-only flags, not plain vars: two threads racing the same + // disclosure must produce exactly one claiming winner, not a duplicated or dropped frame. + val degradationReported = java.util.concurrent.atomic.AtomicBoolean(false) + val stallWarningsReported = AtomicLong(0L) + def reportStartFromDiscardProgress(): Unit = + // Each NEW stall warning bypasses every other gate: a stalled merge drops nothing and + // delivers nothing, so no interval boundary would ever be crossed - and the client + // would show "awaiting new messages" over a session that is stuck WAITING. The + // waiting-streams count rides on every response's consumer stats; what is decided + // here is only WHEN a message-less frame is pushed: once per new warning (the merge + // re-warns only after a stall resolves and a fresh one begins), never per held + // message or per sweep tick. `getAndAccumulate(max)` is the atomic claim - exactly + // one caller sees the advance. + val stallWarningsNow = targets.values.map(_.consumerListener.startFromStallWarnings).maxOption.getOrElse(0L) + val mustDiscloseStall = includeConsumerStats + && stallWarningsReported.getAndAccumulate(stallWarningsNow, math.max(_, _)) < stallWarningsNow + // The debug-level WHY behind the count the client is shown - which streams, and + // whether each is verifiably out of messages or merely lagging. Throttled with the + // disclosure itself, and scheduled off-thread: the classification reads each topic's + // end from the broker, and this closure runs under the ordering lock. + if mustDiscloseStall then scheduleStalledStreamDiagnostic() + reportableStartFromProgress match + case None => + // NO COUNTED SKIP AT ALL - an ordering-only session, whose budget is a shared + // zero. This used to fall out of the disclosure entirely: a stalled GUARANTEED + // session wrote no ResumeResponse ever, indistinguishable from an empty topic. + // The stall is the one thing such a session still has to say. + if mustDiscloseStall then sendResponse(grpcResponseObserver, Seq.empty, Vector.empty) + case Some(progress) => + // The FIRST degraded frame bypasses the interval gate. A give-up may resolve + // only a handful of drops - or none at all - and a session that then goes + // quiet would never cross another interval boundary: the disclosure would sit + // in the server forever, which is the exact silent failure the flag exists to + // prevent. The compare-and-set is the once-only claim. + val mustDiscloseDegradation = progress.degraded && degradationReported.compareAndSet(false, true) + if mustDiscloseDegradation + || mustDiscloseStall + || shouldReportStartFromProgress(progress.messagesSkipped, progress.messagesToSkip, startFromProgressReportInterval) + then + if mustDiscloseDegradation then + // Log-worthy in its own right: this is the moment the session's answer + // became best-effort, and the one frame the client's banner hangs on. + permitLogger.info( + s"Disclosing start-from degradation to the client: abandoned ${progress.abandonedStreams.mkString(", ")}" + ) + // Message-less on purpose: nothing was delivered, only the counters moved. + // The client reads the stats before it looks for a trailing message. + sendResponse(grpcResponseObserver, Seq.empty, Vector.empty) + + /** Write what one attempt prepared, remembering it until the write takes. + * + * The memo goes in BEFORE the send and comes out only after it returned: a send that + * throws leaves the prepared response behind, and the retry - a broker redelivery, or + * Guaranteed's in-place retry - re-sends exactly this instead of running the pipeline + * again. See [[preparedDeliveries]]. + */ + def sendPrepared(deliveryKey: Option[DeliveryKey], prepared: PreparedDelivery): Unit = + deliveryKey.foreach(rememberPrepared(_, prepared)) + sendResponse(grpcResponseObserver, prepared.messages, prepared.errors) + deliveryKey.foreach(forgetPrepared) + // The budget is charged AFTER the send took: a refused or failed send hands the + // message back (nack, retry) and must not burn a unit on a delivery nobody + // received. The last-unit guarantee is intact - the pause lands before this + // handler returns, so everything behind it in a drain batch is requeued. + if prepared.spendsBudgetUnit && remainingToDeliver.decrementAndGet() == 0 then deliveryRateLimiter.pauseDraining() + + /** WHAT THE TARGET HANDLER ASKS BEFORE IT DOES ANYTHING AT ALL. + * + * The generation gate lives here, at the very front of the delivery, rather than in the + * session callback below: by the time a handler reaches that callback it has already + * advanced both processed counters, deserialized the message, leased the JS context and + * run the target's filters, coloring rules and projections - so a second Play or a Stop + * crossing an in-flight delivery left all of that behind, and the redelivery then let the + * user's stateful JavaScript observe the same message a second time. + * + * The same question answers the retry of a FAILED send: a response this play already + * prepared for this message is re-sent here, and the pipeline behind it is skipped. + */ + def admitDelivery(deliveryKey: Option[DeliveryKey]): DeliveryAdmission = + if playGeneration.get != myGeneration then DeliveryAdmission.Superseded + else + deliveryKey.flatMap(preparedFor(_, myGeneration)) match + case Some(prepared) => + sendPrepared(deliveryKey, prepared) + DeliveryAdmission.Replayed + case None => DeliveryAdmission.Prepare def onNext( messageFromTarget: Option[ConsumerSessionMessage], sessionContext: ConsumerSessionContext, stats: ConsumerSessionTargetStats, - errors: Vector[String] + errors: Vector[String], + deliveryKey: Option[DeliveryKey] ): Unit = boundary: - def createAndSendResponse(messages: Seq[consumerPb.Message], additionalErrors: Vector[String] = Vector.empty): Unit = - val allErrors = errors ++ additionalErrors - - val status = allErrors.size match - case 0 => Status(code = Code.OK.index) - case _ => Status(code = Code.UNKNOWN.index, message = allErrors.mkString("\n\n")) - - val response = consumerPb.ResumeResponse( - messages = messages, - status = Some(status) - ) - grpcResponseObserver.onNext(response) + // THE GENERATION GATE AGAIN, immediately before the send. The target checked it before + // its first mutation; this is the recheck for work that CROSSED a replacement while it + // ran - the session filters, coloring rules and projections below all take time. The + // throw routes the message down the delivery-failure path: handed back for redelivery, + // recorded in the failed-delivery registry, and delivered exactly once - late - by the + // machinery of the play that is actually current. + if playGeneration.get != myGeneration then + throw new IllegalStateException("This play was superseded; the message is handed back for redelivery.") + def createAndSendResponse( + messages: Seq[consumerPb.Message], + additionalErrors: Vector[String] = Vector.empty, + spendBudgetUnit: Boolean = false + ): Unit = + sendPrepared(deliveryKey, PreparedDelivery(myGeneration, messages, errors ++ additionalErrors, spendBudgetUnit)) boundary.break(()) messageFromTarget match @@ -59,7 +1129,10 @@ case class ConsumerSessionRunner( numMessageProcessed = numMessageProcessed, numMessageSent = numMessageSent ) - createAndSendResponse(Seq(emptyMsgPb), errors) + // NOT `errors` again: createAndSendResponse already appends the captured + // `errors` to every response, so passing them here too doubled every + // target-filter debug error the client saw. + createAndSendResponse(Seq(emptyMsgPb)) case Some(msg) => val messageFilterChainResult = sessionContext.testMessageFilterChain( @@ -102,7 +1175,7 @@ case class ConsumerSessionRunner( messageFilterChainErrors ++ coloringRuleChainErrors else Vector.empty - numMessageSent = numMessageSent + 1 + numMessageSentCounter.incrementAndGet() val messageToSendPb = msg.messagePb .withSessionContextStateJson(sessionContext.getState) @@ -113,65 +1186,456 @@ case class ConsumerSessionRunner( .withNumMessageSent(numMessageSent) .withNumMessageProcessed(numMessageProcessed) - createAndSendResponse(Seq(messageToSendPb), errors) + createAndSendResponse(Seq(messageToSendPb), errors, spendBudgetUnit = true) targets.values.foreach(_.resume( onNext = onNext, isDebug = isDebug, - incrementNumMessageProcessed = incrementNumMessageProcessed + incrementNumMessageProcessed = incrementNumMessageProcessed, + onStartFromDiscardProgress = reportStartFromDiscardProgress, + admitDelivery = admitDelivery )) + + // LAST, deliberately: only now does every listener's handler point at THIS observer, so a + // queued tail from the previous play drains into the resume that asked for it. See the + // comment where the limiter is configured above. + deliveryRateLimiter.resumeDraining() + // A pause held every source, so paused time proves nothing about a stream's health: hand + // the silent-stream clock a fresh window instead of letting the first sweep after resume + // abandon a stream that was merely held along with everything else. + targets.values.headOption.foreach(_.consumerListener.resetStartFromStallClock()) + armStallSweep() + // Drive the replay barrier once, now that everything is wired and the consumers run: an + // EMPTY replay (Latest, a single non-persistent stream, no delta after an extension) + // will never see an offer to pump it, and its instant caught-up must fire on Play - not + // never. A non-empty chunk's held remainder resumes emission here too, without waiting + // for the first new offer or the next sweep tick. + if guaranteedOrderingArmed then targets.values.headOption.foreach(_.consumerListener.pumpGuaranteedDelivery()) def pause(): Unit = - targets.values.foreach(_.pause()) + // THE DRAINER STOPS FIRST, before a single target is walked. A paused session delivers + // NOTHING, including from the limiter's backlog - and this used to run in the `finally` + // BELOW the target walk, so on a wide session the queued backlog went on being delivered + // and acknowledged for the whole length of a walk over up to 2,000 consumers, long after + // the user asked the session to stop. It returns only once nothing is still on its way out + // (see [[DeliveryRateLimiter.pauseDraining]]), so what follows cannot race a send. + deliveryRateLimiter.pauseDraining() + // The step that must NEVER be skipped runs in the finally: one target throwing used to + // abandon the rest un-attempted AND leave the watchdog sweeping a session the user + // believed paused. Every target is attempted (each target attempts every consumer - see + // ConsumerSessionTargetRunner.pause), failures are aggregated, and the aggregate is thrown + // only after the session is in the safest state this call can reach. + try + val failures = targets.values.toVector.flatMap { target => + Try(target.pause()).failed.toOption.map(err => s"target ${target.targetIndex}: ${err.getMessage}") + } + if failures.nonEmpty then throw new RuntimeException(failures.mkString("; ")) + finally cancelStallSweep() + + /** Release everything this session owns: its consumers and their subscriptions, its GraalVM + * contexts, and the client's response stream. + * + * EVERY step runs even if an earlier one failed, and the failures are aggregated into one + * exception at the end. It used to swallow unsubscribe failures entirely and close nothing at + * all, so `deleteConsumer` removed the only handle to the session and answered OK while the + * subscription stayed on the broker, the consumers stayed connected, the Graal context stayed + * open and the browser kept a stream that was never completed. + * + * Idempotent: each target clears its consumer map, closing a Graal context twice is a no-op, + * and the observer is forgotten once completed. + * + * ENDING THE STREAM IS ITSELF A WRITE TO IT, so it goes through [[sendLock]] like every other + * one. `onCompleted` used to be called outside that lock with no terminal flag at all, so it + * could interleave with an `onNext` still in flight on a listener thread - and any push after + * it wrote to a stream that had already ended. + */ def stop(): Unit = - pause() - targets.values.foreach(_.stop()) + // FIRST and unconditionally: no handler may do stateful work for a session that is on + // its way out (a continuous sweep can be mid-delivery on the session's own timer thread), + // and the shared maintenance task must never keep touching a dying runner. + playGeneration.incrementAndGet() + cancelStallSweep() + Try(pause()) + // The limiter's backlog dies with the session: the messages were never acknowledged, and + // the NonDurable subscriptions being released below take any redelivery question with + // them. Clearing promptly is about freeing the payloads, not about correctness. + Try(deliveryRateLimiter.stop()) + synchronized { rateLimiterExecutor }.foreach(executor => Try(executor.shutdownNow())) + // An UNEXPECTED throw out of a target's own stop used to become an empty failure vector, so + // whatever it failed to release was reported as released. It is a failure like any other. + val targetFailures = targets.values.toVector.flatMap { target => + Try(target.stop()) match + case Success(failures) => failures + case Failure(err) => Vector(s"target ${target.targetIndex}: ${err.getMessage}") + } + // The pool's own failures count too: a JS context that will not close holds its heap for the + // life of the process, and this used to be discarded twice over - swallowed inside `close` + // and then discarded again here. + val poolFailures = Try(sessionContextPool.close()) match + case Success(failures) => failures + case Failure(err) => Vector(s"JS context pool: ${err.getMessage}") + val failures = targetFailures ++ poolFailures + sendLock.synchronized { + if !streamCompleted then + streamCompleted = true + grpcResponseObserver.foreach(observer => Try(observer.onCompleted())) + grpcResponseObserver = None + idleSinceNanos = idleSinceNanos.orElse(Some(System.nanoTime())) + } + if failures.nonEmpty then + throw new RuntimeException(s"Consumer session $sessionName could not be fully released. ${failures.mkString("; ")}") } +/** The logger [[storeConsumerSession]] reports through - a top-level function needs its own name + * for the log line to be attributable (and for a test to listen on). */ +private val storeConsumerSessionLogger = com.typesafe.scalalogging.Logger("consumer.session_runner.storeConsumerSession") + +/** Store a freshly built session under its name, STOPPING whatever it replaced. + * + * Creating a session under a name that already existed simply overwrote the entry, and the old + * runner's consumers went on holding their subscriptions and delivering messages into a session + * nothing could reach - for the life of the process. The browser re-creates a session on an + * ordinary configuration change, so this was the common path, not a corner. + * + * The replacement goes in FIRST and unconditionally: a predecessor that cannot be released must + * not make its name permanently unusable, so the failure is LOGGED HERE rather than propagated - + * it used to be discarded outright, while this comment claimed the caller logged it, so a + * predecessor that failed to release vanished without a trace in exactly the situation an + * operator needs one. `ConcurrentHashMap.put` is atomic, so two concurrent creates leave exactly + * one session stored and the other stopped. + */ +def storeConsumerSession( + sessions: java.util.concurrent.ConcurrentHashMap[String, ConsumerSessionRunner], + sessionName: String, + session: ConsumerSessionRunner +): Unit = + Option(sessions.put(sessionName, session)).foreach(replaced => + Try(replaced.stop()).failed.foreach(err => + storeConsumerSessionLogger.warn( + s"The consumer session being replaced under $sessionName could not be fully released. ${err.getMessage}" + ) + ) + ) + () + object ConsumerSessionRunner: + + /** Whether the selected delivery mode needs an ordering layer. + * + * Best effort needs one only ACROSS sources - one Pulsar topic or partition has no + * cross-source order to decide, so a single stream stays on the free pass-through path. + * GUARANTEED builds its layer at ANY stream count since the exact-replay redesign (owner + * decision 2026-08-09): the replay boundary, the finished-stream barrier, the auto-pause + * and the caught-up signal all live in the layer, and a one-partition replay needs them + * exactly as much as a fifty-partition one. + */ + def needsDeliveryOrderLayer(order: MessageDeliveryOrder, streamCount: Int): Boolean = + order match + case MessageDeliveryOrder.Guaranteed => true + case MessageDeliveryOrder.AsReceived => false + case _ => streamCount > 1 + + /** How many excluded-topic NAMES the caught-up signal carries; the count field carries the + * rest. A regex can match a whole namespace of late joiners, and the banner needs a few + * names, not a wall. */ + val replayExcludedTopicNamesCap: Int = 5 + + private def isReadCompactedTarget(targetConfig: consumer.session_target.ConsumerSessionTarget): Boolean = + targetConfig.consumptionMode.mode match + case _: consumer.session_target.consumption_mode.modes.ReadCompactedConsumptionMode => true + case _ => false + + /** Why ordering by broker publish time must be refused against this broker configuration, + * or None when it is available - or when the configuration is UNREADABLE, which must not + * block anybody (no broker-side steps are ever required to use Dekaf; the per-message + * fallback counter discloses instead). Pure, so the boundary is pinned without a broker. */ + def brokerPublishTimeUnavailableReason(brokerConfig: Option[Map[String, String]]): Option[String] = + brokerConfig.flatMap { config => + val stamps = config + .get("brokerEntryMetadataInterceptors") + .exists(_.contains("AppendBrokerTimestampMetadataInterceptor")) + val exposes = config + .get("exposingBrokerEntryMetadataToClientEnabled") + .exists(_.trim.equalsIgnoreCase("true")) + Option.when(!(stamps && exposes))( + "Broker publish time is unavailable because this cluster does not stamp and expose broker " + + "timestamp entry metadata. Choose Publish time or Event time, or ask an administrator to " + + "enable AppendBrokerTimestampMetadataInterceptor and exposingBrokerEntryMetadataToClientEnabled." + ) + } + + /** The most ENABLED targets one session may run. The per-target topic cap + * ([[ConsumerSessionTargetRunner.maxTopicsPerTarget]]) bounded one target, but the UI appends + * targets without limit, so N targets multiplied that bound right back away. Generous - a + * session this wide is already unusable interactively - but a bound. */ + val maxEnabledTargetsPerSession: Int = 25 + + /** The most physical topic streams one session may run in total, across every enabled target. + * Each stream is a consumer, a subscription and a receiver queue; the per-target and + * per-session-target caps alone would still admit maxEnabledTargets x maxTopicsPerTarget + * streams, which is not an interactive session, it is a load test. */ + val maxStreamsPerSession: Int = 2_000 + + /** Why a session with this many enabled targets must be refused, or None if it is admissible. + * Pure so the boundary is unit-testable without a broker; [[make]] wires it in. */ + def enabledTargetCountRejectionReason(sessionName: String, enabledTargetCount: Int): Option[String] = + Option.when(enabledTargetCount > maxEnabledTargetsPerSession)( + s"Consumer session $sessionName has $enabledTargetCount enabled targets, more than the $maxEnabledTargetsPerSession " + + "one session can run. Disable some targets or split the work across several sessions." + ) + + /** Why a session whose enabled targets resolve to these many physical streams must be refused, + * or None if it is admissible. Counted per target, NOT distinct across targets: two targets + * selecting the same topic each keep their own consumer on it, so each costs a stream. */ + def sessionStreamTotalRejectionReason(sessionName: String, streamCountsByTarget: Vector[Int]): Option[String] = + val total = streamCountsByTarget.sum + Option.when(total > maxStreamsPerSession)( + s"Consumer session $sessionName resolves to $total physical topic streams across its enabled targets " + + s"(${streamCountsByTarget.mkString(" + ")}), more than the $maxStreamsPerSession one session can run. " + + "Narrow the topic selectors (tighter regexes, or specific topics) or split the work across several sessions." + ) + /** ONE daemon thread for every session's periodic upkeep (the start-from stall sweeps). Each + * armed sweep is a tiny check every couple of seconds, and it self-cancels the moment its + * skip resolves - a thread per session outlived its one job by the whole session lifetime. + * The rare give-up DRAIN does run session work here (delivering what a silent stream held + * back), which is accepted: it fires at most once per abandoned stream. The delivery rate + * limiter keeps its per-session timer - its ticks do real per-message work under the + * session's own locks and must not serialize sessions against each other. + */ + private[consumer] lazy val maintenanceScheduler: ScheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(runnable => { + val thread = Thread(runnable, "consumer-session-maintenance") + thread.setDaemon(true) + thread + }) + /** @param sessionContextPool + * the session's GraalVM engine and JS contexts. A parameter (with the production default) + * only so a test can prove the pool is RELEASED when construction fails - it is created + * before anything else and so is the first thing that can be leaked. + */ def make( pulsarClient: PulsarClient, adminClient: PulsarAdmin, sessionName: String, - sessionConfig: ConsumerSessionConfig + sessionConfig: ConsumerSessionConfig, + sessionContextPool: ConsumerSessionContextPool = ConsumerSessionContextPool() ): ConsumerSessionRunner = - val sessionContextPool = ConsumerSessionContextPool() + var targets: Map[ConsumerSessionTargetIndex, ConsumerSessionTargetRunner] = Map.empty + + // Admission on what the CONFIG alone already tells us, before any broker work: past this + // many targets there is nothing to resolve or subscribe, only a refusal to deliver. + enabledTargetCountRejectionReason(sessionName, sessionConfig.targets.count(_.isEnabled)).foreach { reason => + Try(sessionContextPool.close()) + throw new IllegalArgumentException(reason) + } + + // EVERYTHING THIS SESSION OWNS IS RELEASED IF ANY OF IT FAILS, and the pool is inside that + // guard from the very first step. It used to be built before the guard existed, so a target + // that failed to build released the targets built before it (`buildAllOrRelease`) and left + // the GraalVM engine and its JS contexts open with nothing holding a handle to them - a + // whole engine leaked per failed create, and the browser retries a failed create. + def releasingSession[A](build: => A): A = + try build + catch + case err: Throwable => + targets.values.foreach(target => Try(target.stop())) + Try(sessionContextPool.close()) + throw err + + // RESOLVE EVERY SELECTOR BEFORE ANY CONSUMER EXISTS. Three session-wide decisions need + // the full picture first: the per-target and session-total admission caps (refusing + // AFTER subscribing meant a doomed session transiently created hundreds of consumers + // just to release them), the empty-target refusal, and the best-effort-order prefetch + // budget, which divides across the session's ACTUAL stream total - a number no single + // target can know. + val resolvedTargets: Vector[((ConsumerSessionTarget, Int), Vector[NonPartitionedTopicFqn])] = + releasingSession(sessionConfig.targets.filter(_.isEnabled).zipWithIndex.map { (targetConfig, i) => + val fqns = targetConfig.topicSelector.getNonPartitionedTopics(adminClient = adminClient) + // A namespaced-regex selector can match a whole namespace; past this many + // physical topics the session would spend its life subscribing and every counted + // mode's per-message work scales with the count - refusing loudly beats grinding + // into a session nobody can use. + if fqns.size > ConsumerSessionTargetRunner.maxTopicsPerTarget then + throw new IllegalArgumentException( + s"Target $i resolves to ${fqns.size} physical topics, more than the ${ConsumerSessionTargetRunner.maxTopicsPerTarget} " + + "one session can handle. Narrow the topic selector (a tighter regex, or specific topics) or split the work " + + "across several sessions." + ) + ((targetConfig, i), fqns) + }) - var targets = sessionConfig.targets - .filter(_.isEnabled) - .zipWithIndex.map { case (targetConfig, i) => + val emptyTargets = resolvedTargets.collect { case ((_, i), fqns) if fqns.isEmpty => i } + if emptyTargets.nonEmpty then + releasingSession(throw new IllegalArgumentException( + s"Consumer session $sessionName has enabled targets that resolved to no topics: ${emptyTargets.mkString(", ")}." + )) + + sessionStreamTotalRejectionReason(sessionName, resolvedTargets.map(_._2.size)) + .foreach(reason => releasingSession(throw new IllegalArgumentException(reason))) + + // Refuse target/start-position combinations before constructing a subscription. The + // information is already present in `resolvedTargets`; building up to 2,000 consumers only + // to release them again is both observable outside Dekaf and needlessly expensive. + val readCompactedTargetIndexes = resolvedTargets.collect { + case ((targetConfig, targetIndex), _) if isReadCompactedTarget(targetConfig) => targetIndex + }.sorted + readCompactedStartFromRejectionReason(sessionConfig.startFrom, readCompactedTargetIndexes) + .foreach(reason => releasingSession(throw new IllegalArgumentException(reason))) + skipOverlapRejectionReason(sessionConfig.startFrom, resolvedTargets.map(_._2)) + .foreach(reason => releasingSession(throw new IllegalArgumentException(reason))) + + val totalSessionStreams = resolvedTargets.map(_._2.size).sum + // A single delivery stream is already ordered as Pulsar delivers it. The selected mode is + // therefore a no-op: do not reject it, reduce prefetch, or inspect timestamps for a merge + // layer that will not exist. + val orderingLayerNeeded = needsDeliveryOrderLayer(sessionConfig.messageDeliveryOrder, totalSessionStreams) + + // GUARANTEED ordering refuses a MERGE over non-persistent topics up front: such a topic + // retains nothing and may be legitimately silent forever, so waiting on it inside a + // multi-stream barrier would mean a session that can never speak. Scoped to more than + // one stream, exactly as before the replay redesign made Guaranteed build a layer at + // any width: a SINGLE non-persistent stream stays legal - it retains nothing, so its + // replay boundary is empty and the session answers with an instant caught-up instead of + // a refusal (owner decision 2026-08-09, design item on non-persistent). + if totalSessionStreams > 1 && sessionConfig.messageDeliveryOrder == MessageDeliveryOrder.Guaranteed then + val nonPersistent = resolvedTargets.flatMap(_._2).filter(isNonPersistentTopic).distinct + if nonPersistent.nonEmpty then + releasingSession(throw new IllegalArgumentException( + s"Guaranteed ordering is not supported when merging non-persistent topics " + + s"(${nonPersistent.mkString(", ")}). Choose Best effort or Fastest, or remove those topics." + )) + + // Ordering by BROKER PUBLISH TIME needs the broker to stamp and expose entry metadata. + // Dekaf requires NO broker changes: when the broker's effective configuration is + // readable and clearly lacks the two settings, the create is refused with the exact + // remediation; when it is not readable (permissions, old broker), the session proceeds + // and the per-message fallback counter discloses instead. Scoped to MORE THAN ONE + // stream, the pre-replay boundary of this refusal: a single-stream Guaranteed session + // now builds a layer too, but one log is already in append order - the key only feeds + // seam detection - so a session that was accepted before must not become a refusal, and + // the fallback counter discloses there instead. + if orderingLayerNeeded && totalSessionStreams > 1 && sessionConfig.deliveryOrderKey == DeliveryOrderKey.BrokerPublishTime then + ConsumerSessionRunner + .brokerPublishTimeUnavailableReason( + Try(adminClient.brokers().getRuntimeConfigurations.asScala.toMap).toOption + ) + .foreach(reason => releasingSession(throw new IllegalArgumentException(reason))) + + // ALL OR NOTHING, at the target level as well as inside each target: a session builds one + // runner per enabled target, and a plain `map` left every target built before a failing one + // subscribed and unreachable. + targets = releasingSession(buildAllOrRelease[((ConsumerSessionTarget, Int), Vector[NonPartitionedTopicFqn]), (Int, ConsumerSessionTargetRunner)]( + inputs = resolvedTargets, + build = resolvedTarget => + val ((targetConfig, i), fqns) = resolvedTarget i -> ConsumerSessionTargetRunner.make( sessionName = sessionName, targetIndex = i, pulsarClient = pulsarClient, - adminClient = adminClient, schemasByTopic = Map.empty, sessionContextPool = sessionContextPool, - targetConfig = targetConfig + targetConfig = targetConfig, + nonPartitionedTopicFqns = fqns, + receiverQueueSize = + if orderingLayerNeeded then mergeReceiverQueueSizeFor(totalSessionStreams) + else receiverQueueSizeFor(fqns.size) ) - }.toMap + , + release = (_, target) => target.stop() + ).toMap) + + // A session with nothing to consume from used to be accepted: `make` returned a runner with + // an empty consumer map and ConsumerServiceImpl.createConsumer answered Code.OK, so the UI + // showed a session in state `running` that could never deliver a message and never said + // why. Reject it here, before any seeking, so the client gets a real non-OK status. + if targets.isEmpty then + Try(sessionContextPool.close()) + throw new IllegalArgumentException( + s"Consumer session $sessionName has no enabled targets." + ) - val nonPartitionedTopicFqns = targets.values.flatMap(_.nonPartitionedTopicFqns).toVector - val schemasByTopic = getSchemasByTopic(adminClient, nonPartitionedTopicFqns) + + + // DISTINCT: two enabled targets may legitimately select the same topic, and each keeps its + // own consumer on it. What this vector feeds - schema lookup, the non-persistent rejection, + // the Message-ID lookup and the single-topic fast path - all ask ABOUT a topic rather than + // consume from it, and asking twice about one topic broke the Message-ID mode outright. + val nonPartitionedTopicFqns = targets.values.flatMap(_.nonPartitionedTopicFqns).toVector.distinct + val schemasByTopic = releasingSession(getSchemasByTopic(adminClient, nonPartitionedTopicFqns)) targets = targets.map { case (targetIndex, target) => targetIndex -> target.copy(schemasByTopic = schemasByTopic) } + // The delivery order's timestamp, wired per listener so each keeps its own fallback + // counter (a message without the selected timestamp uses publish time and is counted - + // the client turns the first count into a remediation notice). Only an ordering session + // pays any of this; as-received keeps the plain publish-time default untouched. + if orderingLayerNeeded then + targets.values.foreach { target => + val listener = target.consumerListener + listener.orderingTimeOf = sessionConfig.deliveryOrderKey match + case DeliveryOrderKey.PublishTime => _.getPublishTime + case DeliveryOrderKey.BrokerPublishTime => + msg => + val stamped = msg.getBrokerPublishTime + if stamped.isPresent then stamped.get + else + listener.orderKeyFallbacks.incrementAndGet() + msg.getPublishTime + case DeliveryOrderKey.EventTime => + msg => + val eventTime = msg.getEventTime + if eventTime > 0 then eventTime + else + listener.orderKeyFallbacks.incrementAndGet() + msg.getPublishTime + } + val consumers = targets.values.flatMap(_.consumers).map(_._2).toVector - handleStartFrom( - startFrom = sessionConfig.startFrom, - consumers = consumers, - adminClient = adminClient, - pulsarClient = pulsarClient, - nonPartitionedTopicFqns = nonPartitionedTopicFqns + val startFromPlan = releasingSession { + handleStartFrom( + startFrom = sessionConfig.startFrom, + consumers = consumers, + adminClient = adminClient, + pulsarClient = pulsarClient, + nonPartitionedTopicFqns = nonPartitionedTopicFqns, + deliveryOrdering = sessionConfig.messageDeliveryOrder + ) + } + + // Arm the start-from discard ONCE, here: the seek has happened and no consumer has been + // resumed yet. `resume` deliberately knows nothing about it - re-arming on every play would + // skip a fresh batch of messages each time the session is paused and resumed. + // + // A SharedTotal plan is ONE counter over the merged stream, so every target must get the + // SAME instance; a PerTopic plan gets a fresh counter per target, because two targets may + // select the same topic and each has its own consumer to correct. + val sharedDiscard = startFromPlan.discard match + case StartFromDiscardPlan.SharedTotal(n) => StartFromDiscard.shared(n) + case _ => StartFromDiscard.none + + // The global ordering layer is ONE object for the whole session for the same reason a + // SharedTotal counter is: "the globally-first n" and "the globally-last n" are counted over + // every target's stream at once, so a layer per target would answer n per target. + val ordering = StartFromOrdering.make[HeldMessage]( + startFromPlan.ordering, + // The byte half of the merge's memory watermarks: what one held message costs. + payloadBytesOf = held => scala.util.Try(Option(held.message.getData).map(_.length.toLong).getOrElse(0L)).getOrElse(0L) ) + targets.values.foreach { target => + target.consumerListener.startFromDiscard = + StartFromDiscard.forTarget(startFromPlan.discard, sharedDiscard, target.nonPartitionedTopicFqns) + target.consumerListener.startFromOrdering = ordering + } + ConsumerSessionRunner( sessionName = sessionName, sessionConfig = sessionConfig, schemasByTopic = schemasByTopic, sessionContextPool = sessionContextPool, targets = targets, - grpcResponseObserver = None + grpcResponseObserver = None, + adminClient = Some(adminClient) ) diff --git a/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetRunner.scala b/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetRunner.scala index c5e792a2d..07a3a49dd 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetRunner.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetRunner.scala @@ -14,6 +14,7 @@ import org.apache.pulsar.client.api.Message import scala.util.boundary import boundary.break import scala.util.{Failure, Success, Try} +import java.util.concurrent.atomic.AtomicLong type NonPartitionedTopicFqn = String @@ -24,32 +25,115 @@ case class ConsumerSessionTargetRunner( schemasByTopic: SchemasByTopic, sessionContextPool: ConsumerSessionContextPool, var consumers: Map[NonPartitionedTopicFqn, Consumer[Array[Byte]]], + // One arbiter per consumer: user pause, merge flow control and the delivery pacer all hold + // and release through these, never through consumer.pause()/resume() directly - see + // [[ConsumerPauseArbiter]] for the stomping this ended. + val pauseArbiters: Map[NonPartitionedTopicFqn, ConsumerPauseArbiter], var consumerListener: ConsumerListener, var stats: ConsumerSessionTargetStats ) { + /** @param admitDelivery + * what the SESSION says about this delivery attempt before this target does anything for + * it. It carries the play-generation gate - which has to be asked HERE, in front of the + * first mutation, not in the session callback at the end - and the re-send of a response a + * previous attempt already prepared for the same message. See [[DeliveryAdmission]]. + */ def resume( onNext: ( msg: Option[ConsumerSessionMessage], sessionContext: ConsumerSessionContext, stats: ConsumerSessionTargetStats, - errors: Vector[String] + errors: Vector[String], + deliveryKey: Option[DeliveryKey] ) => Unit, isDebug: Boolean, - incrementNumMessageProcessed: () => Unit + incrementNumMessageProcessed: () => Unit, + onStartFromDiscardProgress: () => Unit, + admitDelivery: Option[DeliveryKey] => DeliveryAdmission ): Unit = val listener = consumerListener val targetMessageHandler = listener.targetMessageHandler + // Rewired on every play, like the message handler above it, so a skip still in flight + // reports to whichever client is listening NOW. The discard counter itself is NOT touched + // here - it is armed once, at session creation. + listener.onStartFromDiscardProgress = onStartFromDiscardProgress + targetMessageHandler.onNext = (msg: Message[Array[Byte]]) => - boundary: - stats.messageProcessed += 1 - incrementNumMessageProcessed() + val deliveryKey = ConsumerSessionTargetRunner.deliveryKeyOf(msg) + admitDelivery(deliveryKey) match + // A HANDLER FROM A SUPERSEDED PLAY: a listener thread still mid-flight across a + // second Play, or across a Stop or a stream ended by a failed resume. It must + // touch NOTHING - not the counters, not the JS context, not the user's stateful + // filters - because the message is coming back and the play that is actually + // current will process it properly. Throwing routes it to the caller's failure + // path, which hands it back for redelivery. + case DeliveryAdmission.Superseded => + throw new IllegalStateException("This play was superseded; the message is handed back for redelivery.") + + // THE RETRY OF A FAILED SEND: the session re-sent what the first attempt had + // already prepared, so this pipeline must not run a second time for it. + case DeliveryAdmission.Replayed => () + + case DeliveryAdmission.Prepare => deliverPrepared(msg, deliveryKey, onNext, isDebug, incrementNumMessageProcessed) + + permitLock.synchronized { + listener.startAcceptingNewMessages() + // Release the USER's hold AND the replay-boundary hold - a Resume extends the replay + // boundary (the runner re-captured it before this walk), so a past-end racer's pause + // and the auto-pause at caught-up both lift with it. A merge- or limiter-held + // consumer stays paused until its own owner lets go - resuming it here is exactly + // the stomp the arbiter ended. EVERY consumer is attempted; a client call that + // throws is aggregated and rethrown, because a session that claims to be running + // with a dead consumer must fail the resume loudly (the service turns this into the + // terminal gate). + val failures = pauseArbiters.toVector.flatMap { (topicFqn, arbiter) => + val userReleased = arbiter.release(PauseReason.User) + val boundaryReleased = arbiter.release(PauseReason.Boundary) + Option.when(!(userReleased && boundaryReleased))(topicFqn) + } + if failures.nonEmpty then + throw new RuntimeException(s"Some consumers could not be resumed: ${failures.mkString(", ")}") + } + + /** The target's half of ONE delivery attempt: count it, deserialize it, and run the target's + * JavaScript over it before handing it to the session callback. Everything in here is + * stateful and none of it is repeatable, which is why the admission above stands in front. */ + private def deliverPrepared( + msg: Message[Array[Byte]], + deliveryKey: Option[DeliveryKey], + onNext: ( + msg: Option[ConsumerSessionMessage], + sessionContext: ConsumerSessionContext, + stats: ConsumerSessionTargetStats, + errors: Vector[String], + deliveryKey: Option[DeliveryKey] + ) => Unit, + isDebug: Boolean, + incrementNumMessageProcessed: () => Unit + ): Unit = + // Both counters are atomic because these two lines run OUTSIDE the per-message context + // lease below, on one listener thread per partition. Kept here rather than moved inside + // the lease so that a message failing to deserialize still counts as processed. + stats.messageProcessed.incrementAndGet() + incrementNumMessageProcessed() - val sessionContext = sessionContextPool.getNextContext - val consumerSessionMessage = converters.serializeMessage(schemasByTopic, msg, targetConfig.messageValueDeserializer) - val messageJson = consumerSessionMessage.messageAsJsonOmittingValue - val messageValueToJsonResult = consumerSessionMessage.messageValueAsJson + // Deliberately OUTSIDE the lease below. Deserialization is a pure function of the + // message, the session's schemas and the configured deserializer - it touches no JS - + // and it is the expensive part of handling a message. Holding the session's single JS + // context across it would make every partition decode in single file for no reason. + val consumerSessionMessage = converters.serializeMessage(schemasByTopic, msg, targetConfig.messageValueDeserializer) + val messageJson = consumerSessionMessage.messageAsJsonOmittingValue + val messageValueToJsonResult = consumerSessionMessage.messageValueAsJson + // ONE lease for the WHOLE message, not one per JS call - and it deliberately spans the + // `onNext` callback, because the session-level filter chain, coloring rules, value + // projections and `getState` all run in there, off the SAME current message this thread + // just set. Pulsar delivers each partition on its own listener thread and they all share + // this context; a lease per call would still let another partition's message overwrite + // `globalThis.__dekaf_currentMessage` midway through this one. + sessionContextPool.withNextContext { sessionContext => + boundary: sessionContext.setCurrentMessage(messageJson, messageValueToJsonResult) val messageFilterChainResult: ChainTestResult = sessionContext.testMessageFilterChain( @@ -63,7 +147,8 @@ case class ConsumerSessionTargetRunner( msg = None, sessionContext = sessionContext, stats = stats, - errors = if isDebug then messageFilterChainErrors else Vector.empty + errors = if isDebug then messageFilterChainErrors else Vector.empty, + deliveryKey = deliveryKey ) boundary.break() @@ -103,6 +188,12 @@ case class ConsumerSessionTargetRunner( .withSessionTargetMessageFilterChainTestResult(ChainTestResult.toPb(messageFilterChainResult)) .withSessionTargetColorRuleChainTestResults(coloringRuleChainResult.map(ChainTestResult.toPb)) .withSessionTargetValueProjectionListResult(valueProjectionListResult.map(ValueProjectionResult.toPb)) + // ORDERED delivery (Guaranteed or, since 2026-08-11, Best effort): + // the layer decided this delivery undercuts an already-emitted order + // key - the row is on screen out of order, LOUDLY. Thread-local, so + // the stamp read here pairs with the delivery running on THIS call + // stack; false on every unordered path. + .withDeliveredOutOfOrder(consumerListener.outOfOrderFlagPending.get) ) ) @@ -110,28 +201,147 @@ case class ConsumerSessionTargetRunner( msg = msgToSend, sessionContext = sessionContext, stats = stats, - errors = errors + errors = errors, + deliveryKey = deliveryKey ) + } - listener.startAcceptingNewMessages() - consumers.foreach((_, consumer) => consumer.resume()) + /** Serializes every touch of the consumers' pause/resume state: the user's pause and resume, + * and the delivery pacer's permit holds, which arrive on other threads. Two writers taking + * turns unserialized could interleave a pacer resume into the middle of a user pause and leave + * the broker delivering into a session the user just stopped. + * + * LOCK ORDER: this lock nests under nothing of the pacer's - the pacer invokes its callbacks + * outside its own lock precisely so this one stays a leaf. + */ + private val permitLock = Object() - def pause(): Unit = - consumers.foreach((_, consumer) => consumer.pause()) + /** The delivery pacer's half of flow control: stop asking the broker for more while the paced + * backlog is over its watermark, without touching the gate. + * + * DELIBERATELY NOT [[pause]]. The user's pause closes the gate first, so everything already + * prefetched is rejected and handed back for redelivery - correct for "stop showing me + * things", and exactly wrong for a throttle, which wants the prefetched tail to drain through + * the queue instead of cycling as nacks. Holding permits leaves the gate open: what has + * arrived flows on, and only the ASKING stops. + * + * Refused outright while the gate is shut - the user's pause outranks the pacer, and a hold + * "released" onto a paused session must not resume its consumers. + */ + /** `appliesTo` selects WHICH of this target's streams the hold covers. A hold is per stream + * because a counted start-from can still be waiting for one particular topic's next message + * while its peers are the ones flooding: holding all of them (or, as it used to be, none of + * them) is what let one stuck stream disable the whole session's backpressure. A RELEASE + * always covers everything - the default - so nothing can be left behind by a hold that + * covered a different set. + */ + def setPermitHold(hold: Boolean, appliesTo: NonPartitionedTopicFqn => Boolean = _ => true): Boolean = permitLock.synchronized { + // EVERY selected consumer is attempted, and the hold is recorded even while the user has + // the session paused: with per-reason arbitration a release onto a paused session cannot + // resume anything (the User reason still holds), so the old gate-shut refusal is no + // longer needed to protect the pause - and recording keeps the pacer's books honest. + val results = pauseArbiters.toVector.collect { + case (topicFqn, arbiter) if appliesTo(topicFqn) => + if hold then arbiter.hold(PauseReason.Limiter) else arbiter.release(PauseReason.Limiter) + } + results.forall(identity) + } + + /** GATE FIRST, CONSUMERS SECOND, and the order is the point. + * + * `Consumer.pause` only stops the client asking the broker for more permits; whatever the + * client has already received is still handed to the listener afterwards. Pausing first and + * closing the gate second therefore left a window in which every buffered callback was + * delivered into a session the user had just paused - and spent its start-from budget doing so. + * Closing the gate first makes that window empty: everything already buffered is rejected and + * handed back for redelivery on resume, which is exactly what a paused session promises. + * + * `resume` is deliberately the mirror image: it opens the gate and only then resumes the + * consumers, so nothing is ever delivered while the gate is shut. + */ + def pause(): Unit = permitLock.synchronized { consumerListener.stopAcceptingNewMessages() + // EVERY consumer is attempted: one mid-close consumer throwing must not leave the rest + // delivering into a gate that is already shut (they would cycle as nacks) while the + // caller believes nothing was paused. Failures are aggregated and thrown AFTER the + // session has reached the safest state this call can produce. + val failures = pauseArbiters.toVector.flatMap { (topicFqn, arbiter) => + Option.when(!arbiter.hold(PauseReason.User))(topicFqn) + } + if failures.nonEmpty then + throw new RuntimeException(s"Some consumers could not be paused (their pause is still recorded): ${failures.mkString(", ")}") + } - def stop(): Unit = - consumers.foreach((_, consumer) => - Try { - consumer.unsubscribe() - } match - case Success(_) => () - case Failure(err) => println(s"Failed to stop consumer session target. ${err.getMessage}") - ) + /** Release this target's consumers, and answer with what could not be released. + * + * ANSWERS rather than throws: a session has other targets to release, and one broker that will + * not delete a subscription must not strand every consumer after it. The caller aggregates. + * + * `close` follows `unsubscribe` WHATEVER the unsubscribe did, and that is the important part: + * unsubscribing deletes the subscription on the broker, while closing releases the consumer, + * its connection and its listener thread here. Only the first was ever done, so every session + * that was stopped leaked its consumers - and a failed unsubscribe leaked them while reporting + * success to the client. + * + * BOTH failures are reported. The close was wrapped in a bare `Try` whose result was thrown + * away, so a consumer that refused to close - still connected, still holding its listener + * thread - was invisible: `deleteConsumer` answered OK and nothing said the consumer was still + * there. + * + * AND A CONSUMER THAT COULD NOT BE RELEASED IS KEPT, not forgotten. The map used to be cleared + * unconditionally, so a consumer that was mid-reconnect at that instant - the ordinary reason + * either call fails - kept its subscription and its listener thread for the life of the + * process with nothing anywhere holding a handle to it: this runner is discarded by its caller + * a moment later. It stays in `consumers` (so a repeated stop retries it) and goes into + * `quarantine`, which is what actually retries it once this runner is gone. + * + * @param quarantine + * where un-released consumers are retained. A parameter with the production default only so + * a test can drive the retry without touching the process-wide one. + */ + def stop(quarantine: CleanupQuarantine = consumerCleanupQuarantine): Vector[String] = + val outcomes = consumers.toVector.map { (topicFqn, consumer) => + val unsubscribed = Try(consumer.unsubscribe()).failed.toOption.map(err => s"$topicFqn: could not unsubscribe. ${err.getMessage}") + val closed = Try(consumer.close()).failed.toOption.map(err => s"$topicFqn: could not close the consumer. ${err.getMessage}") + val failures = unsubscribed.toVector ++ closed.toVector + if failures.nonEmpty then + quarantine.quarantine( + s"consumer session $sessionLabel on $topicFqn", + () => + // BOTH again on retry, and in the same order: the unsubscribe may have been + // the half that failed, and closing without it would leave the subscription. + consumer.unsubscribe() + consumer.close() + ) + (topicFqn, consumer, failures) + } + // Only what really was released is forgotten. + consumers = outcomes.collect { case (topicFqn, consumer, failures) if failures.nonEmpty => topicFqn -> consumer }.toMap + outcomes.flatMap(_._3) + + /** How this target names itself in cleanup diagnostics. The consumer name is + * `${sessionName}-${targetIndex}`, so the first consumer's name identifies the session without + * this runner having to carry the session name separately. */ + private def sessionLabel: String = + consumers.values.headOption.flatMap(consumer => Option(consumer.getConsumerName)).getOrElse(s"target $targetIndex") } object ConsumerSessionTargetRunner: val logger: Logger = Logger(getClass.getName) + + /** This message's identity for the session's single-attempt delivery memo, or None when it has + * none that can be trusted. Every message on a NON-PERSISTENT topic arrives as ledger 0, + * entry 0 - measured against a real broker - so a key there would name the next unrelated + * message of the topic instead of this one; nothing on such a topic is ever redelivered + * either, so there is no retry for a memo to serve. Same identity rule as the failed-ack and + * decided-batch registries in [[ConsumerListener]]. */ + private[session_runner] def deliveryKeyOf(msg: Message[Array[Byte]]): Option[DeliveryKey] = + Option(msg.getTopicName).filterNot(isNonPersistentTopic).map(topicFqn => (topicFqn, msg.getMessageId)) + + /** The most physical topics one target may resolve to. Generous - a session at this size is + * already hard to use interactively - but a bound: past it, setup time, per-topic consumers + * and the counted modes' per-message costs stop being an interactive workload at all. */ + val maxTopicsPerTarget: Int = 1_000 def make( sessionName: String, @@ -139,29 +349,53 @@ object ConsumerSessionTargetRunner: targetConfig: ConsumerSessionTarget, sessionContextPool: ConsumerSessionContextPool, schemasByTopic: SchemasByTopic, - adminClient: PulsarAdmin, - pulsarClient: PulsarClient + pulsarClient: PulsarClient, + // Resolved - and ADMITTED - by the session builder before any consumer exists anywhere: + // the session-total stream cap and the merge-mode prefetch budget are session-wide facts + // no single target can compute, and resolving here used to mean a session could + // subscribe hundreds of consumers only to be refused by a total it was always going to + // exceed. + nonPartitionedTopicFqns: Vector[NonPartitionedTopicFqn], + // Computed by the caller from the WHOLE session's stream count: the racing default for + // as-received sessions, the shared prefetch budget for best-effort-ordered ones. + receiverQueueSize: Int ): ConsumerSessionTargetRunner = Try { - val nonPartitionedTopicFqns = targetConfig.topicSelector.getNonPartitionedTopics(adminClient = adminClient) val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) - val consumers: Map[NonPartitionedTopicFqn, Consumer[Array[Byte]]] = nonPartitionedTopicFqns.map { topicFqn => - val consumerName = s"$sessionName-$targetIndex" - - buildConsumer( - pulsarClient = pulsarClient, - consumerName = consumerName, - topicsToConsume = Vector(topicFqn), - listener = listener, - targetConfig = targetConfig - ) match - case Right(consumerBuilder) => - val consumer = consumerBuilder.subscribe() - topicFqn -> consumer - case Left(err) => - throw new RuntimeException(s"Failed to build consumer for topic $topicFqn. $err") - }.toMap + // ALL OR NOTHING. Subscribing in a plain `map` meant a topic that failed part-way + // through left every consumer created before it subscribed and running, with nothing + // holding a handle to close them: the partly-built runner is never returned. + val consumers: Map[NonPartitionedTopicFqn, Consumer[Array[Byte]]] = buildAllOrRelease[NonPartitionedTopicFqn, (NonPartitionedTopicFqn, Consumer[Array[Byte]])]( + inputs = nonPartitionedTopicFqns, + build = topicFqn => + val consumerName = s"$sessionName-$targetIndex" + + buildConsumer( + pulsarClient = pulsarClient, + consumerName = consumerName, + topicsToConsume = Vector(topicFqn), + listener = listener, + targetConfig = targetConfig, + receiverQueueSize = receiverQueueSize + ) match + case Right(consumerBuilder) => + val consumer = consumerBuilder.subscribe() + topicFqn -> consumer + case Left(err) => + throw new RuntimeException(s"Failed to build consumer for topic $topicFqn. $err") + , + release = (_, consumer) => consumer.close(), + // A consumer that will not close during the unwind is exactly the one nothing else + // can ever reach: this partly-built map is never returned. + onReleaseFailure = (entry, _) => + consumerCleanupQuarantine.quarantine(s"consumer session $sessionName-$targetIndex on ${entry._1}", () => entry._2.close()) + ).toMap + + val pauseArbiters = consumers.map((topicFqn, consumer) => topicFqn -> ConsumerPauseArbiter(consumer)) + // The merge's flow-control hooks hold and release through the same arbiters, keyed by + // the stream id the ordering layer uses. + listener.pauseArbiters = pauseArbiters.map((topicFqn, arbiter) => startFromStreamId(s"$sessionName-$targetIndex", topicFqn) -> arbiter) ConsumerSessionTargetRunner( targetIndex = targetIndex, @@ -169,10 +403,11 @@ object ConsumerSessionTargetRunner: sessionContextPool = sessionContextPool, nonPartitionedTopicFqns = nonPartitionedTopicFqns, consumers = consumers, + pauseArbiters = pauseArbiters, consumerListener = listener, schemasByTopic = schemasByTopic, stats = ConsumerSessionTargetStats( - messageProcessed = 0 + messageProcessed = AtomicLong(0) ) ) } match { diff --git a/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetStats.scala b/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetStats.scala index 702cf5190..3ff213975 100644 --- a/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetStats.scala +++ b/server/src/main/scala/consumer/session_runner/ConsumerSessionTargetStats.scala @@ -1,5 +1,17 @@ package consumer.session_runner +import java.util.concurrent.atomic.AtomicLong + +/** Per-target counters. + * + * ATOMIC deliberately: Pulsar delivers each partition of a partitioned topic on its own listener + * thread, and this is incremented from the message handler BEFORE the per-message context lease, so + * a plain `var Long` loses read-modify-write updates and a partitioned session under-counts. + * + * The increment stays OUTSIDE the lease on purpose - moving it inside would also move it after + * `converters.serializeMessage`, changing whether a message that fails to deserialize still counts + * as processed. Making the counter atomic fixes the race without touching that ordering. + */ case class ConsumerSessionTargetStats( - var messageProcessed: Long + messageProcessed: AtomicLong ) diff --git a/server/src/main/scala/consumer/session_runner/StartFromDiscard.scala b/server/src/main/scala/consumer/session_runner/StartFromDiscard.scala new file mode 100644 index 000000000..2d6da2293 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/StartFromDiscard.scala @@ -0,0 +1,142 @@ +package consumer.session_runner + +import java.util.concurrent.atomic.AtomicLong + +/** What a start-from seek could NOT achieve on its own, expressed as messages to drop from the head + * of the delivered stream. + * + * A seek can only ever land on an ENTRY boundary: `PulsarAdmin.examineMessage` - the only primitive + * that addresses a position without reading the whole log - counts ENTRIES, a batching producer + * (the Java client default) puts many messages into one entry, and batch-index positions such as + * `1696:1:25` are rejected by the broker ("must be in format: ledgerId:entryId"). Landing on an + * exact MESSAGE therefore means seeking to the entry that contains it and dropping the messages + * that precede it inside that entry. There is no other exact mechanism. + */ +/** How many messages a discard swallows between progress reports. + * + * "Skip first n" is deliberately uncapped, so n can be in the millions; one gRPC frame per skipped + * message would be millions of frames for a progress bar. Coarse enough to be a trickle, fine + * enough that a skip big enough for the UI to bother showing (it surfaces above 1,000,000) still + * moves visibly. + */ +val startFromProgressReportInterval: Long = 10_000 + +/** Whether the discard should tell the client about the message it has just swallowed, `skipped` + * being the running count INCLUDING that one. + * + * The first is always reported, so the client learns the total as soon as the skip starts rather + * than one interval later. The last is always reported, so the run is seen to complete. In between + * only every `reportEvery`-th. + * + * `skipped >= total` rather than `==`: the counters are claimed from one listener thread per + * physical topic, and a thread that reads the running count a moment late must still report the + * end rather than sail past it. + * + * APPROXIMATE BY DESIGN. Under contention two threads can read the same count, or step over an + * interval boundary between them, so an intermediate tick may be reported twice or missed. Both are + * harmless for a progress indicator, and the end is not: every thread that reads a spent counter + * reports it. + */ +def shouldReportStartFromProgress(skipped: Long, total: Long, reportEvery: Long): Boolean = + if total <= 0 || skipped <= 0 then false + else skipped == 1 || skipped >= total || (reportEvery > 0 && skipped % reportEvery == 0) + +enum StartFromDiscardPlan: + /** The seek landed exactly - deliver everything from it. */ + case Nothing + + /** One counter for the WHOLE session: drop the first `n` messages of the merged delivered + * stream, whichever topic or partition each came from. */ + case SharedTotal(n: Long) + + /** An independent counter per physical topic: drop the first `counts(topic)` messages that + * topic delivers. */ + case PerTopic(counts: Map[NonPartitionedTopicFqn, Long]) + +/** The live counters behind a [[StartFromDiscardPlan]]. + * + * Armed ONCE, after the seek and before any consumer is resumed, and never re-armed: pausing and + * resuming a session must not skip a second batch of messages. + */ +final class StartFromDiscard private ( + private val shared: Option[AtomicLong], + private val perTopic: Map[NonPartitionedTopicFqn, AtomicLong] +): + /** Claims a delivered message for the discard. `true` means DROP it (and count it as dropped), + * `false` means deliver it. Called from the Pulsar client's listener threads - one per consumer + * - so the counters have to be atomic. */ + def claim(topicFqn: NonPartitionedTopicFqn): Boolean = + shared.orElse(perTopic.get(topicFqn)) match + case Some(counter) => counter.getAndUpdate(left => if left > 0 then left - 1 else 0) > 0 + case None => false + + /** Give a claimed drop back, because the message it was claimed for could not be acknowledged. + * + * A claim COSTS A MESSAGE - the message is acknowledged into nothing and nobody sees it - so a + * claim that no acknowledgment ever matched is a message Pulsar will redeliver against an + * already-spent budget, and the redelivery is then shown. Refunding keeps the invariant + * "units spent == messages actually dropped": the redelivery claims the unit again. + * + * Never above what was armed, because a refund only ever follows a claim. + */ + def refund(topicFqn: NonPartitionedTopicFqn): Unit = + shared.orElse(perTopic.get(topicFqn)).foreach(_.incrementAndGet()) + () + + /** Messages still to be dropped. Exposed so tests can assert "exactly n were skipped" rather + * than inferring it from what came out. */ + def remaining: Long = shared.map(_.get).getOrElse(perTopic.values.map(_.get).sum) + + /** What this discard still has to drop FROM ONE TOPIC - the per-stream question the delivery + * pacer asks before it holds a consumer's permits. + * + * A SHARED counter answers for every topic and deliberately so: it is one budget over the + * merged stream, so any topic can supply its next unit and none of them may be throttled + * quiet while it counts. A PER-TOPIC counter answers only for its own topic, which is what + * stops one topic's unfinished latest-n correction from suppressing backpressure everywhere. + */ + def remainingFor(topicFqn: NonPartitionedTopicFqn): Long = + shared.orElse(perTopic.get(topicFqn)).map(_.get).getOrElse(0L) + + /** Whether this counter is the USER'S skip, and so something to report progress for. + * + * Only the session-wide counter is: it exists because the user asked to skip the first n + * messages, which is O(n) and can take a long time with nothing to show for it. A PER-TOPIC + * counter is the opposite - it is the internal correction that makes "the latest n" exact, + * because a seek can only land on an entry boundary and each topic therefore over-fetches. + * Reporting that told a client asking for the last 5 messages that it was "skipping 95". + * + * The proto says the same thing: "Only NthMessageAfterEarliest needs this". + */ + def reportsProgress: Boolean = shared.isDefined + + /** How many messages this discard was ARMED with - what the user asked to skip. + * + * Captured once, at construction: read off the live counters it would shrink to zero as the + * skip progressed, and the client would be told the total was however much was left. + */ + val total: Long = shared.map(_.get).getOrElse(perTopic.values.map(_.get).sum) + +object StartFromDiscard: + /** Drops nothing, ever. A single immutable instance - it holds no counters to share. */ + val none: StartFromDiscard = new StartFromDiscard(None, Map.empty) + + def shared(n: Long): StartFromDiscard = new StartFromDiscard(Some(new AtomicLong(n max 0)), Map.empty) + + def perTopic(counts: Map[NonPartitionedTopicFqn, Long]): StartFromDiscard = + new StartFromDiscard(None, counts.map((topicFqn, n) => topicFqn -> new AtomicLong(n max 0))) + + /** Arms the counters for one consumer session target. `shared` must be the SAME instance for + * every target of a session, so that a [[StartFromDiscardPlan.SharedTotal]] really is one + * counter over the merged stream. */ + def forTarget( + plan: StartFromDiscardPlan, + shared: StartFromDiscard, + topicFqns: Vector[NonPartitionedTopicFqn] + ): StartFromDiscard = + plan match + case StartFromDiscardPlan.Nothing => none + case StartFromDiscardPlan.SharedTotal(_) => shared + // A fresh counter per target: two targets may select the SAME topic, and each has its + // own consumer on it that has to drop its own overshoot. + case StartFromDiscardPlan.PerTopic(counts) => perTopic(counts.view.filterKeys(topicFqns.toSet).toMap) diff --git a/server/src/main/scala/consumer/session_runner/buildAllOrRelease.scala b/server/src/main/scala/consumer/session_runner/buildAllOrRelease.scala new file mode 100644 index 000000000..630382be4 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/buildAllOrRelease.scala @@ -0,0 +1,50 @@ +package consumer.session_runner + +import scala.util.Try + +/** Build one resource per input, releasing everything ALREADY BUILT if any later one fails. + * + * A consumer session is built out of resources the broker holds on its behalf: one Pulsar consumer + * per physical topic, one target runner per enabled target. They were created in a plain `map`, so + * a failure part-way through - a topic that vanished, a broker that refused the subscription, + * anything at all - propagated out and left every consumer created before it subscribed, running, + * and unreachable: the partly-built runner was never returned, so nothing had a handle to close. + * The session appeared to fail while its consumers went on holding subscriptions until the process + * ended. + * + * The original failure is what propagates: a release that fails on the way out is a second failure + * while handling the first, and REPLACING the cause with a consequence would hide why the build + * failed at all. It is not DISCARDED, though, which is what used to happen - it rides along as a + * SUPPRESSED cause, so the one exception the caller sees says both what went wrong and what was + * left behind, and the resource itself is handed to `onReleaseFailure` so something still holds a + * handle to it. Swallowing both meant a consumer that refused to close kept its subscription, its + * connection and its listener thread with nothing anywhere able to reach it again. + * + * PURE: `build`, `release` and `onReleaseFailure` are plain functions, so the partial-failure path + * is driven with lambdas and no broker. + * + * @param onReleaseFailure + * what to do with a resource whose release failed. The default does nothing - callers that + * have nowhere to put it are no worse off than before; the session paths pass a quarantine. + */ +def buildAllOrRelease[A, R]( + inputs: Vector[A], + build: A => R, + release: R => Unit, + onReleaseFailure: (R, Throwable) => Unit = (_: R, _: Throwable) => () +): Vector[R] = + val built = Vector.newBuilder[R] + try + inputs.foreach(input => built += build(input)) + built.result() + catch + case err: Throwable => + built.result().foreach { resource => + Try(release(resource)).failed.foreach { releaseErr => + // Guarded: `addSuppressed` throws if handed the exception it is being added to, + // and nothing about a cleanup path may introduce a new way to fail. + Try(err.addSuppressed(releaseErr)) + Try(onReleaseFailure(resource, releaseErr)) + } + } + throw err diff --git a/server/src/main/scala/consumer/session_runner/buildConsumer.scala b/server/src/main/scala/consumer/session_runner/buildConsumer.scala index 578000efd..6413ed674 100644 --- a/server/src/main/scala/consumer/session_runner/buildConsumer.scala +++ b/server/src/main/scala/consumer/session_runner/buildConsumer.scala @@ -3,15 +3,54 @@ package consumer.session_runner import consumer.session_target.ConsumerSessionTarget import consumer.session_target.consumption_mode.modes.ReadCompactedConsumptionMode import org.apache.pulsar.client.api.* +import org.apache.pulsar.client.impl.MultiplierRedeliveryBackoff import scala.jdk.CollectionConverters.* +/** The client-side prefetch budget one TARGET may spend across all its topic consumers, in + * messages. A flat 2000-per-consumer receiver queue scaled the session's prefetch memory with + * the topic count - a 1000-topic regex selector configured ~2 million prefetched messages + * (payloads included) before anything was even shown. The budget divides across the target's + * consumers instead, floored so a huge selector still makes progress and capped at the old + * per-consumer value so small sessions keep their throughput. */ +val receiverQueueBudgetPerTarget: Int = 20_000 +val receiverQueueMin: Int = 50 +val receiverQueueMax: Int = 2_000 + +/** The per-consumer receiver queue for a target consuming `topicCount` topics: the budget divided + * evenly, clamped to [[receiverQueueMin]]..[[receiverQueueMax]]. */ +def receiverQueueSizeFor(topicCount: Int): Int = + val even = receiverQueueBudgetPerTarget / (topicCount max 1) + even.max(receiverQueueMin).min(receiverQueueMax) + +/** The session-wide prefetch budget while a delivery-order merge is active, and its clamps. + * + * The merge's own watermarks bound what IT holds, but `consumer.pause()` cannot un-prefetch: + * everything already in a receiver queue is still delivered after a pause, so an ordered + * session's real memory exposure is prefetch PLUS merge holds PLUS the delivery limiter's + * queue. This budget makes the prefetch term a TRUE CEILING: it divides across the session's + * ACTUAL stream total (resolved before anything subscribes), and the floor is ONE - at the + * admitted maximum of 2,000 streams every consumer still gets 2 slots, so no admitted topology + * can exceed the budget through the floor (the old floor of 10 turned "5,000" into 20,000 at + * that width). The whole-session bound is then: <= 5,000 prefetched messages, <= 10,000 / 256 + * MiB merge-held, <= 2,000 / 128 MiB limiter-queued, plus one in-flight send per stream. */ +val mergeReceiverQueueBudgetPerSession: Int = 5_000 +val mergeReceiverQueueMin: Int = 1 +val mergeReceiverQueueMax: Int = 500 + +/** The per-consumer receiver queue for a BEST-EFFORT-ORDERED session: the session budget divided + * across the session's total resolved streams. */ +def mergeReceiverQueueSizeFor(totalSessionStreams: Int): Int = + val even = mergeReceiverQueueBudgetPerSession / (totalSessionStreams max 1) + even.max(mergeReceiverQueueMin).min(mergeReceiverQueueMax) + def buildConsumer( pulsarClient: PulsarClient, consumerName: String, topicsToConsume: Vector[String], listener: MessageListener[Array[Byte]], - targetConfig: ConsumerSessionTarget + targetConfig: ConsumerSessionTarget, + receiverQueueSize: Int = receiverQueueMax ): Either[String, ConsumerBuilder[Array[Byte]]] = val isReadCompacted = targetConfig.consumptionMode.mode match case _: ReadCompactedConsumptionMode => true @@ -19,12 +58,21 @@ def buildConsumer( val consumer = pulsarClient.newConsumer .consumerName(consumerName) - .receiverQueueSize(2000) + .receiverQueueSize(receiverQueueSize) .autoUpdatePartitions(true) .maxPendingChunkedMessage(2) .autoAckOldestChunkedMessageOnQueueFull(true) .expireTimeOfIncompleteChunkedMessage(1, java.util.concurrent.TimeUnit.MINUTES) - .negativeAckRedeliveryDelay(0, java.util.concurrent.TimeUnit.SECONDS) + // DECAYING redelivery, not a flat delay. The old `negativeAckRedeliveryDelay(0, SECONDS)` + // is floored to 100ms by the client, which turned every sustained nack source into a + // ten-per-second redelivery hammer with no exit - and nacks are routine here: a paused + // session nacks everything it receives, and a delivery that throws mid-batch is handed + // back the same way. The first redelivery still comes at 100ms - pause/resume feels as + // immediate as before - while a message bounced over and over backs off toward a 10s + // ceiling, so a sustained source decays instead of hammering. (The start-from merge used + // to be the loudest source, nacking whatever it declined at its memory cap; it now PAUSES + // hot consumers instead and declines nothing.) + .negativeAckRedeliveryBackoff(MultiplierRedeliveryBackoff.builder.minDelayMs(100).maxDelayMs(10_000).build) .messageListener(listener) .startMessageIdInclusive() .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) diff --git a/server/src/main/scala/consumer/session_runner/cleanupQuarantine.scala b/server/src/main/scala/consumer/session_runner/cleanupQuarantine.scala new file mode 100644 index 000000000..aa80f8d21 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/cleanupQuarantine.scala @@ -0,0 +1,85 @@ +package consumer.session_runner + +import com.typesafe.scalalogging.Logger + +import scala.collection.mutable.ArrayDeque +import scala.util.{Failure, Success, Try} + +/** How many un-released resources one quarantine holds before the oldest is given up on. + * + * A bound rather than a guess about how many can fail: the entries are closures over consumers, + * and a server whose broker refuses every release must not accumulate them without limit. Dropping + * the OLDEST is deliberate - the newest failure is the one most likely still to be recoverable, + * and an entry that has survived this many later failures has had that many retry rounds already. + */ +val cleanupQuarantineCapacity: Int = 256 + +/** RESOURCES WHOSE RELEASE FAILED, KEPT ADDRESSABLE AND RETRIED. + * + * A failed `unsubscribe` or `close` is the ordinary outcome of stopping a session whose consumer + * is mid-reconnect, and both cleanup paths used to answer as though it had succeeded: the + * partial-build unwind swallowed the failure whole, and a target's stop cleared its consumer map + * whatever the calls did. The consumer then kept its subscription, its connection and its listener + * thread for the life of the process, with nothing anywhere holding a reference that could close + * it - the session runner having been discarded by its caller a moment later. + * + * The handle is therefore RETAINED here and the release retried. Retrying costs nothing when it + * works (closing an already-closed Pulsar consumer is a no-op) and is the only thing that can + * recover the case it exists for. + * + * PLAIN AND SYNCHRONOUS. It owns no thread: `retryPending` is called by whoever already sweeps - + * in production, the consumer service's idle janitor - which is also what makes every property + * here a test rather than a wait. + */ +final class CleanupQuarantine(capacity: Int = cleanupQuarantineCapacity): + private val logger: Logger = Logger(getClass.getName) + + private final case class Pending(label: String, release: () => Unit) + + /** Guarded by this object's monitor. A LEAF: the release closures run OUTSIDE it, because they + * call the broker. */ + private val pendingItems = ArrayDeque.empty[Pending] + + /** Retain one resource that could not be released. Idempotent per `label`, so a stop repeated + * over the same still-failing consumer does not accumulate copies of it. */ + def quarantine(label: String, release: () => Unit): Unit = synchronized { + if !pendingItems.exists(_.label == label) then + if pendingItems.size >= capacity then + val dropped = pendingItems.removeHead() + // Maintainer-visible: past this point the resource really is leaked, and the number + // of them says the broker has been refusing releases for a long time. + logger.warn( + s"The consumer cleanup quarantine is full at $capacity entries, so ${dropped.label} was given up on. " + + "Its broker-side resources may still exist." + ) + pendingItems.append(Pending(label, release)) + () + } + + /** What is still waiting to be released - a test's window, and what a saturation metric reads. */ + def pendingLabels: Vector[String] = synchronized(pendingItems.map(_.label).toVector) + + /** Retry every retained release; answers the labels that STILL could not be released, which are + * put back for the next round. The snapshot is taken under the lock and the releases run + * outside it: each is a broker call. */ + def retryPending(): Vector[String] = + val snapshot = synchronized { + val items = pendingItems.toVector + pendingItems.clear() + items + } + snapshot.flatMap { item => + Try(item.release()) match + case Success(_) => + logger.info(s"A consumer resource that could not be released earlier was released on retry: ${item.label}") + None + case Failure(err) => + logger.debug(s"Retrying the release of ${item.label} failed again; it stays quarantined. ${err.getMessage}") + quarantine(item.label, item.release) + Some(item.label) + } + +/** THE server-wide cleanup quarantine. One per process, like the session maintenance scheduler: + * the sites that fail to release are deep inside session teardown and have nowhere else to hand a + * resource to. Every one of them takes it as a defaulted parameter so a test can pass its own. */ +val consumerCleanupQuarantine: CleanupQuarantine = CleanupQuarantine() diff --git a/server/src/main/scala/consumer/session_runner/consumerPauseArbiter.scala b/server/src/main/scala/consumer/session_runner/consumerPauseArbiter.scala new file mode 100644 index 000000000..15ceed0ef --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/consumerPauseArbiter.scala @@ -0,0 +1,54 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.Consumer + +import scala.util.Try + +/** Why a consumer is currently held still. A consumer runs exactly when NOBODY holds it. */ +enum PauseReason: + /** The session's own pause - the user pressed the button (or the gate is closed). */ + case User + + /** Start-from flow control holding a hot stream at its memory watermark. */ + case Merge + + /** The delivery rate limiter's permit hold while its paced backlog is over its watermark. */ + case Limiter + + /** The GUARANTEED replay boundary: this consumer delivered past the recorded end mid-replay + * (its output belongs to the next chunk), or the whole session auto-paused at caught-up. + * Released by Resume - which extends the boundary - and by the live switch to Best effort. */ + case Boundary + +/** The ONE place a consumer's paused/running state is decided. + * + * Three mechanisms legitimately pause consumers - the user, the start-from merge's flow control, + * and the delivery rate limiter - and they used to call `consumer.pause()` / `resume()` directly, + * at each other's expense: the merge's settle-time resume woke consumers the LIMITER had just + * held (its permit flag then said "held" while nothing was, so it never re-paused and the paced + * queue grew without bound), and a user resume woke merge-held hot streams. Each mechanism now + * only adds or removes ITS OWN reason; the consumer resumes only once the reason set is empty, + * so no owner can stomp another's hold. + * + * `hold` RE-ASSERTS the client-local pause flag every time (idempotent and effectively free), + * so even a rogue direct `resume()` somewhere is healed by the next hold of any reason. + */ +final class ConsumerPauseArbiter(consumer: Consumer[Array[Byte]]): + private val reasons = scala.collection.mutable.Set.empty[PauseReason] + + /** Add `reason` and ensure the consumer is paused. False when the client call threw - the + * reason is recorded regardless, so a later release still balances the books. */ + def hold(reason: PauseReason): Boolean = synchronized { + reasons += reason + Try(consumer.pause()).isSuccess + } + + /** Drop `reason`; the consumer resumes only once NO reason remains. False when the resume + * call threw (the reason is dropped regardless). */ + def release(reason: PauseReason): Boolean = synchronized { + reasons -= reason + if reasons.isEmpty then Try(consumer.resume()).isSuccess else true + } + + /** The reasons currently holding this consumer - the observable truth, exposed for tests. */ + def heldReasons: Set[PauseReason] = synchronized(reasons.toSet) diff --git a/server/src/main/scala/consumer/session_runner/deliveryOrderSwitch.scala b/server/src/main/scala/consumer/session_runner/deliveryOrderSwitch.scala new file mode 100644 index 000000000..17c58cf20 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/deliveryOrderSwitch.scala @@ -0,0 +1,76 @@ +package consumer.session_runner + +import consumer.session_config.MessageDeliveryOrder + +/** What a request to change a LIVE session's delivery order resolves to. + * + * Split out as a pure decision so the whole matrix - which direction is honoured, which is + * refused and with what reason - is pinned by test without a session, a broker or an RPC, and so + * the service and the runner cannot disagree about it. + */ +enum DeliveryOrderSwitch: + /** The session already delivers in the requested order. A no-op, answered OK: a repeated + * click, or two clients asking for the same thing, must not become an error. */ + case AlreadyThere + + /** Guaranteed -> Best effort on a live session: the one transition that is honestly + * deliverable mid-stream. */ + case RelaxToBestEffort + + /** Refused, with the reason the client is shown. */ + case Refused(reason: String) + +/** Decide what a live delivery-order change means. + * + * ONLY ONE TRANSITION IS HONOURED: Guaranteed -> Best effort. That is the whole point of the + * operation - Guaranteed (the product default since 2026-08-11) can wait forever on a stream + * whose recorded range cannot be delivered (trimmed by retention, or seeked past the end), so + * the disclosed stall needs an action that does not throw the session away. + * + * WHY THE OTHER DIRECTIONS ARE REFUSED RATHER THAN FAKED: + * + * - ANYTHING -> GUARANTEED is a promotion, and Guaranteed is strictly stronger: it promises + * that Dekaf introduced no cross-stream disorder into what the user has seen. A session that + * has been running under Best effort has already emitted heads past streams that were silent, + * and no future behaviour can un-emit them - the promise would be false for this session's + * output the moment it was made. Honouring the request would therefore be a lie about + * messages already on screen, so it is refused and the client is told to restart the session + * to get the guarantee from the beginning. + * + * - ANYTHING -> AS_RECEIVED (Fastest) is not a weaker ordering of the merged stream, it is the + * ABSENCE of the merge. The same merge object also resolves a counted start-from cut (skip + * first n is decided over the merged stream, not over any one log), so dismantling it while a + * cut is unresolved would silently change WHICH messages the cut selects - the exact failure + * mode that made recreating the session unacceptable in the first place. It also owns the + * per-stream flow control and the late-delivery accounting. Fastest therefore stays a + * configuration choice that takes effect on the next Play, and Best effort - which already + * has an unconditional liveness escape - is the live remedy. + * + * `current` is the order the session is delivering in RIGHT NOW, which after an earlier switch is + * no longer the one its saved configuration names. + */ +def deliveryOrderSwitchFor(current: MessageDeliveryOrder, requested: MessageDeliveryOrder): DeliveryOrderSwitch = + if current == requested then DeliveryOrderSwitch.AlreadyThere + else + (current, requested) match + case (MessageDeliveryOrder.Guaranteed, MessageDeliveryOrder.BestEffort) => + DeliveryOrderSwitch.RelaxToBestEffort + case (_, MessageDeliveryOrder.Guaranteed) => + DeliveryOrderSwitch.Refused( + "This session cannot be switched to Guaranteed ordering while it is running: it has already delivered " + + "messages without waiting for every stream, and that cannot be undone. Start the session again with " + + "Guaranteed selected." + ) + case (_, MessageDeliveryOrder.AsReceived) => + DeliveryOrderSwitch.Refused( + "Fastest is not a weaker ordering, it is no ordering at all - the same layer also resolves a counted " + + "start-from position, so removing it mid-session could change which messages this session shows. " + + "Select Fastest in the session configuration; it takes effect the next time you press Play." + ) + // Unreachable today - AsReceived and Guaranteed are matched above and equality is + // matched first, so nothing else can arrive here - but stated rather than left to a + // MatchError if a fourth mode is ever added. + case _ => + DeliveryOrderSwitch.Refused( + s"Changing the delivery order from $current to $requested is not supported on a running session." + ) diff --git a/server/src/main/scala/consumer/session_runner/deliveryRateLimiter.scala b/server/src/main/scala/consumer/session_runner/deliveryRateLimiter.scala new file mode 100644 index 000000000..6f01025d5 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/deliveryRateLimiter.scala @@ -0,0 +1,584 @@ +package consumer.session_runner + +import com.typesafe.scalalogging.Logger + +import java.util.concurrent.atomic.AtomicBoolean +import scala.collection.mutable.ArrayDeque +import scala.util.Try + +/** The delivery rate limiter: a token bucket and a FIFO queue between "the session decided to show + * this message" and "the browser was sent it". + * + * WHERE IT SITS, AND WHY EXACTLY THERE. The limiter is fed only with DELIVER outcomes - after the + * start-from discard and the global merge have decided a message is going to be shown. Drops fly + * past it at full speed, so a counted skip or a latest-n walk positions as fast as the broker can + * serve it; the limit shapes what the user actually watches, which is the only thing a + * messages-per-second number can honestly mean to them. Feeding it any earlier would slow the + * positioning the user is waiting on; any later (at the gRPC send) would leave the expensive part + * - deserialization and the GraalVM filter chain - running at firehose rate with the limit + * protecting nothing but the tab. + * + * WHY A QUEUE AND NOT THE BROKER'S PERMITS. `Consumer.pause()` only stops asking the broker for + * more; everything already prefetched (up to receiverQueueSize per partition) is still handed to + * the listener afterwards - this codebase measured that, see ConsumerSessionTargetRunner.pause. + * Pacing with permits alone therefore bursts by thousands. Here the burst lands in the queue + * instead of the browser, and the drain releases EXACTLY the configured rate; the permits are + * used for what they are good at - bounding how much piles up - via the high/low watermarks. + * + * WHY NOTHING EVER BLOCKS OR NACKS HERE. Offering is an enqueue under a lock held for + * nanoseconds, so a Pulsar listener thread is never parked and the global merge - whose offers + * run under the session's ordering lock - is never starved into its silent-stream give-up. + * The unlimited (rate 0) inline shortcut is the one exception, and it is REFUSED while a live + * merge holds its ordering lock around the offer: the caller forces the queue then (see + * [[ConsumerListener.deliver]]), because processing inline there ran the JS lease and the + * blocking gRPC write inside that lock. Rejecting instead of queueing would nack, and a + * sustained rate limit built on nacks is the ~100ms redelivery storm the round-3 review put a + * bound on, recreated deliberately. + * + * ORDER IS THE QUEUE'S ORDER, ALWAYS. Merge-ordered sessions enqueue under the ordering lock, so + * the queue holds the merge's decision order; the single drainer releases FIFO; and while a drain + * is in flight or a backlog exists, nothing is processed inline - `drainInProgress` closes the + * window in which a later message could overtake a queued one into the stateful session filters. + * + * The core is deliberately free of threads and clocks: the wrapper below owns scheduling and side + * effects, this class owns every decision, and the tests drive it with a hand-cranked clock. + */ +final class DeliveryRateLimiterCore[A]( + nowMs: () => Long, + // What one queued payload costs, for the BYTE watermark. The count watermark alone let a + // rate-limited session on fat messages hold gigabytes: 2000 queued 5 MB payloads is 10 GB + // the count marks called fine. The default counts nothing (tests that don't care); + // production wires the real payload size in. + payloadBytesOf: A => Long = (_: A) => 0L +): + /** Messages per second; 0 means unlimited (offers process inline whenever no backlog exists). */ + private var ratePerSecond: Long = 0 + + /** The bucket. Capacity is one second's worth (= ratePerSecond), and it STARTS FULL on every + * rate change: the first screenful after Play appears at once, and the cap shapes what follows. + * A session limited to 100/s that begins with "latest 50" paints all 50 immediately - the user + * asked for exactly those - and then trickles. + */ + private var tokens: Double = 0 + private var lastRefillAtMs: Long = nowMs() + + private val queue = ArrayDeque.empty[A] + + // Payload bytes currently queued - maintained at every append/remove, like the merge's own + // byte bookkeeping, so reading it never scans the queue. + private var queuedBytes: Long = 0L + + /** True from the moment a drain is scheduled until [[beginDrain]] hands its batch out, so a + * second timer is never armed for the same backlog. */ + private var drainScheduled = false + + /** True between [[beginDrain]] and [[finishDrain]], i.e. while the wrapper is processing a + * batch OUTSIDE this lock. Inline processing is refused while it is set - see the class note + * on ordering. */ + private var drainInProgress = false + + /** THE USER'S PAUSE AND THE DELIVERIES IT MUST OUTRANK, under ONE lock - which is the whole + * point of them living here rather than in the wrapper. + * + * The wrapper used to hold the flag in an AtomicBoolean and read it as the drain loop's + * condition, so a Pause landing between that read and the delivery call was answered to the + * client while the already-decided item went on to be SENT and ACKNOWLEDGED. "Paused" has to + * mean two things at once - no delivery may start, and none is still running - and two + * separate pieces of state can never mean that together. + * + * `deliveriesInFlight` counts deliveries that have been ADMITTED here and are executing + * outside this lock, on the drain thread or (for the unlimited inline shortcut) on the + * offering thread. + */ + private var deliveriesPaused = false + private var deliveriesInFlight = 0 + + def isDeliveryPaused: Boolean = synchronized(deliveriesPaused) + + /** Admit the next delivery of a drain batch, or refuse it because the session is paused. + * Refusing leaves the item for the caller to requeue - see [[DeliveryRateLimiter.tick]]. */ + def beginDelivery(): Boolean = synchronized { + if deliveriesPaused then false + else + deliveriesInFlight += 1 + true + } + + /** One admitted delivery has finished - whether it succeeded, threw, or was skipped. */ + def endDelivery(): Unit = synchronized { + deliveriesInFlight -= 1 + if deliveriesInFlight <= 0 then notifyAll() + } + + /** Stop deliveries and WAIT FOR THE BOUNDARY: no delivery may start after this returns, and + * every delivery already admitted has finished. Answers whether it reached that boundary. + * + * `reentrant` is the delivery budget's case - `sendPrepared` pauses the drain from INSIDE the + * send that spent the last unit, on the delivery thread itself, and waiting there would be a + * thread waiting for its own completion. The flag is still set (the rest of the batch is + * refused, which is exactly the budget's contract); only the wait is skipped. + * + * BOUNDED, and deliberately by the WALL clock rather than by this class's injected `nowMs`: + * `nowMs` is the rate bucket's clock and the tests hand-crank it, while this is a safety bound + * on a delivery that has stopped making progress at all - a user filter in an infinite loop, + * a transport that will not take the write. Expiring means the pause is not linearized against + * that one delivery; the alternative is holding a gRPC thread for the life of the process. + */ + def pauseDeliveries(reentrant: Boolean, timeoutMs: Long): Boolean = synchronized { + deliveriesPaused = true + if reentrant then true + else + val deadlineNanos = java.lang.System.nanoTime() + timeoutMs * 1_000_000L + var remainingMs = timeoutMs + while deliveriesInFlight > 0 && remainingMs > 0 do + wait(remainingMs) + remainingMs = (deadlineNanos - java.lang.System.nanoTime()) / 1_000_000L + deliveriesInFlight <= 0 + } + + def resumeDeliveries(): Unit = synchronized { deliveriesPaused = false } + + def setRate(newRatePerSecond: Long): Unit = synchronized { + ratePerSecond = newRatePerSecond + tokens = newRatePerSecond.toDouble + lastRefillAtMs = nowMs() + } + + /** True while a DELIVERY BUDGET is armed. The budget is counted downstream (at the send, where + * "loaded" is decided), but it needs every message to pass through the QUEUE: the inline + * unlimited shortcut hands the message to processing on the calling thread, past any chance + * of stopping the batch behind the one that spent the last unit. + */ + private var forceQueue: Boolean = false + + def setForceQueue(force: Boolean): Unit = synchronized { forceQueue = force } + + def rate: Long = synchronized(ratePerSecond) + def queuedCount: Int = synchronized(queue.size) + def queuedBytesCount: Long = synchronized(queuedBytes) + + /** Put back what a drain took but did not process - the tail behind a delivery budget that ran + * out mid-batch. FRONT of the queue, order intact, and the tokens they were charged handed + * back: they were never delivered, and the next resume must find them exactly where they were. + */ + def requeueFront(items: Vector[A]): Unit = synchronized { + queue.prependAll(items) + queuedBytes += items.map(payloadBytesOf).sum + if ratePerSecond > 0 then tokens = math.min(ratePerSecond.toDouble, tokens + items.size) + } + + /** What [[offer]] told the caller to do. `processNow` and `scheduleDrainAfterMs` are mutually + * exclusive; `queuedCount` is the size AFTER this offer, for the caller's watermark check. */ + final case class OfferOutcome(processNow: Boolean, queuedCount: Int, scheduleDrainAfterMs: Option[Long]) + + def offer(a: A, forceQueueOnce: Boolean = false): OfferOutcome = synchronized { + // Unlimited and nothing queued and nobody mid-drain: the message may go straight through on + // the calling thread - byte-for-byte the unlimited path. Any backlog forces the queue, so + // a flush after a rate change cannot be overtaken - and so does an armed delivery budget, + // which has to be able to stop the line BETWEEN messages, and a caller-supplied reason + // (the wrapper passes "the user has the session paused"). + // `!deliveriesPaused` is part of the SAME critical section as the admission below, not a + // flag the caller read a moment ago: an inline delivery admitted just after a Pause + // returned is a message sent into a session the user had stopped. + if ratePerSecond == 0 && !forceQueue && !forceQueueOnce && !deliveriesPaused && queue.isEmpty && !drainInProgress then + deliveriesInFlight += 1 + OfferOutcome(processNow = true, 0, None) + else + queue.append(a) + queuedBytes += payloadBytesOf(a) + val schedule = if !drainScheduled && !drainInProgress then + drainScheduled = true + Some(nextDrainDelayMs) + else None + OfferOutcome(processNow = false, queue.size, schedule) + } + + /** Take the batch this tick has earned. Refills the bucket from the elapsed clock, spends it, + * and caps the batch so one tick never monopolizes the drainer thread. */ + def beginDrain(): Vector[A] = synchronized { + drainScheduled = false + drainInProgress = true + refill() + val earned = + if ratePerSecond == 0 then queue.size + else math.min(tokens.toLong, queue.size.toLong).toInt + val take = math.min(earned, deliveryRateLimitMaxDrainBatch) + if ratePerSecond > 0 then tokens -= take + val batch = Vector.fill(take)(queue.removeHead()) + queuedBytes = (queuedBytes - batch.map(payloadBytesOf).sum) max 0L + batch + } + + /** The batch is processed; decide what happens next. `rescheduleAfterMs` is set while a backlog + * remains, sized by how long the next token takes to arrive. */ + final case class FinishOutcome(queuedCount: Int, rescheduleAfterMs: Option[Long]) + + def finishDrain(): FinishOutcome = synchronized { + drainInProgress = false + val reschedule = if queue.nonEmpty then + drainScheduled = true + Some(nextDrainDelayMs) + else None + FinishOutcome(queue.size, reschedule) + } + + /** A tick fired while draining was paused: the timer is consumed, so the flag it represents + * must be handed back or no future offer would ever arm another. */ + def cancelScheduledDrain(): Unit = synchronized { drainScheduled = false } + + /** Take one delivery token if the bucket has one - the pacing primitive the GUARANTEED + * pump uses instead of the queue (queueing after the ordering decision would break its + * deliver-then-advance barrier). Unlimited (rate 0) always answers true. */ + def tryAcquireDeliveryToken(): Boolean = synchronized { + refill() + if ratePerSecond == 0 then true + else if tokens >= 1.0 then + tokens -= 1 + true + else false + } + + /** Hand back ONE delivery token - the guaranteed pump's failed-send refund. The send never + * took, so the message never left the merge and its retry must not be charged a second + * token for the same delivery; mirrors [[requeueFront]]'s refund on the queued path. + * Clamped at the bucket size, and a no-op when unlimited (nothing was acquired). */ + def returnDeliveryToken(): Unit = synchronized { + if ratePerSecond > 0 then tokens = math.min(ratePerSecond.toDouble, tokens + 1.0) + } + + /** Re-arm after a pause ended. Answers the delay to schedule, or None when there is nothing + * queued or a timer is already armed. */ + def rearmDrain(): Option[Long] = synchronized { + if queue.nonEmpty && !drainScheduled && !drainInProgress then + drainScheduled = true + Some(nextDrainDelayMs) + else None + } + + /** Empty the queue - the session is stopping and these deliveries are moot. */ + def clear(): Unit = synchronized { + queue.clear() + queuedBytes = 0L + drainScheduled = false + } + + private def refill(): Unit = + val now = nowMs() + if ratePerSecond > 0 then + val earned = (now - lastRefillAtMs).max(0).toDouble / 1000.0 * ratePerSecond + tokens = math.min(ratePerSecond.toDouble, tokens + earned) + lastRefillAtMs = now + + /** How long until the next token is worth waking up for. Zero when a whole token is already + * banked; otherwise the exact wait, floored so a high rate coalesces into batches instead of + * waking per message. The floor costs no throughput - tokens accrue while asleep and the next + * batch is larger by exactly the wait. + */ + private def nextDrainDelayMs: Long = + refill() + if ratePerSecond == 0 || tokens >= 1.0 then 0L + else + val exact = math.ceil((1.0 - tokens) * 1000.0 / ratePerSecond.toDouble).toLong + math.max(exact, deliveryRateLimitMinRescheduleDelayMs) + +/** One drain never processes more than this, so a fat bucket (high rate or a long sleep) cannot + * hold the drainer - and with it the session's single JS context - for an unbounded stretch. The + * remainder reschedules at delay zero. */ +val deliveryRateLimitMaxDrainBatch = 500 + +/** The floor under drain wake-ups. At 1000/s the exact next-token wait is 1ms; waking per token + * would burn a thread on timers, so waits are coalesced and the batch grows to match. */ +val deliveryRateLimitMinRescheduleDelayMs = 25L + +/** Backlog size above which the session stops ASKING the broker for more - `Consumer.pause()`, the + * permits half of flow control. One receiverQueueSize (2000), because that is the burst a pause + * cannot prevent anyway: whatever was prefetched still arrives after it. */ +val deliveryRateLimitHoldPermitsAboveQueued = 2000 + +/** Backlog size below which the permits are released again. The gap to the hold mark is the + * hysteresis that keeps a hovering queue from flapping pause/resume at the broker. */ +val deliveryRateLimitReleasePermitsBelowQueued = 500 + +/** Queued payload BYTES above which permits are held whatever the count says - the fat-message + * half of the same backpressure. 2000 queued 5 MB payloads is 10 GB the count watermark calls + * fine; this one does not. */ +val deliveryRateLimitHoldPermitsAboveQueuedBytes: Long = 128L * 1024 * 1024 + +/** Queued-bytes mark below which the byte hold lets go - hysteresis, like the count pair. */ +val deliveryRateLimitReleasePermitsBelowQueuedBytes: Long = 32L * 1024 * 1024 + +/** THE ABSOLUTE CEILING on the paced backlog - twice the hold watermark, in both dimensions, and + * the point at which memory safety outranks every reason not to apply backpressure. + * + * The ordinary hold at [[deliveryRateLimitHoldPermitsAboveQueued]] can legitimately be REFUSED: + * a counted start-from still waiting for a stream's next message must not have that stream's + * permits held, because a throttled-quiet stream is indistinguishable from the silent stream the + * merge gives up on after 30 seconds. That refusal is a correctness protection, and it is bounded + * here rather than open-ended: once the backlog is twice the watermark the hold is applied to + * every stream regardless, because a degraded start-from cut is disclosed to the user + * (`degraded` / `abandonedStreams`) while an unbounded queue is a dead process. + * + * Nothing is dropped, nacked or blocked when this fires - a permit hold only stops ASKING the + * broker for more, so the messages stay on the broker, unacknowledged, and arrive once the + * backlog drains. The gap to the watermark is the room the resolution gets to finish first. + */ +val deliveryRateLimitForceHoldPermitsAboveQueued = 2 * deliveryRateLimitHoldPermitsAboveQueued + +/** The byte twin of [[deliveryRateLimitForceHoldPermitsAboveQueued]] - fat payloads reach the + * ceiling on bytes long before they reach it on count. */ +val deliveryRateLimitForceHoldPermitsAboveQueuedBytes: Long = 2 * deliveryRateLimitHoldPermitsAboveQueuedBytes + +/** How long after a FAILED permit release its retry fires. Without it, a release that failed once + * on a drained queue was never retried - no messages means no crossings means no ticks - and the + * consumers stayed paused until the user manually cycled the session. */ +val deliveryRateLimitReleaseRetryDelayMs: Long = 200L + +/** How long Pause waits for an already-admitted delivery to finish before giving up on + * linearizing against it - see [[DeliveryRateLimiterCore.pauseDeliveries]]. Generous against any + * ordinary send (a JS filter chain and one gRPC write), short against a delivery that has stopped + * making progress at all, which is the only thing this bound exists for. */ +val deliveryPauseQuiesceTimeoutMs: Long = 5_000L + +/** The impure shell around [[DeliveryRateLimiterCore]]: it owns the timer, runs the processing, and + * turns the queue's watermarks into consumer permit holds. + * + * THE CALLBACKS RUN OUTSIDE THE CORE'S LOCK, every one of them. `process` leases the session's JS + * context and takes the send lock; `holdPermits`/`releasePermits` take the targets' permit locks. + * Holding the limiter's own lock across any of those would nest it under locks it must never meet. + * + * PERMIT ARBITRATION. `holdPermits` answers whether it actually paused anything - the runner + * refuses while start-from counting is still in flight (a throttled-quiet stream must not look + * like a SILENT one to the merge's give-up) - and the held flag is only set when it really did. + * A refused hold retries on the next offer, so backpressure engages the moment the refusal's + * reason has passed. The hold itself lands on each consumer's pause ARBITER (reason Limiter), + * so neither a user resume nor the merge releasing a hot stream can wake a consumer this + * limiter still holds - the flag stays truthful without watching anybody else. + */ +final class DeliveryRateLimiter[A]( + val core: DeliveryRateLimiterCore[A], + schedule: (Long, Runnable) => Unit, + process: A => Unit, + holdPermits: () => Boolean, + releasePermits: () => Boolean, + // The ceiling's escalation: hold EVERY stream, whatever the ordinary hook is currently + // refusing to suppress. Refusing by default is inert for a limiter whose ordinary hold is + // never refused - the ceiling is only ever reached once a hold has been declined. + forceHoldPermits: () => Boolean = () => false +): + private val logger: Logger = Logger(getClass.getName) + + /** Whether the ORDINARY watermark hold is applied. It can be a PARTIAL hold: the runner leaves + * running any stream whose next message a counted start-from is still waiting for. */ + private val permitsHeld = AtomicBoolean(false) + + /** Whether the ABSOLUTE ceiling has forced a hold on every stream, suppression included. + * + * ITS OWN LATCH rather than [[permitsHeld]], because the two are not the same event: the + * ordinary hold can be latched while some streams were deliberately left running, and those + * are exactly the streams that can carry the backlog on to the ceiling afterwards. + */ + private val permitsForceHeld = AtomicBoolean(false) + + /** Whether THIS THREAD is inside a delivery this limiter admitted. + * + * The delivery budget pauses the drain from inside the send that spends its last unit + * ([[ConsumerSessionRunner]]'s `sendPrepared`), so the pause boundary below would otherwise + * be a thread waiting for itself to finish. Thread-local rather than "the" delivery thread + * because the unlimited inline path delivers on whichever listener thread offered. + */ + private val inDelivery: ThreadLocal[java.lang.Boolean] = ThreadLocal.withInitial(() => java.lang.Boolean.FALSE) + + def offer(a: A, forceQueueOnce: Boolean = false): Unit = + // NEVER inline while the user has the session paused: the unlimited fast path used to + // process (and acknowledge) a message that arrived between the pause RPC returning and + // its consumers actually stopping - delivery after pause, on the offering thread. The + // core refuses it under its own lock and the queue takes it instead; resume drains it. + // The CALLER's forceQueueOnce is the other reason never to run inline: an offer made + // under the session's ordering lock (a live merge) must stay an enqueue, or the JS lease + // and the gRPC write run inside that lock and a backpressured client parks every listener + // thread - see [[ConsumerListener.deliver]]. + val outcome = core.offer(a, forceQueueOnce = forceQueueOnce) + if outcome.processNow then runAdmittedDelivery(a) + else + val queuedBytes = core.queuedBytesCount + if (outcome.queuedCount >= deliveryRateLimitHoldPermitsAboveQueued + || queuedBytes >= deliveryRateLimitHoldPermitsAboveQueuedBytes) + && !permitsHeld.get + then + // CAS first so two crossing offers cannot double-pause; un-set on refusal so the + // next offer retries once the refusal's reason (start-from still counting) is + // gone. A refused AGGREGATE may still have paused SOME consumers (one target of + // several threw), so the partial holds are rolled back - without this they sat + // in their arbiters with the limiter believing it owned nothing, paused forever. + if permitsHeld.compareAndSet(false, true) then + if !holdPermits() then + permitsHeld.set(false) + releasePermits() + () + // THE FINAL SAFETY NET, and deliberately independent of the latch above: the ordinary + // hold is allowed to be refused, or to be applied to only SOME streams, while a + // counted start-from still needs the rest running. Past the ceiling that allowance + // ends - see [[deliveryRateLimitForceHoldPermitsAboveQueued]]. + if (outcome.queuedCount >= deliveryRateLimitForceHoldPermitsAboveQueued + || queuedBytes >= deliveryRateLimitForceHoldPermitsAboveQueuedBytes) + && !permitsForceHeld.get + then + if permitsForceHeld.compareAndSet(false, true) then + if forceHoldPermits() then + // Per SESSION and once per crossing, not per message, and it means a + // maintainer-visible degradation really did happen: the session was + // pushing more than its limit could pace while its start position was + // still being resolved. + logger.warn( + s"A rate-limited consumer session's paced backlog reached ${outcome.queuedCount} messages / " + + s"$queuedBytes bytes, so broker flow control was applied to every stream even though a counted " + + "start-from was still resolving. The resolved start position may be reported as degraded." + ) + else permitsForceHeld.set(false) + outcome.scheduleDrainAfterMs.foreach(scheduleTick) + + /** STOP DELIVERING, AND DO NOT ANSWER UNTIL NOTHING IS STILL ON ITS WAY OUT. + * + * The wait is what makes a Pause the client can act on: without it the drain loop's flag read + * and its delivery call were two steps, and a Pause landing between them returned OK while + * that message was sent to the browser and acknowledged to the broker. + */ + def pauseDraining(): Unit = + val quiesced = core.pauseDeliveries(reentrant = inDelivery.get, timeoutMs = deliveryPauseQuiesceTimeoutMs) + if !quiesced then + // Maintainer-visible, once per pause and never per message: a delivery that outlasts + // this bound is a stuck user filter or a transport that will not take the write, and + // it means this pause is NOT linearized against that one message. + logger.warn( + s"A consumer session's delivery did not finish within ${deliveryPauseQuiesceTimeoutMs}ms of Pause, so the pause " + + "could not be linearized against it. That delivery may still reach the client." + ) + + def resumeDraining(): Unit = + core.resumeDeliveries() + core.rearmDrain().foreach(scheduleTick) + // A FAILED permit release has no queue to ride back on: the re-arm above only schedules + // when something is QUEUED, and the release only failed because the drain had just + // emptied the queue. Retrying here is what stops a hold this limiter owns from outliving + // the pause that swallowed its retry. The retry re-checks the watermark, so a hold that is + // still EARNED is left exactly where it is. + if permitsHeld.get || permitsForceHeld.get then scheduleReleaseRetry() + + def stop(): Unit = + core.pauseDeliveries(reentrant = inDelivery.get, timeoutMs = deliveryPauseQuiesceTimeoutMs) + core.clear() + + def queuedCount: Int = core.queuedCount + + /** Whether draining is paused right now (user pause, spent budget) - the guaranteed pump's + * stop signal, since it bypasses the queue this flag normally guards. */ + def isDrainingPausedNow: Boolean = core.isDeliveryPaused + + /** See [[DeliveryRateLimiterCore.tryAcquireDeliveryToken]]. */ + def tryAcquireDeliveryToken(): Boolean = core.tryAcquireDeliveryToken() + + /** See [[DeliveryRateLimiterCore.returnDeliveryToken]] - the failed-send refund. */ + def returnDeliveryToken(): Unit = core.returnDeliveryToken() + + private def scheduleTick(delayMs: Long): Unit = schedule(delayMs, () => tick()) + + private def tick(): Unit = + if core.isDeliveryPaused then + // The timer this tick consumed must not stay recorded as armed, or the backlog would + // never get another one. Resume re-arms explicitly. + core.cancelScheduledDrain() + rearmIfResumedUnderneath() + else + val batch = core.beginDrain() + // ADMITTED between items, not merely CHECKED between them. The delivery budget stops + // the line from INSIDE a delivery, and a user pause can land at any instant; both need + // the flag test and the delivery's claim on it to be one step, or a message decided a + // moment before Pause is still sent and acknowledged a moment after it answered. + // Everything behind the refusal goes back - untouched, unacknowledged, order intact - + // to the front of the queue for the next resume. + var processed = 0 + while processed < batch.size && core.beginDelivery() do + runAdmittedDelivery(batch(processed)) + processed += 1 + if processed < batch.size then core.requeueFront(batch.drop(processed)) + val fin = core.finishDrain() + if underReleaseWatermark && (permitsHeld.get || permitsForceHeld.get) then releaseHeldPermits() + fin.rescheduleAfterMs.foreach { delay => + // A drain stopped by the budget leaves its backlog armed in the core; consuming + // the reschedule without scheduling would strand it, so the armed flag is handed + // back for the next resumeDraining to re-arm. + if core.isDeliveryPaused then + core.cancelScheduledDrain() + rearmIfResumedUnderneath() + else scheduleTick(delay) + } + + /** THE OTHER HALF OF HANDING THE DRAIN FLAG BACK, and it is not optional. + * + * A Resume that lands between a tick DECIDING it is paused and that tick clearing the armed + * flag calls `rearmDrain` while the flag still reads "armed", so it schedules nothing - and + * the clear then removes the only timer the backlog had. Queue nonempty, no timer, and + * nothing further is required to arrive: the backlog was stuck for good while the session + * reported Running. + * + * Re-reading the flag AFTER the clear makes whoever runs last arm the backlog, under every + * order, and never twice: `rearmDrain` refuses while a timer is already recorded. + */ + private def rearmIfResumedUnderneath(): Unit = + if !core.isDeliveryPaused then core.rearmDrain().foreach(scheduleTick) + + /** Whether the paced backlog is back under the marks at which the permits are handed back. */ + private def underReleaseWatermark: Boolean = + core.queuedCount <= deliveryRateLimitReleasePermitsBelowQueued + && core.queuedBytesCount <= deliveryRateLimitReleasePermitsBelowQueuedBytes + + /** Let go of every permit this limiter owns - the ordinary hold and the ceiling's forced one + * together, since the release hook lets go of every stream either way. + * + * A FAILED release puts the flags back and arms a retry: a drained queue produces no further + * crossings, so nothing else would ever notice, and the consumers would stay held until the + * user cycled the session. + */ + private def releaseHeldPermits(): Unit = + val hadHold = permitsHeld.compareAndSet(true, false) + val hadForcedHold = permitsForceHeld.compareAndSet(true, false) + if (hadHold || hadForcedHold) && !releasePermits() then + if hadHold then permitsHeld.set(true) + if hadForcedHold then permitsForceHeld.set(true) + scheduleReleaseRetry() + + /** THE RELEASE RETRY IS ITS OWN WORK, never a drain tick. + * + * Armed as a tick it was swallowed whole by a user pause - the paused branch hands the drain + * flag back and returns - and Resume only re-arms a drain when the QUEUE is nonempty, which it + * never is after the drain that failed the release. The Limiter's hold then sat on every + * consumer for the rest of the session: no data, and no recovery from the user's Resume. + */ + private def scheduleReleaseRetry(): Unit = + schedule(deliveryRateLimitReleaseRetryDelayMs, () => retryPermitRelease()) + + private def retryPermitRelease(): Unit = + // Re-checked rather than assumed: by the time a retry fires the backlog may have crossed + // the hold watermark again, and letting go then would be flapping rather than recovery. + // Deliberately NOT gated on the user's pause - with per-reason arbitration releasing the + // Limiter reason can never resume a consumer the User still holds. + if (permitsHeld.get || permitsForceHeld.get) && underReleaseWatermark then releaseHeldPermits() + + /** Run a delivery the core has ADMITTED, and hand the admission back however it ends. + * + * The `finally` is the pause boundary's other half: a delivery that threw still has to release + * the count, or Pause would wait out its whole timeout for a delivery that finished long ago. + * The thread-local marks the reentrant case - see [[inDelivery]]. + */ + private def runAdmittedDelivery(a: A): Unit = + inDelivery.set(java.lang.Boolean.TRUE) + try runProtected(a) + finally + inDelivery.set(java.lang.Boolean.FALSE) + core.endDelivery() + + /** One failing delivery must cost that delivery alone - the drain has a whole batch behind it, + * and [[ConsumerListener.deliverNow]] already nacks its own failures. This is the same + * per-message containment the resolved-batch loop learned in round 3, applied to the drainer. */ + private def runProtected(a: A): Unit = + Try(process(a)).failed.foreach(err => logger.warn(s"A rate-limited delivery failed and was skipped. ${err.getMessage}")) diff --git a/server/src/main/scala/consumer/session_runner/globalStartFrom.scala b/server/src/main/scala/consumer/session_runner/globalStartFrom.scala new file mode 100644 index 000000000..01d78e082 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/globalStartFrom.scala @@ -0,0 +1,1703 @@ +package consumer.session_runner + +import _root_.consumer.start_from.{ConsumerSessionStartFrom, NthMessageAfterEarliest, NthMessageBeforeLatest} +import org.apache.pulsar.client.api.{Consumer, MessageIdAdv, Message as PulsarMessage, MessageId as PulsarMessageId} + +import com.typesafe.scalalogging.Logger + +import scala.collection.mutable +import scala.jdk.CollectionConverters.* +import scala.util.{Failure, Success, Try} + +/** Where a message sits in its own log: the entry that holds it, plus its place inside that entry + * when the entry is a producer batch. + * + * `batchIndex` is -1 for a message that was not batched, which is also how Pulsar's own ids report + * it - so an unbatched message sorts before batch index 0 of the same entry, and the two can never + * be confused. `batchSize` is how many messages the entry holds (1 when unbatched); it is only + * needed to answer "was that the last message of this entry". + */ +final case class EntryPosition(ledgerId: Long, entryId: Long, batchIndex: Int, batchSize: Int) + +object EntryPosition: + /** Nothing retained: what `MessageId.earliest` reports, and what an empty topic answers with. */ + val empty: EntryPosition = EntryPosition(-1L, -1L, -1, 0) + + /** The position a delivered Pulsar message occupies. + * + * `MessageIdAdv` is the interface every addressable id implements - plain, batched, and the + * topic-qualified wrapper a multi-topic consumer hands back, which forwards these accessors to + * the id underneath. It reports `batchIndex` -1 and `batchSize` 0 for an unbatched message. + * + * Anything else is an id nothing here can address, and is reported as [[empty]] rather than + * guessed at - an unaddressable id must not be allowed to sort somewhere plausible. + */ + def of(messageId: PulsarMessageId): EntryPosition = messageId match + // An EMPTY topic answers with MessageId.earliest, whose ledger and entry are both -1. It has + // to land on exactly [[empty]] and not merely near it: "nothing retained" is recognised by + // equality, and a position of (-1, -1, -1, 1) is not equal to (-1, -1, -1, 0), so an empty + // partition would be waited on forever and hold the whole session at zero messages. + case id: MessageIdAdv if id.getLedgerId >= 0 && id.getEntryId >= 0 => + EntryPosition(id.getLedgerId, id.getEntryId, id.getBatchIndex, id.getBatchSize max 1) + case _ => empty + +/** The total order used by the streaming cross-source merge: selected timestamp, topic name, then + * position in the log (ledger, entry, batch index). + * + * A total order is not a nicety here, it is what makes the result deterministic. Timestamp ties + * are common, so an order + * that stopped at the timestamp would leave the result of "the last 5" up to whichever partition + * the broker happened to deliver first, and a test asserting an exact set would flake. + * + * The merge compares only the current head of each source, preserving Pulsar append order within + * a topic or partition. It is therefore globally timestamp-sorted only while each source's + * timestamps are nondecreasing; a timestamp inversion already stored in one source passes through + * and is counted. + * + * For a multi-stream Skip-first-n, Fastest uses publish time for the cut; Guaranteed and Best + * effort use their selected timestamp and keep the same merge after the cut. Latest-n is separate: + * it resolves per-view anchors from publish-time entry metadata before delivery begins. + */ +final case class MessageOrderKey(orderTime: Long, topicFqn: String, ledgerId: Long, entryId: Long, batchIndex: Int) + +object MessageOrderKey: + given ordering: Ordering[MessageOrderKey] = + Ordering.by(key => (key.orderTime, key.topicFqn, key.ledgerId, key.entryId, key.batchIndex)) + + def of(orderTime: Long, topicFqn: String, position: EntryPosition): MessageOrderKey = + MessageOrderKey(orderTime, topicFqn, position.ledgerId, position.entryId, position.batchIndex) + +/** Whether `delivered` is at or past the last message the topic held when the session started - + * i.e. whether the PRE-EXISTING BACKLOG of that topic is now drained. + * + * ONLY [[GlobalSkipMerge]] needs this, and it needs exactly one bit: a k-way merge stalls forever + * on a partition that has nothing left to offer unless that partition can be declared "+infinity". + * "This is the recorded end" and "this proves the recorded end will never be delivered" both mean + * stop waiting, so one Boolean is the whole answer. + * + * It used to be a "latest n" input too, where the same Boolean was NOT enough: a live message past + * an undeliverable recorded end entered the historical top-n heap and could evict a backlog message + * the user had asked for. There is no heap any more - "latest n" resolves its cut from entry + * metadata before anything is delivered (see `resolveLatestN`) - so no delivered message can change + * the answer, and the distinction has nothing left to affect. + * + * THE BATCH WRINKLE: `Consumer.getLastMessageIds` may answer with a plain entry id (batch index -1) + * even when that entry is a producer batch, so "same entry, batch index >= -1" would declare the + * backlog drained on the FIRST message of the final batch and lose the rest of it. When the + * recorded end carries no batch index, the end of that entry is taken from the DELIVERED message's + * own batch size instead, which every batched message carries. + */ +def isPastBacklogEnd(delivered: EntryPosition, lastAtStart: EntryPosition): Boolean = + val deliveredEntry = (delivered.ledgerId, delivered.entryId) + val lastEntry = (lastAtStart.ledgerId, lastAtStart.entryId) + if deliveredEntry != lastEntry then Ordering[(Long, Long)].gt(deliveredEntry, lastEntry) + else if lastAtStart.batchIndex >= 0 then delivered.batchIndex >= lastAtStart.batchIndex + // The recorded end named the entry but not a position inside it. An unbatched message IS the + // whole entry, so reaching it is the end; a batched one has to be the last of its own batch. + else if delivered.batchIndex < 0 then true + else delivered.batchIndex >= delivered.batchSize - 1 + +/** Whether `delivered` sits STRICTLY past the recorded end - in a LATER entry than the last one + * the topic held when the boundary was captured. + * + * The GUARANTEED replay's next-chunk test. Entry-level on purpose: a producer batch is written + * as one entry atomically, so the whole final entry either existed at the capture (every member + * of it is in the replay) or did not - no batch can straddle the boundary, and no batch-index + * arithmetic is needed. Against an EMPTY recorded end every real position is strictly past: + * a stream that held nothing at the boundary has nothing to replay, so everything it delivers + * belongs to the next chunk. Per-stream append order makes such an arrival PROOF that the + * recorded range is drained - nothing at or before the end can follow it. + */ +def isStrictlyPastBacklogEnd(delivered: EntryPosition, lastAtStart: EntryPosition): Boolean = + Ordering[(Long, Long)].gt((delivered.ledgerId, delivered.entryId), (lastAtStart.ledgerId, lastAtStart.entryId)) + +/** What a global start-from layer decided about one message it was holding. */ +enum StartFromOutcome: + /** Not part of the position the user asked for - acknowledge it and show it to nobody. */ + case Drop + + /** Part of it - hand it to the session. */ + case Deliver + + /** Deliver, AND the merge emitted it below an already-emitted order key (Best effort's late + * emission - the message outlived its reorder window; since 2026-08-11 the row carries the + * marker for it). In the OUTCOME rather than on the payload because the merge is generic in + * what it holds; the production handler stamps the verdict onto the HeldMessage. */ + case DeliverOutOfOrder + + /** A broker COPY whose original's fate is still open: hand it back, decide nothing - no + * acknowledgment, no budget, no cursor. The broker redelivers it with backoff until the + * original settles: its ack ends the copies, its recorded failure makes the next copy the + * retry. Both eager answers were message-loss paths - see the merge's duplicate arm. */ + case Requeue + + /** GUARANTEED replay: recorded PAST the boundary, so it belongs to the NEXT chunk. Hand it + * back and PAUSE its consumer (reason Boundary) - no acknowledgment, no start-from budget, + * no delivery memo, no cursor, no duplicate watermark: after a Resume extends the boundary + * the broker's redelivery must arrive as an ordinary first-time offer of the new chunk. + * The arrival itself is proof its stream's recorded range is drained (append order), which + * the ordering layer feeds to the barrier separately. */ + case NextChunk + + +/** A session-wide reordering layer sitting between the per-topic delivery streams and the session. + * + * Every physical topic delivers in its own order and on its own thread; a GLOBAL position can only + * be decided by looking at all of them at once, so a message has to be HELD until enough is known + * about the others to place it. `offer` answers with the messages that offer RESOLVED - which is + * usually not the one just offered, and may be none at all. + */ +trait StartFromMerge[P]: + /** Take one delivered message. `atBacklogEnd` says this message is at (or past) the last one + * its topic held when the replay boundary was captured, so the merge must stop waiting for + * that topic. `knownFailedRetry` is the LISTENER's explicit lifecycle fact - "this exact + * message's delivery already failed downstream" - which is the only license to deliver a + * watermarked copy. `entryPosition` is the message's full entry position (batch size + * included); the GUARANTEED replay records it per stream so a boundary EXTENSION can tell a + * finished stream from one with a delta - the key alone cannot, it carries no batch size. */ + def offer( + streamId: String, + key: MessageOrderKey, + atBacklogEnd: Boolean, + payload: P, + knownFailedRetry: Boolean = false, + entryPosition: EntryPosition = EntryPosition.empty + ): Vector[(P, StartFromOutcome)] + + /** Messages held right now. Exposed so a test can pin the MEMORY PROFILE - the whole point of + * these two algorithms is what they refuse to buffer. */ + def heldCount: Int + + /** The counter start-from progress is read off, when this layer owns one. */ + def progressDiscard: Option[StartFromDiscard] + + /** The TIME-DRIVEN half of the stall bound. The offer-driven check runs only when some stream + * speaks - and after the last held backlog message there may be nobody left to speak, so a + * waited-for stream that was retention-trimmed used to hold the merge FOREVER with the window + * long expired. A watchdog calls this on a timer; anything it resolves is handled exactly as + * an offer's resolutions are. The default holds nothing, so it has nothing to sweep. */ + def sweepStalled(): Vector[(P, StartFromOutcome)] = Vector.empty + + /** Restart the silent-stream clock. Called on session RESUME: a pause holds every source, so + * time spent paused proves nothing about a stream's health - counting it toward the give-up + * window abandoned a healthy stream the moment a long-paused session came back. The default + * tracks no clock. */ + def resetStallClock(): Unit = () + + /** True once this layer can never hold or reorder anything again - its work is done and every + * message from here on passes straight through. From that point the session-wide ordering + * lock is pure overhead (it would serialize every partition's deserialization and JS + * evaluation for the rest of the session), so [[StartFromOrdering.inOrder]] stops taking it. + * ONE-WAY: nothing may ever un-settle a merge. */ + def isSettled: Boolean = false + + /** Flip [[isSettled]] once the budget is spent and nothing is held. Called by the LISTENER + * after a resolved batch is fully handled - never from inside the merge's own advance - so + * the settling batch itself is processed under the ordering lock before any later message is + * allowed to bypass it. The default has nothing to settle. */ + def settleIfDone(): Unit = () + + /** The streams flow control wants PAUSED right now; absent means running. Memory is bounded by + * holding a hot source STILL (consumer.pause) instead of bouncing its messages back through + * redelivery - see [[startFromMergeMaxHeld]] for why the bounce was the bug. The default + * holds nothing and pauses nobody. */ + def desiredPausedStreams: Set[String] = Set.empty + + /** The streams a give-up abandoned - the DEGRADATION record, in the order they were dropped. + * Non-empty means the position was resolved best-effort: the count stayed exact, the exact + * SET may differ. Sticky for the layer's life, so the client can keep showing it. */ + def abandonedStreamIds: Vector[String] = Vector.empty + + /** How many times the merge has newly crossed its stall-WARN window (a waited-for stream + * silent past [[startFromMergeStallWarnMs]]). Monotonic; the sweep pushes a progress frame + * when it grows, so a stalled positioning is VISIBLE to the client while the merge is still + * only waiting - not first at the give-up half a minute later. */ + def stallWarningCount: Long = 0L + + /** Streams still missing a head after the stall-warning interval. Unlike + * [[stallWarningCount]], this returns to zero when delivery can progress. */ + def stalledStreamCount: Int = 0 + + /** The stalled streams BY ID - non-empty exactly while [[stalledStreamCount]] is non-zero. + * Feeds the server-side stall diagnostic only: the wire's ConsumerStats carries the count + * alone, and a per-stream payload would need a proto field that does not exist. */ + def stalledStreamIds: Vector[String] = Vector.empty + + /** Whether this layer keeps ordering deliveries for the session's whole life rather than only + * until a counted cut resolves. A continuous layer never + * settles, so the runner must keep its sweep armed forever - and at the grace cadence, not + * the stall one. */ + def isContinuousOrdering: Boolean = false + + /** How many emissions were LATE - ordered before something already emitted. The honest + * quality gauge of a best-effort order: zero on a healthy merge, growing when producer + * clocks skew past the grace, a stream outwaits its residence bound, or a failed delivery + * is retried. Under GUARANTEED ordering it counts exactly the source-log inversions that + * passed through in append order. Monotonic; the default merges nothing and is never late. */ + def lateDeliveryCount: Long = 0L + + /** Whether this layer runs the no-escape GUARANTEED ordering - delivery then goes through + * the peek/commit barrier below instead of [[offer]]'s resolutions. */ + def isGuaranteedOrdering: Boolean = false + + /** RELAX a LIVE guaranteed layer to best effort, and answer with whatever that released - in + * the new order, handled exactly as an offer's resolutions are. See + * [[GlobalSkipMerge.relaxGuaranteedToBestEffort]] for the whole contract. The default layer + * orders nothing and has nothing to relax. */ + def relaxGuaranteedToBestEffort(): Vector[(P, StartFromOutcome)] = Vector.empty + + /** GUARANTEED mode: the single safe head, WITHOUT advancing - or None while any waited + * stream is silent (the no-escape wait) or a prior peek is still uncommitted elsewhere. */ + def peekGuaranteed(): Option[P] = None + + /** The peeked head's send SUCCEEDED: pop it and advance the bookkeeping. */ + def commitGuaranteed(): Unit = () + + /** The peeked head's send FAILED: keep it, clear the in-flight slot; the next peek retries + * the same head - in place, in order. */ + def abortGuaranteed(): Unit = () + + /** GUARANTEED replay: every stream has delivered its recorded range and nothing is held - + * the chunk is COMPLETE and the session should auto-pause with the caught-up signal. The + * default layer replays nothing and is never caught up. */ + def isReplayCaughtUp: Boolean = false + + /** GUARANTEED replay: this stream's recorded range is PROVEN drained - a strictly-past- + * boundary arrival, which per-stream append order guarantees nothing recorded can follow. + * The stream leaves the barrier; the arrival itself was handed back as [[StartFromOutcome.NextChunk]]. */ + def noteStreamReplayFinished(streamId: String): Unit = () + + /** GUARANTEED replay: EXTEND the boundary to the freshly captured `ends` - what a Resume + * does. A stream whose new end lies past everything it has offered re-enters the barrier + * (it has a delta to replay); the rest stay finished. One rule for a manual pause's resume + * and the auto-pause's: the un-emitted remainder and the delta merge in the same heap. */ + def extendReplayBoundary(ends: Map[String, EntryPosition]): Unit = () + + /** Whether the head [[peekGuaranteed]] just answered would be emitted with an order key + * LOWER than one already emitted - the per-message seam-violation flag, computed at peek so + * the outgoing row can carry it. Agrees with [[replaySeamViolationCount]] by construction: + * nothing else can emit between a peek and its commit. */ + def peekIsSeamViolation: Boolean = false + + /** How many guaranteed emissions went out with an order key lower than one already emitted - + * the session-level seam counter the client's banner hangs on. By the SELECTED key, against + * previously emitted keys of the same kind, never against wall clock: ordinary old event + * times under the event-time key are not violations. Monotonic. */ + def replaySeamViolationCount: Long = 0L + +/** The TOTAL-held high watermark: past this many held messages, every stream with a queue is + * marked for PAUSE until the total drains back under four fifths of it. + * + * A HEALTHY merge holds one message per stream - it advances as soon as every stream it is + * still waiting for has a head - but that is the happy case, not a bound: whenever one stream + * is silent or slow, the others QUEUE here up to these watermarks (the wide-session test holds + * ~21,000 for 1,000 streams before its sweep). A stream that goes quiet without reaching the + * end of its backlog - a stalled broker connection, a trimmed partition - would otherwise let + * the other streams queue up without bound while the merge waited. + * + * THE BOUND IS FLOW CONTROL, NOT REFUSAL. The merge used to DECLINE offers past the cap and hand + * them back for ~100ms redelivery, and that opened the one door a k-way merge cannot leave open: + * the broker redelivers a declined message while still delivering its successors, so a stream + * could re-enter the merge OUT OF ITS OWN APPEND ORDER. A one-key floor guard held the door for + * the FIRST declined message, but successors declined by the guard itself were not remembered - + * two of them returning out of order could still spend the budget's last unit on the wrong + * message, on a perfectly monotonic stream. Pausing the source closes the whole class: nothing + * is handed back, so nothing can return out of order, and the permanent nack storm at the cap is + * gone with it. What arrives between the watermark and the pause taking effect is ACCEPTED - + * in-order arrivals are always safe to hold - so this is a watermark with a bounded overshoot, + * not a hard wall. THE OVERSHOOT IS THE RECEIVER QUEUE, not one callback: `consumer.pause()` + * only stops asking for more while Pulsar keeps delivering what the consumer already prefetched + * (see DeliveryRateLimiter's note on the same behavior) - which is why merge-ordered sessions + * get a deliberately small per-consumer receiver queue (`receiverQueueSizeFor`'s merge budget). + * These watermarks bound what the MERGE holds; they are not a whole-session byte ceiling. + * + * THIS IS THE ONLY START-FROM LAYER THAT HOLDS MESSAGES AT ALL - transiently for a counted cut, + * for the session's life in continuous mode - and it holds them only because a global order + * cannot be decided without candidates in hand: Pulsar keeps no message-ordinal index and no + * cross-topic sequence. "Latest n" CAN be resolved from metadata, and is - see `resolveLatestN` + * - so it buffers nothing. + * + * A stream the merge is BLIND on has an empty queue by definition, so no watermark ever marks it: + * the one stream whose next message can unblock the merge is always left running. + */ +val startFromMergeMaxHeld: Int = 10_000 + +/** PER-STREAM queue high watermark: a stream whose own queue reaches this is paused even while + * the total is fine - one hot stream must not own the whole budget while a slow one catches up. */ +val startFromMergePauseStreamAt: Int = 1_000 + +/** Per-stream LOW watermark: a stream paused for its own queue resumes once it drains to this. + * The gap to [[startFromMergePauseStreamAt]] is the hysteresis that keeps pause/resume from + * flapping around one boundary. */ +val startFromMergeResumeStreamAt: Int = 100 + +/** Held-BYTES high watermark. The count caps above know nothing about payload size, and ten + * thousand held 5 MB messages would be 50 GB: past this many held payload bytes every stream + * with a queue is paused, whatever the counts say. */ +val startFromMergePauseBytesAt: Long = 256L * 1024 * 1024 + +/** Held-bytes LOW watermark for resuming what the byte cap paused. */ +val startFromMergeResumeBytesAt: Long = 64L * 1024 * 1024 + +/** How long the merge stays blind on a waited-for stream that has produced no head before it SAYS SO + * in the log - naming the topic-partition an operator can then act on. + * + * The merge cannot decide anything while a stream it is waiting for has said nothing: it holds + * the others (and pauses the hot ones at the watermarks). That is correct for a stream that is + * merely slow, but a partition whose backlog was trimmed by retention AFTER the session recorded + * its end will never deliver that end - the wait and the holding are then permanent and, until + * this, entirely silent. Warning before acting keeps a slow-but-alive stream from being cut + * early. + */ +val startFromMergeStallWarnMs: Long = 5_000L + +/** How long the merge stays blind on a silent waited-for stream before it STOPS waiting for it - + * treating it as drained, logging which stream it abandoned, and advancing. + * + * This is the bound on the otherwise-permanent wait. It is deliberately far longer than the + * warning and than any healthy broker read, so a stream that is slow rather than gone is not cut: + * only a stream that has delivered nothing for this long - the trimmed-partition case - is given up + * on. A given-up stream that later speaks after all has its messages passed straight through, so the + * cost of abandoning it early is the same "which messages" fuzziness the append-order contract + * already carries, never a lost count. + */ +val startFromMergeStallWindowMs: Long = 30_000L + +/** How often the runner's watchdog sweeps a merge that might be stalled with no offers arriving. + * Small against the give-up window, so the bound the window promises is met within one period of + * the promised time even in total silence. */ +val startFromStallSweepPeriodMs: Long = 2_000L + +/** The Best-effort reorder window: how long a held message may RESIDE in + * the merge, waiting for quieter streams to get their say, before it is emitted anyway. + * + * This is what "best effort" means, made concrete: BOUNDED LATENESS, never blocked delivery. + * Publish time is stamped by the PRODUCER's clock (see [[MessageOrderKey]]), so comparing it + * against this process's clock proves nothing - a producer running minutes ahead or behind is + * ordinary. The merge therefore never compares publish times to wall time at all: it orders + * whatever co-resides in the window, and RESIDENCE EXPIRY on the merge's own monotonic clock is + * the unconditional liveness escape - a head that has waited this long is emitted whatever any + * silent stream might still say, future-dated timestamps included. The price is worn openly: + * delivery lags by up to this window PLUS one sweep period (the timer that notices expiry - + * ~0.5-0.75s in total) whenever some stream is quiet, and a message arriving after its peers + * were released is emitted LATE, out of order, and counted. */ +val mergeTopicsGraceMs: Long = 500L + +/** Which job(s) one merge instance performs. An explicit phase model, replacing a pair of + * boolean constructor flags (`continuous`, `waitStreamsAtStart`) whose four combinations + * included two nonsense states - a merge that neither cuts nor orders, and an order-only layer + * that still gates on recorded backlog ends it never fetched. Illegal states are now simply + * unrepresentable, and each name carries its contract: + * + * - [[OrderingPolicy.ExactCutOnly]]: resolve the exact counted cut, then go inert - the + * default skip-n merge. + * - [[OrderingPolicy.ExactCutThenBestEffort]]: the same exact cut, then keep ordering + * deliveries best-effort for the session's life. + * - [[OrderingPolicy.BestEffortOnly]]: no cut to make - best-effort ordering from the first + * message (every non-counted mode with Best effort). Needs no recorded + * ends and must carry no budget. + */ +enum OrderingPolicy: + case ExactCutOnly + case ExactCutThenBestEffort + case BestEffortOnly + + /** Resolve the exact counted cut, then GUARANTEED ordering for the session's life. */ + case ExactCutThenGuaranteed + + /** No cut - the GUARANTEED EXACT REPLAY (owner decision 2026-08-09): the exact k-way merge + * rule over the RECORDED RANGE of every stream, bounded by the per-stream ends captured + * when Play was pressed. A stream that has delivered its recorded range is FINISHED and + * leaves the barrier ([[isPastBacklogEnd]] at the offer, or a strictly-past-boundary + * arrival as proof - see [[StartFromMerge.noteStreamReplayFinished]]); when every stream is + * finished the remaining heap drains in key order - the last message is deliverable because + * every end is known - and [[StartFromMerge.isReplayCaughtUp]] turns true, which is the + * caller's cue to AUTO-PAUSE and signal. A Resume EXTENDS the boundary + * ([[StartFromMerge.extendReplayBoundary]]) and the same session replays the delta. Within + * the un-finished range there is still no liveness escape of any kind: a silent stream + * holds delivery until it speaks or proves itself drained, a failed send retries the same + * head in place (see the peek/commit barrier), and nothing is ever abandoned - exactness + * over history is the entire promise, and history is immutable so it costs no stall on a + * healthy topic. (Before the replay redesign this mode waited on every stream FOREVER - + * the recorded ends were ignored - so a completed backlog was an eternal hold.) + * + * THE WAIT IS NOT A MEMORY HOLE. An ahead stream's backlog accumulates only up to the same + * flow-control watermarks as every other mode ([[startFromMergePauseStreamAt]] per stream, + * [[startFromMergeMaxHeld]] / [[startFromMergePauseBytesAt]] in total): past them the ahead + * stream's CONSUMER is paused until the barrier drains its queue back down. Delaying receipt + * can never violate the order - only early emission could - so the backpressure is free + * semantically. A blind stream holds nothing and is never paused, so the one stream whose + * next message can unblock the barrier always runs; the overshoot past a watermark is + * bounded by the consumer receiver queue (`mergeReceiverQueueSizeFor`: a 5,000-message + * session budget divided across the streams). */ + case GuaranteedOnly + +/** Sweep cadence while a BEST-EFFORT continuous merge is armed: the timer half of the grace above. + * An all-idle tail emits nothing until either a new offer arrives or this sweep notices the grace + * has passed, so the effective added latency is grace + one period. The skip watchdog's 2s + * cadence is fine against a 30s give-up window and far too coarse against a 500ms grace. */ +val mergeTopicsSweepPeriodMs: Long = 250L + +/** Sweep cadence for a GUARANTEED continuous merge, which has no residence to expire and never + * gives a stream up: its ticks exist only to surface a stall and to re-attempt a head whose send + * failed. Neither is worth the best-effort cadence on an idle session. */ +val guaranteedSweepPeriodMs: Long = 1_000L + +/** How often a continuous ordering layer's sweep runs, by the policy actually in force. + * + * POLICY-SPECIFIC, because the two modes want the tick for different things. Best effort needs it + * at the grace cadence: it is the timer half of the residence bound, and an all-idle tail emits + * nothing until a tick notices the grace has passed. Guaranteed has no residence to expire and + * never gives a stream up, so its ticks only surface a stall and re-attempt a head whose send + * failed - neither worth four ticks a second on a session that is simply waiting. Any offer pumps + * the barrier anyway, so the slower cadence costs latency only in the rare case of a failed send + * with no further traffic behind it. + * + * Pure, so the choice is pinned by test rather than inferred from a scheduled task. */ +def continuousSweepPeriodMs(guaranteed: Boolean): Long = + if guaranteed then guaranteedSweepPeriodMs else mergeTopicsSweepPeriodMs + +/** How many stream names one stall log line may carry. A session may hold 2,000 streams and a + * stall names every silent one of them; the whole set in a single line is not a diagnostic. */ +val stallLogStreamNameCap: Int = 10 + +/** The stream names for a stall log, capped - the rest are counted, not spelled out. */ +def stalledStreamsForLog(names: Iterable[String], cap: Int = stallLogStreamNameCap): String = + val named = names.take(cap).mkString(", ") + val rest = names.size - math.min(cap, names.size) + if rest <= 0 then named else s"$named and $rest more" + +/** "Skip the globally-first n messages, by publish time, across every physical topic." + * + * A STREAMING K-WAY MERGE over per-stream heads. It holds at most one message per physical topic: + * take the smallest head under [[MessageOrderKey]], drop it, wait for that stream to produce its + * next one, repeat. Memory is O(number of topics) and NEVER O(n) - "skip first n" has deliberately + * no cap, so buffering n messages to sort them would let a user type a number that exhausts the + * heap. + * + * A k-way merge is exact only if each input is already sorted, and a Pulsar log is sorted by APPEND + * order rather than by producer clock - see [[MessageOrderKey]] for what that narrows the contract + * to, and why buffering to fix it is the one thing this class must not do. + * + * ONCE THE BUDGET IS SPENT THE MERGE STOPS - unless `continuous` is set. In the default mode + * everything held is released in order and every later message is passed straight through, so + * the delivery SEQUENCE after the cut is the brokers' and not this order: a stream can deliver a + * newer message before another stream delivers an older one. Continuing to merge would mean + * holding one message from every stream for the whole life of a session that is now only + * streaming - which is exactly what a continuous delivery-order mode pays for. + * + * CONTINUOUS BEST-EFFORT MODE. The same queues, heap, flow + * control and dedup keep running for the session's life; what changes is the emission rule. + * While a counted cut is unresolved the exact rule holds (every waited stream must have a head). + * After it - and for the non-counted modes, from the start - two rules govern: + * + * - ORDER-SAFE FAST PATH: a head is emitted at once when every silent stream has already + * offered something at or past its publish time - nothing older can follow it from a + * stream that preserves append order and roughly monotonic producer time. + * - BOUNDED RESIDENCE: otherwise the head waits, but never past the grace measured on the + * merge's own monotonic clock from the moment it arrived. Residence expiry is the + * unconditional liveness escape: producer clocks are nobody's to trust (publish time is + * stamped by the PRODUCER - a future-dated head must not wedge its stream), and a slow + * peer must cost bounded latency, not a wedge. + * + * A message that arrives after its peers were already released is emitted immediately, LATE and + * out of order, and counted ([[lateDeliveryCount]]) - never blocked, never dropped. A + * session-wide startup hold of one grace lets every stream deliver its first message before + * anything is emitted, so replaying history does not open with a scramble while slow streams + * are still subscribing. Best EFFORT, not a guarantee - and the guarantee it does keep is + * bounded lateness plus per-stream append order. + * + * A stream that has reached the end of its pre-existing backlog counts as +infinity: the merge + * stops waiting for it, so a small partition cannot stall a session whose other partitions still + * have millions of messages to go. + * + * ONE STREAM NEEDS NO MERGE, and does not get one - a single log is already in order, so the + * caller uses the plain head-drop counter and holds nothing at all. + * + * `discard` is both the budget and the progress source: `claim` answers "yes, still dropping" and + * counts the drop, and answering "no" is what ends the skip. It must be a SHARED counter (one for + * the whole session), since the merge counts the merged stream and not any one topic. + */ +final class GlobalSkipMerge[P]( + streamIds: Vector[String], + drainedAtStart: Set[String], + discard: StartFromDiscard, + maxHeld: Int = startFromMergeMaxHeld, + stallWindowMs: Long = startFromMergeStallWindowMs, + // MONOTONIC, not wall-clock: the stall window is an elapsed-time promise, and an NTP step + // under currentTimeMillis either abandoned a healthy stream early or held a dead one longer + // than promised. + nowMs: () => Long = () => System.nanoTime() / 1_000_000L, + pauseStreamAt: Int = startFromMergePauseStreamAt, + resumeStreamAt: Int = startFromMergeResumeStreamAt, + pauseBytesAt: Long = startFromMergePauseBytesAt, + resumeBytesAt: Long = startFromMergeResumeBytesAt, + // How many payload bytes a held message costs, for the byte watermarks. The default counts + // nothing, which disables byte-based pausing - production wires the real payload size in. + payloadBytesOf: P => Long = (_: P) => 0L, + policy: OrderingPolicy = OrderingPolicy.ExactCutOnly, + graceMs: Long = mergeTopicsGraceMs +) extends StartFromMerge[P]: + require( + policy != OrderingPolicy.BestEffortOnly || discard.remaining == 0, + "A BestEffortOnly layer has no cut to make and must carry no skip budget" + ) + + /** The policy in force RIGHT NOW. A var, not the constructor parameter, because + * [[relaxGuaranteedToBestEffort]] can lower it on a live session - the one mutation this + * class accepts, and only ever in the relaxing direction (see that method for why the other + * direction is refused rather than faked). Read under this object's monitor like every other + * piece of merge state. + */ + private var activePolicy: OrderingPolicy = policy + + /** Whether ordering continues past any cut. The exact-cut phase itself is tracked by + * `dropping` below. */ + private def continuous: Boolean = activePolicy != OrderingPolicy.ExactCutOnly + + /** GUARANTEED ordering: the exact rule with no liveness escape - no residence bound, no + * startup release, no give-up, and delivery through the peek/commit barrier below. */ + private def guaranteed: Boolean = + activePolicy == OrderingPolicy.ExactCutThenGuaranteed || activePolicy == OrderingPolicy.GuaranteedOnly + + override def isGuaranteedOrdering: Boolean = guaranteed + // A global skip counts the WHOLE merged stream, so its budget must be the SHARED counter - the + // one `claim` keys by nothing (it ignores its topic-FQN argument, see StartFromDiscard.claim). + // That is what lets `advance` hand `claim` a STREAM id rather than a topic FQN below. Asserted so + // the type confusion cannot silently become real if a per-topic counter is ever passed here. + require(discard.reportsProgress, "GlobalSkipMerge requires a shared (session-wide) discard counter") + + private val logger: Logger = Logger(getClass.getName) + + private val pending: mutable.Map[String, mutable.Queue[(MessageOrderKey, P, Long)]] = + mutable.Map.from(streamIds.map(_ -> mutable.Queue.empty[(MessageOrderKey, P, Long)])) + // Who gates emission at the start. BestEffortOnly waits on nobody (the residence bound and + // the fast path carry it); every other policy - the exact-cut pair AND the guaranteed + // replay - waits on each stream exactly until its recorded range is drained: a stream that + // held nothing at the boundary has nothing to replay and is never waited for. (The + // pre-replay Guaranteed waited on EVERYBODY forever, recorded ends ignored - the eternal + // hold the exact-replay redesign removed, owner decision 2026-08-09.) + private var waiting: Set[String] = policy match + case OrderingPolicy.BestEffortOnly => Set.empty + case _ => streamIds.toSet -- drainedAtStart + private var dropping: Boolean = discard.remaining > 0 + + // Whether this layer still decides anything at all: the counted cut, the continuous order, or + // both. The default layer goes inert once its budget is spent; a continuous one never does. + private def merging: Boolean = dropping || continuous + + // INCREMENTAL bookkeeping, so a decision costs O(log streams) instead of a scan per message: + // the held count (was a sum over every queue per offer), the waited-for streams with no head + // (was a filter over `waiting` per offer), and a min-heap of stream heads (was a minBy over + // every ready stream per drop). Heads only change on enqueue-to-empty and dequeue, so the + // heap is maintained at exactly those points; entries are validated against the live queue + // head on pop, and a mismatch is simply a stale entry to discard. + private var heldNow: Int = 0 + private val headless: mutable.Set[String] = mutable.Set.from(waiting) + private val headsHeap: mutable.PriorityQueue[(MessageOrderKey, String)] = + mutable.PriorityQueue.empty[(MessageOrderKey, String)](Ordering[(MessageOrderKey, String)].reverse) + + // Payload bytes currently held, for the byte watermarks. Maintained on every enqueue/dequeue. + private var heldBytes: Long = 0L + + // CONTINUOUS-MODE bookkeeping (untouched by the default layer). + // + // `tailHeadless` is the SILENT streams: not waited for (their backlog, if any ever existed, + // does not gate the cut) and currently queue-empty. `lastOffered` is the newest selected time + // each has offered. The order-safe fast path emits a head once every silent stream's + // lastOffered is at or past it - nothing older can then follow from a stream that preserves + // append order and roughly monotonic producer time. The min over the set is cached + // (dirty-flagged on membership changes), sound because members' values are frozen: a stream + // only updates lastOffered by offering, and offering removes it from the set. Everything + // else - a stream that is merely slow, quiet forever, or stamped by a skewed producer clock - + // is covered by the RESIDENCE bound on the held head itself, not by any per-stream state. + private val lastOffered: mutable.Map[String, Long] = mutable.Map.empty + private val tailHeadless: mutable.Set[String] = mutable.Set.from(streamIds.toSet -- waiting) + private var tailFloorCacheDirty: Boolean = true + private var tailMinOfferedCache: Long = Long.MinValue + + // The session-wide startup hold: nothing is emitted until one grace has passed since the + // FIRST offer, unless every stream has spoken by then. This is what keeps a multi-topic + // history replay from opening with a scramble while slow streams are still subscribing - + // before a stream's first offer there is no lastOffered to pin the fast path. Irrelevant + // while a counted cut is unresolved - the exact rule holds there. Bounded by construction: + // it can never outlast the first held message's own residence bound. + private val neverOffered: mutable.Set[String] = mutable.Set.from(streamIds) + private var firstOfferMonoMs: Option[Long] = None + + // The lateness ledger: the newest key emitted so far, and how many emissions were LATE + // (ordered before it). Volatile so diagnostics and tests read it without the merge lock. + private var lastEmittedKey: Option[MessageOrderKey] = None + @volatile private var lateEmissions: Long = 0L + + override def lateDeliveryCount: Long = lateEmissions + + /** Every emission - ordered, residence-expired, or a late redelivery - passes through here so + * the lateness ledger cannot miss a path. LATENESS IS BY THE SELECTED TIME, exactly as the + * statistic is advertised: two messages sharing a millisecond are a tie-break, not a "late + * delivery", however the deterministic full key orders them. The newest-emitted key itself + * advances by the full order, so ties keep a stable frontier. */ + private def noteEmitted(key: MessageOrderKey): Boolean = + val late = lastEmittedKey.exists(last => key.orderTime < last.orderTime) + if late then lateEmissions += 1 + if lastEmittedKey.forall(last => Ordering[MessageOrderKey].gt(key, last)) then lastEmittedKey = Some(key) + late + + private def tailMinOffered: Long = + if tailFloorCacheDirty then + tailMinOfferedCache = + if tailHeadless.isEmpty then Long.MaxValue + else tailHeadless.iterator.map(id => lastOffered.getOrElse(id, Long.MinValue)).min + tailFloorCacheDirty = false + tailMinOfferedCache + + /** The ORDER-SAFE fast path: no silent stream at all, or every silent stream has already + * offered something STRICTLY past this head (the cached min - O(1) per message). Strictly: + * producer timestamps are non-decreasing, so a stream that offered through time t may offer + * another AT t - and if that stream's topic sorts first in the documented tie-break, emitting + * a tied head now would be avoidable disorder. An equal-timestamp head waits its residence + * out instead. */ + private def orderSafeToEmit(orderTimeMs: Long): Boolean = + tailHeadless.isEmpty || orderTimeMs < tailMinOffered + + private def startupHoldActive: Boolean = + neverOffered.nonEmpty && firstOfferMonoMs.exists(first => nowMs() - first < graceMs) + + /** The counted cut just resolved and this layer is continuous: from here the two continuous + * rules govern. Whatever was still waited for joins the silent set - its unread backlog, if + * the give-up window would have found any, is the same best-effort fuzziness the residence + * bound carries. */ + private def convertToOrderingOnly(): Unit = + dropping = false + if guaranteed then + // The REPLAY BARRIER carries straight over: streams that reached their recorded ends + // during the cut stay finished, the rest keep gating until they drain - `waiting` and + // `headless` are already exactly that. (The pre-replay design re-armed waiting on + // EVERY stream here, recorded ends ignored - the hold-forever the exact-replay + // redesign removed.) + () + else if waiting.nonEmpty || headless.nonEmpty then + tailHeadless ++= headless + tailFloorCacheDirty = true + waiting = Set.empty + headless.clear() + + // The highest APPEND position (ledger, entry, batchIndex) each stream has offered. The + // duplicate guard: a broker unload redelivers everything un-acked while the originals may + // still sit in `pending` or already be decided, and re-deciding a copy would spend a second + // budget unit on one message. Append position is strictly increasing within a stream - publish + // TIME is not (see [[MessageOrderKey]]) - so "at or below the watermark" is exactly "offered + // before", never a legitimate successor. + private val acceptedThrough: mutable.Map[String, (Long, Long, Int)] = mutable.Map.empty + + // The FULL entry position (batch size included) each stream last offered - what a guaranteed + // boundary EXTENSION compares the freshly captured ends against ([[isPastBacklogEnd]] needs + // the delivered side's batch size when the recorded end names an entry without an index). + // Kept beside `acceptedThrough` rather than widening it: the duplicate guard's ordering + // comparisons stay exactly as they were. + private val lastOfferedPositions: mutable.Map[String, EntryPosition] = mutable.Map.empty + + // What flow control wants paused right now. Maintained on TRANSITIONS (a queue crossing a + // watermark, the totals crossing theirs), never by rescanning every queue per message - at a + // thousand streams the old full recompute dominated the merge's actual work. `totalsOver` + // latches the totals crossing so the expensive all-streams sweep runs once per crossing, not + // once per offer. + private val pausedDesired = mutable.Set.empty[String] + private var totalsOver: Boolean = false + + // Every stream a give-up abandoned, in give-up order: the degradation record the client shows. + private val abandoned = mutable.ArrayBuffer.empty[String] + + override def abandonedStreamIds: Vector[String] = synchronized(abandoned.toVector) + + // When the merge first became unable to advance because a WAITED-FOR stream had no head, and + // whether that has already been surfaced. Both reset the moment the merge can advance again - + // they exist only to bound and name a stream that never speaks (see the stall constants). + private var blindSince: Option[Long] = None + private var stallWarned: Boolean = false + + // Grows by one each time stallWarned newly flips - the sweep's cue to push a progress frame. + // Volatile: written under the merge lock, but the runner's report closure reads it from + // whichever listener thread is sending. + @volatile private var stallWarningsIssued: Long = 0L + + override def stallWarningCount: Long = stallWarningsIssued + + // HOW MANY, not which: read straight off the incrementally-maintained set. This is on the + // report path - once per delivered frame - and materializing the whole set to take its size + // made a 2,000-stream session pay a full pass per frame for a number it already had. + override def stalledStreamCount: Int = synchronized { + if stallWarned then headless.size else 0 + } + + override def stalledStreamIds: Vector[String] = synchronized { + if stallWarned then blindStreams(exclude = "").toVector.sorted else Vector.empty + } + + // One-way: set by settleIfDone once the budget is spent and nothing is held. Volatile because + // inOrder reads it WITHOUT the ordering lock - that read being lock-free is its whole point. + @volatile private var settled: Boolean = false + + override def isSettled: Boolean = settled + + override def settleIfDone(): Unit = synchronized { + // A continuous layer NEVER settles: settling is how the ordering lock is released for + // good, and a continuous mode is precisely the promise to keep ordering. + if !continuous && !dropping && held == 0 then settled = true + } + + override def isContinuousOrdering: Boolean = continuous + + override def resetStallClock(): Unit = synchronized { + blindSince = None + stallWarned = false + // The residence clocks too: a pause holds every source, so time spent paused proves + // nothing about how long a held message has genuinely waited for its peers. Without the + // re-stamp, the first sweep after a long-paused resume found every residence expired and + // dumped the held set past backlogs whose consumers had simply not restarted yet. + if continuous then + val now = nowMs() + pending.values.foreach(queue => queue.mapInPlace { case (key, payload, _) => (key, payload, now) }) + } + + override def progressDiscard: Option[StartFromDiscard] = Some(discard) + + override def heldCount: Int = synchronized(held) + + private def held: Int = heldNow + + /** The streams the merge is still WAITING ON that have delivered no head yet - what it is blocked + * by. Surfaced (alongside [[heldCount]]) so a stall is diagnosable rather than invisible. */ + def waitingOn: Set[String] = synchronized(blindStreams(exclude = "")) + + override def sweepStalled(): Vector[(P, StartFromOutcome)] = synchronized { + if !merging then Vector.empty + else + // No stream is delivering, so nothing is excluded from "silent" - the exclusion in the + // offer path exists only because the offering stream is about to speak. For a + // continuous layer past its cut this is also the TIMER half of the residence bound: an + // all-idle tail emits nothing until a tick notices the grace has passed. + surfaceOrGiveUpOnSilentStreams(currentStreamId = "") + advance() + } + + override def offer( + streamId: String, + key: MessageOrderKey, + atBacklogEnd: Boolean, + payload: P, + knownFailedRetry: Boolean = false, + entryPosition: EntryPosition = EntryPosition.empty + ): Vector[(P, StartFromOutcome)] = + synchronized { + if !merging then Vector(payload -> StartFromOutcome.Deliver) + else + // Name, and eventually give up on, any OTHER stream that has gone silent - otherwise + // a waited-for partition whose backlog was trimmed holds the merge and, at the cap, + // triggers a permanent ~100ms nack storm that nothing surfaces. The current stream is + // excluded: it is about to deliver, so it must never be the one given up on. + surfaceOrGiveUpOnSilentStreams(currentStreamId = streamId) + // The declined message itself returning lifts its stream's floor. + val position = (key.ledgerId, key.entryId, key.batchIndex) + // A message with NO KNOWABLE position can never be a redelivered duplicate + // either - nothing unstored is ever redelivered - so it must skip the watermark + // entirely. NON-PERSISTENT topics are the case that matters, and their ids LIE + // plausibly: every message arrives as ledger 0, entry 0 (measured against a real + // broker - not -1), so the first one recorded a watermark that damned every + // later message as a "copy": swallowed during a cut, shunted down the late-retry + // path (instant, out of order) in continuous mode. The topic's persistency is + // the discriminator; the id-shape check stays as a belt for exotic ids. + val positionKnown = !isNonPersistentTopic(key.topicFqn) && key.ledgerId >= 0 && key.entryId >= 0 + // A COPY at or below the append position this stream has already offered. Only a + // broker redelivery produces one - an unload hands back everything un-acked while + // the originals may still be held here or already decided. What happens next is + // an EXPLICIT lifecycle decision, never an inference from queue membership: + // + // - the listener SAW this exact message's delivery fail downstream + // (`knownFailedRetry`): the copy IS the retry - deliver it, late by + // definition, counted as such; + // - anything else: hand it back (Requeue). The original may be held here, + // queued in the delivery limiter, mid-send, or delivered with its ack still + // in flight - and both eager answers were message-loss paths. Acknowledging + // the copy as a semantic Drop burned the id, so an original whose delivery + // later failed could never be redelivered; delivering it whenever the + // original was merely not-held-any-more could show one message twice. The + // broker redelivers a handed-back copy with backoff until the original's + // fate settles. + if positionKnown && acceptedThrough.get(streamId).exists(seen => Ordering[(Long, Long, Int)].lteq(position, seen)) then + if knownFailedRetry then + // A retry may re-emit the newest key (nothing newer went out meanwhile), + // which the publish-time lateness compare would not see - but a second + // emission of a decided message is a quality event by definition. + lateEmissions += 1 + Vector(payload -> StartFromOutcome.Deliver) + else Vector(payload -> StartFromOutcome.Requeue) + else + // ALWAYS ACCEPTED: an in-order arrival is safe to hold, and refusing it was + // the bug (see [[startFromMergeMaxHeld]] - a declined message's redelivery + // races its own successors). Memory pressure pauses the SOURCE instead, via + // the recompute below. + if positionKnown then acceptedThrough(streamId) = position + // The FULL position (batch size included) is what a guaranteed boundary + // EXTENSION compares the freshly captured ends against: a stream whose last + // offer already reached the new end has no delta and stays finished. + if positionKnown then + lastOfferedPositions(streamId) = + if entryPosition != EntryPosition.empty then entryPosition + else EntryPosition(key.ledgerId, key.entryId, key.batchIndex, 1) + val queue = pending.getOrElseUpdate(streamId, mutable.Queue.empty) + if queue.isEmpty then headsHeap.enqueue(key -> streamId) + // The arrival stamp is the message's own residence clock: it may wait for + // quieter streams, but never past the grace from THIS moment. + queue.enqueue((key, payload, nowMs())) + heldNow += 1 + heldBytes += payloadBytesOf(payload) + headless -= streamId + // Reaching the recorded end FINISHES the stream for the counted cut and for + // the guaranteed REPLAY alike: its recorded range is fully in hand, so the + // barrier must stop waiting for it. (The pre-replay Guaranteed ignored this + // past the cut and waited forever.) + if atBacklogEnd then waiting -= streamId + if continuous then + // Fast-path bookkeeping: this stream has spoken (through this publish + // time), and while it holds a head the heap speaks for it instead. + if firstOfferMonoMs.isEmpty then firstOfferMonoMs = Some(nowMs()) + neverOffered -= streamId + lastOffered(streamId) = key.orderTime max lastOffered.getOrElse(streamId, Long.MinValue) + if tailHeadless.remove(streamId) then tailFloorCacheDirty = true + onStreamGrew(streamId) + advance() + } + + /** Per-stream inspections the silent-stream bookkeeping has made - one per waited-for stream + * every time the set is MATERIALIZED. A test's window onto the sweep's cost: an idle tick with + * nothing new to say must not add to it. */ + private var streamInspections: Long = 0L + + private[session_runner] def silentStreamInspections: Long = synchronized(streamInspections) + + /** The waited-for streams with no head, optionally excluding one (the stream currently + * delivering, which must not be counted as silent). Backed by the incrementally-maintained + * `headless` set - membership means "in `waiting` AND queue empty", by construction. + * + * MATERIALIZING COSTS ONE PASS PER WAITED-FOR STREAM, so callers ask only when they are going + * to use the names. Everything that merely needs HOW MANY reads `headless.size` instead. */ + private def blindStreams(exclude: String): Set[String] = + streamInspections += headless.size + if exclude.isEmpty then headless.toSet else headless.toSet - exclude + + /** Surface, and past the give-up window abandon, any waited-for stream that has delivered no head + * for too long. The merge holds messages (and pauses hot sources) for as long as it waits, so a + * stream that never speaks - a partition trimmed after its end was recorded - would block it + * forever in silence. Warn first (a slow-but-alive stream must not be cut early), then stop + * waiting for it, which lets the next `advance` drain what was held. Runs under the same lock as + * everything else; `nowMs` is injected so the window is testable without real time. */ + private def surfaceOrGiveUpOnSilentStreams(currentStreamId: String): Unit = + // FAST PATH, taken on every single offer of a healthy merge: nothing waited-for is + // headless (or only the stream that is speaking right now is). No set is materialized. + if headless.isEmpty || (headless.size == 1 && headless.contains(currentStreamId)) then + blindSince = None + stallWarned = false + else + val now = nowMs() + if blindSince.isEmpty then blindSince = Some(now) + val elapsed = now - blindSince.getOrElse(now) + val disclosing = !stallWarned && elapsed >= startFromMergeStallWarnMs + val givingUp = elapsed >= stallWindowMs && !guaranteed + // THE SET IS MATERIALIZED ONLY WHEN ITS NAMES ARE ABOUT TO BE USED. Guaranteed never + // gives up and discloses once, so every later tick of its sweep has nothing to say - + // and used to pay a full pass over every waited-for stream to say it. At 2,000 idle + // streams that was four whole-set scans a second, per session, forever. + if disclosing || givingUp then + val silent = blindStreams(exclude = currentStreamId) + if disclosing then + logger.warn( + s"Delivery ordering is waiting on ${stalledStreamsForLog(silent)}, which has delivered nothing for ${elapsed}ms while " + + s"$held message(s) are held." + ) + stallWarned = true + stallWarningsIssued += 1 + if givingUp then + logger.warn( + s"Start-from skip is giving up on ${stalledStreamsForLog(silent)} after ${elapsed}ms with no delivery; treating it as " + + "drained and continuing. The skip count stays exact; which messages were dropped may differ if that stream was only slow." + ) + waiting --= silent + headless --= silent + abandoned ++= silent.toVector.sorted + // In continuous mode the given-up stream is not gone, only demoted: from here + // the floors speak for it, and whatever it delivers late is the same + // best-effort fuzziness the give-up already conceded. + if continuous then + tailHeadless ++= silent + tailFloorCacheDirty = true + blindSince = None + stallWarned = false + + /** Take heads while the smallest one is KNOWN to be the smallest - that is, while every stream + * still being waited for has a head to compare. Answers with what that resolved. + */ + private def advance(): Vector[(P, StartFromOutcome)] = + val out = Vector.newBuilder[(P, StartFromOutcome)] + var going = true + while going do + // A stream that is still waited for but has no head could yet turn out to hold the + // smallest message, so the merge cannot decide ANYTHING until it speaks. Memory is + // bounded by pausing hot sources (see `offer`), never by deciding blind. + // + // GUARANTEED ordering emits through the peek/commit barrier instead of here: the + // head must not leave the merge until its send SUCCEEDED, or a failed send would + // have to choose between losing the message and delivering it late - the two + // compromises this mode exists to refuse. + if guaranteed && !dropping then going = false + else if headless.nonEmpty then going = false + else if continuous && !dropping && startupHoldActive then going = false + else + // Pop heads until one matches its stream's LIVE queue head; anything else is a + // stale leftover from an earlier dequeue, discarded on sight. The (key, stream id) + // tuple breaks a PERFECT tie (the same message reached by two targets), so the + // choice never depends on heap internals or iteration order. + var chosen: Option[(MessageOrderKey, String)] = None + while chosen.isEmpty && headsHeap.nonEmpty do + val candidate = headsHeap.dequeue() + val live = pending.get(candidate._2).exists(queue => queue.nonEmpty && queue.head._1 == candidate._1) + if live then chosen = Some(candidate) + chosen match + case None => going = false + case Some((key, streamId)) => + // The continuous emission gate, cheapest-and-most-decisive checks first: + // 1. residence expiry - the unconditional liveness escape no silent + // stream, slow peer, or future-dated producer clock can hold shut; + // 2. IRREVERSIBLY LATE - older by the selected time than something already + // emitted: waiting cannot un-late it, so it goes out now and is + // counted, exactly as the contract says; + // 3. only then the order-safe floor, whose cached min may need an + // O(silent) rebuild - never paid for a head the first two rules + // already release. + val headArrivalMs = pending(streamId).head._3 + if continuous && !dropping + && nowMs() - headArrivalMs < graceMs + && !lastEmittedKey.exists(last => key.orderTime < last.orderTime) + && !orderSafeToEmit(key.orderTime) + then + // The head waits. Its heap entry was popped by the selection above + // and goes straight back - nothing else may emit past it meanwhile. + headsHeap.enqueue(key -> streamId) + going = false + else + val queue = pending(streamId) + // `claim` is keyed by topic FQN, but the constructor `require`d a + // SHARED counter, which ignores that key - so handing it a stream id + // (not a topic FQN) is sound. + val drop = dropping && discard.claim(streamId) + if dropping && !drop then + // The budget is spent: nothing is dropped from here on. The + // default layer releases everything in the order the merge would + // have produced it and goes inert; a continuous layer switches to + // the continuous rules instead - the popped head goes back to be + // judged on the next pass. + if continuous then + convertToOrderingOnly() + headsHeap.enqueue(key -> streamId) + else + dropping = false + out ++= drainInOrder().map(_ -> StartFromOutcome.Deliver) + going = false + else + val taken = queue.dequeue()._2 + heldNow -= 1 + heldBytes = (heldBytes - payloadBytesOf(taken)) max 0L + if queue.nonEmpty then headsHeap.enqueue(queue.head._1 -> streamId) + else if waiting.contains(streamId) then headless += streamId + else if continuous then + tailHeadless += streamId + tailFloorCacheDirty = true + onStreamShrank(streamId, queue.size) + // The verdict is computed here anyway for the lateness ledger; + // riding it on the OUTCOME is what lets the row carry the marker + // (Message.delivered_out_of_order) at zero extra cost. + val emittedLate = continuous && !drop && noteEmitted(key) + out += (taken -> ( + if drop then StartFromOutcome.Drop + else if emittedLate then StartFromOutcome.DeliverOutOfOrder + else StartFromOutcome.Deliver + )) + // The budget can hit zero on THIS claim. Waiting for one more head + // just to have the NEXT claim answer "no" held the boundary batch + // hostage: when this drop emptied a still-waited stream, everything + // already held stayed invisible until that stream spoke again or + // the stall window gave up on it - for an answer that was already + // fully decided. + if drop && discard.remaining <= 0 then + if continuous then convertToOrderingOnly() + else + dropping = false + out ++= drainInOrder().map(_ -> StartFromOutcome.Deliver) + going = false + out.result() + + private def drainInOrder(): Vector[P] = + val all = pending.toVector.flatMap((streamId, queue) => queue.map { case (key, payload, _) => (key, streamId, payload) }) + pending.values.foreach(_.clear()) + heldNow = 0 + heldBytes = 0L + headsHeap.clear() + headless.clear() + headless ++= waiting + pausedDesired.clear() + totalsOver = false + all.sortBy((key, streamId, _) => (key, streamId)).map((_, _, payload) => payload) + + /** Flow-control transitions, the event-driven replacement for a per-message full rescan. + * + * A stream is marked for pause when its OWN queue crosses `pauseStreamAt`, or - while the + * TOTALS (messages or bytes) are over their high watermark - whenever it holds anything at + * all, since any queued stream may be the next to grow. It is unmarked only once its queue + * is back under `resumeStreamAt` with the totals under their low watermarks (the gap is the + * hysteresis), and always the moment its queue empties - a BLIND stream is never held, that + * would deadlock the merge. The totals crossing DOWN prunes the whole set once; crossing UP + * marks every queued stream once. Once the budget is spent the drain clears the set, and + * the caller's reconcile releases everything it paused. + */ + private def totalsOverHigh: Boolean = heldNow >= maxHeld || heldBytes >= pauseBytesAt + + private def totalsUnderLow: Boolean = heldNow <= (maxHeld * 4 / 5) && heldBytes <= resumeBytesAt + + private def onStreamGrew(streamId: String): Unit = + if totalsOver then pausedDesired += streamId + else + if pending.get(streamId).exists(_.size >= pauseStreamAt) then pausedDesired += streamId + if totalsOverHigh then + totalsOver = true + pending.foreach((id, queue) => if queue.nonEmpty then pausedDesired += id) + + private def onStreamShrank(streamId: String, sizeAfter: Int): Unit = + if totalsOver && totalsUnderLow then + totalsOver = false + pausedDesired.filterInPlace(id => pending.get(id).exists(_.size > resumeStreamAt)) + if sizeAfter == 0 then pausedDesired -= streamId + else if !totalsOver && sizeAfter <= resumeStreamAt then pausedDesired -= streamId + + override def desiredPausedStreams: Set[String] = synchronized(pausedDesired.toSet) + + /** Held payload bytes right now - the byte half of the memory profile, exposed for tests. */ + def heldBytesCount: Long = synchronized(heldBytes) + + // GUARANTEED-MODE delivery barrier: the head handed out by [[peekGuaranteed]] and not yet + // committed. The merge does not advance past it; a failed send retries the SAME head in + // place on the next tick - which is what keeps even retries in order. + private var guaranteedInFlight: Option[(MessageOrderKey, String)] = None + + override def peekGuaranteed(): Option[P] = synchronized { + if !guaranteed || dropping then None + else if headless.nonEmpty then None + else + guaranteedInFlight match + case Some((key, streamId)) => + // The retry path: hand the same head out again, validated against the queue. + // A mismatch means the in-flight record is STALE. Nothing reachable mutates a + // queue head between peek and commit/abort today (offers append at the tail, + // resetStallClock preserves keys, drainInOrder is unreachable for continuous + // policies), so this is defence in depth - but left in place a stale record + // made every later peek retry a head that no longer exists, and the merge + // never delivered again. Clear it; the next peek reselects. + val head = pending.get(streamId).flatMap(_.headOption).collect { case (k, payload, _) if k == key => payload } + if head.isEmpty then guaranteedInFlight = None + head + case None => + var chosen: Option[(MessageOrderKey, String)] = None + while chosen.isEmpty && headsHeap.nonEmpty do + val candidate = headsHeap.dequeue() + val live = pending.get(candidate._2).exists(q => q.nonEmpty && q.head._1 == candidate._1) + if live then chosen = Some(candidate) + // The live entry goes back: a peek must not consume heap state - only a + // commit advances anything. + chosen.foreach(c => headsHeap.enqueue(c)) + guaranteedInFlight = chosen + chosen.flatMap((_, streamId) => pending(streamId).headOption.map(_._2)) + } + + override def commitGuaranteed(): Unit = synchronized { + guaranteedInFlight.foreach { (key, streamId) => + pending.get(streamId).foreach { queue => + if queue.nonEmpty && queue.head._1 == key then + val taken = queue.dequeue()._2 + heldNow -= 1 + heldBytes = (heldBytes - payloadBytesOf(taken)) max 0L + if queue.nonEmpty then headsHeap.enqueue(queue.head._1 -> streamId) + // A stream still replaying re-enters the barrier's waited set; a FINISHED one + // joins the silent-floor bookkeeping instead, so a later relax to best effort + // starts with sound floors. (Pre-replay, every stream was waited here.) + else if waiting.contains(streamId) then headless += streamId + else + tailHeadless += streamId + tailFloorCacheDirty = true + onStreamShrank(streamId, queue.size) + // The SEAM LEDGER, counted where the emission is decided: a key lower than + // one already emitted is on screen out of order - loudly. Same tie rule as + // the lateness ledger: an equal selected time is a tie-break, not disorder. + if lastEmittedKey.exists(last => key.orderTime < last.orderTime) then seamViolations += 1 + val _ = noteEmitted(key) + } + } + guaranteedInFlight = None + } + + override def abortGuaranteed(): Unit = synchronized { guaranteedInFlight = None } + + // The seam-violation ledger for the guaranteed replay: emissions whose selected key undercut + // one already emitted. Volatile for the same reason lateEmissions is - the report path reads + // it from whichever thread is sending. + @volatile private var seamViolations: Long = 0L + + override def replaySeamViolationCount: Long = seamViolations + + override def peekIsSeamViolation: Boolean = synchronized { + guaranteedInFlight.exists((key, _) => lastEmittedKey.exists(last => key.orderTime < last.orderTime)) + } + + override def isReplayCaughtUp: Boolean = synchronized { + // The chunk is complete exactly when nobody is expected to speak again AND nothing is + // still in hand: every recorded end was reached (or proven unreachable-because-drained), + // and the barrier emptied the heap. `dropping` deliberately does not gate this - a skip + // budget larger than the recorded history spends what history there is and the replay is + // then complete with nothing shown; the leftover budget continues into the next chunk. + guaranteed && waiting.isEmpty && heldNow == 0 + } + + override def noteStreamReplayFinished(streamId: String): Unit = synchronized { + if guaranteed && waiting.contains(streamId) then + waiting -= streamId + if headless.remove(streamId) then + tailHeadless += streamId + tailFloorCacheDirty = true + } + + override def extendReplayBoundary(ends: Map[String, EntryPosition]): Unit = synchronized { + if guaranteed then + ends.foreach { (streamId, end) => + // Finished under the NEW boundary: nothing was recorded for it, or its last + // offer already reached the new end (everything up to it is in hand or emitted). + // Anything else has a delta to replay and re-enters the barrier. + val finished = + end == EntryPosition.empty + || lastOfferedPositions.get(streamId).exists(offered => isPastBacklogEnd(offered, end)) + if finished then + if waiting.contains(streamId) then + waiting -= streamId + if headless.remove(streamId) then + tailHeadless += streamId + tailFloorCacheDirty = true + else + waiting += streamId + if tailHeadless.remove(streamId) then tailFloorCacheDirty = true + if pending.get(streamId).forall(_.isEmpty) then headless += streamId + } + // A fresh chunk gets a fresh silence clock: the wait so far proves nothing about the + // streams this extension just re-armed. + blindSince = None + stallWarned = false + } + + /** SWITCH A LIVE GUARANTEED LAYER TO BEST EFFORT, and answer with everything that released - + * in the new order, handled exactly as an offer's resolutions are. + * + * WHY THIS EXISTS. The replay waits only on streams still inside their recorded range - but + * a range that can no longer be delivered (trimmed by retention, or a start position seeked + * past the end) holds delivery indefinitely: the barrier cannot tell "nothing left" from + * "nothing yet" there. That wait is disclosed; this is what makes it ACTIONABLE + * without throwing the session away. Everything already received stays received: the held + * messages have not been acknowledged and exist nowhere else, so re-creating the session + * would lose them, re-resolve the start-from against a log that has moved, and redeliver + * what is already on screen. + * + * WHAT CHANGES, AND WHAT DELIBERATELY DOES NOT: + * + * - the POLICY drops one step (GuaranteedOnly -> BestEffortOnly, ExactCutThenGuaranteed -> + * ExactCutThenBestEffort), so from here the two continuous rules govern: the order-safe + * fast path, and the residence bound that is the liveness escape guaranteed refused to + * have. Messages already held AND messages still arriving are both covered - there is one + * rule set and it is now the best-effort one; + * - every stream the guaranteed rule was blind on becomes a SILENT stream: its floor speaks + * for it, and nothing waits on it forever any more. Held heads whose residence has + * already run out - which is every head of a session that stalled long enough for anyone + * to notice - are released by the `advance` below, in merge order, exactly once. A head + * that arrived a moment ago waits out its own residence like any other and is released by + * the continuous sweep, which is already armed; + * - an UNRESOLVED COUNTED CUT is untouched. While `dropping` the exact rule still holds and + * the waited set must stay exactly as it was, or the cut would select different messages; + * the budget is neither spent, refunded nor skipped by the switch. Only the guaranteed + * ADDITIONS - waiting on everyone forever, and never giving up - are what relax; + * - acknowledgment identity, the duplicate watermarks, the flow-control marks and the byte + * and count accounting are all the same objects, carried straight over. What the drain + * empties, the caller's reconcile then un-pauses. + * + * ONE WAY ONLY. There is no promotion back: Guaranteed is a claim about what the user has + * ALREADY been shown, and a session that has emitted past a silent stream cannot un-emit it - + * see [[deliveryOrderSwitchFor]], which refuses the other directions rather than pretending. + * + * CONCURRENCY. Under this object's monitor like every other merge decision, and the caller + * holds the session's ordering lock - the same nesting an offer, a sweep and the guaranteed + * pump all use, so no new lock ordering is introduced. A guaranteed peek cannot be + * outstanding across it (peek and its commit/abort live inside one ordering-lock block), and + * the in-flight slot is cleared anyway so a stale commit can never pop a head this switch has + * already released. + */ + override def relaxGuaranteedToBestEffort(): Vector[(P, StartFromOutcome)] = synchronized { + if !guaranteed then Vector.empty + else + // Exactly one step down, keeping the cut phase where there is one. GuaranteedOnly is + // only ever built for the Ordered plan, whose budget is a shared ZERO - which is + // precisely what BestEffortOnly requires of itself at construction. + activePolicy = activePolicy match + case OrderingPolicy.ExactCutThenGuaranteed => OrderingPolicy.ExactCutThenBestEffort + case _ => OrderingPolicy.BestEffortOnly + // Nothing left the merge under the barrier, so there is nothing to hand over - and a + // commit arriving late must not pop a head the release below is about to emit. + guaranteedInFlight = None + // The waited set relaxes only once the counted cut is done with it. While `dropping`, + // "every waited stream must have a head" is what keeps the cut exact. + if !dropping then + tailHeadless ++= headless + tailFloorCacheDirty = true + waiting = Set.empty + headless.clear() + // The stall clock measured a wait that no longer exists. + blindSince = None + stallWarned = false + advance() + } + +/** One physical topic's delivery stream, as a global start-from layer knows it. + * + * `lastAtStart` is where that topic ENDED when the session was created; it is what tells the layer + * that the stream has run out of pre-existing messages and must stop being waited for. + */ +final case class StartFromStream(id: String, lastAtStart: EntryPosition) + +/** The identity of one delivery stream. + * + * NOT the topic alone: two enabled targets may select the SAME topic, and each has its own consumer + * delivering it independently. Merging both under one key would let one target's head hide the + * other's. + */ +def startFromStreamId(consumerName: String, topicFqn: String): String = s"$consumerName@$topicFqn" + +/** Where every consumer's topic ended when the session started. + * + * ONLY "SKIP FIRST N" ASKS FOR THIS, and only on a multi-stream session: it is what lets the merge + * stop waiting for a partition that has run out of pre-existing messages. "Latest n" used to need + * it too, to know when its heap could be released; it resolves its position from entry metadata + * now, so it costs one broker call per topic less than it did. + * + * `getLastMessageIds` rather than the deprecated singular form: it answers with one id per topic + * behind the consumer, which is the shape that stays right if a consumer is ever built over more + * than one. The LAST of them is taken, so a consumer covering several topics is not declared + * drained before all of them are. + * + * A NON-PERSISTENT topic is recorded as [[EntryPosition.empty]] - "already drained" - without + * being asked: it retains nothing, so it has no end to reach and must never be one the merge waits + * for. That is decided from the FQN, exactly as [[startFromNeedsRetainedHistory]] decides its own + * case, and NOT from whatever the call happens to throw. + * + * A PERSISTENT topic whose end cannot be read FAILS the session. It used to be recorded as empty + * too, which reads as "already drained": the merge then stopped waiting for that whole partition, + * so a global skip silently left it out of the count and out of the order while it went on + * delivering messages. An empty partition needs no special case here - it ANSWERS, with + * `MessageId.earliest`, which maps to [[EntryPosition.empty]] on its own. + */ +def startFromStreamsAt(consumers: Vector[Consumer[Array[Byte]]]): Vector[StartFromStream] = + consumers.map { consumer => + val topicFqn = consumer.getTopic + val lastAtStart = + if isNonPersistentTopic(topicFqn) then EntryPosition.empty + else + Try(consumer.getLastMessageIds.asScala.toVector) match + case Success(messageIds) => + messageIds + .map(EntryPosition.of) + .maxOption(Ordering.by[EntryPosition, (Long, Long, Int)](p => (p.ledgerId, p.entryId, p.batchIndex))) + .getOrElse(EntryPosition.empty) + case Failure(err) => + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: reading where the topic ends failed for $topicFqn. ${err.getMessage}", + err + ) + StartFromStream(startFromStreamId(consumer.getConsumerName, topicFqn), lastAtStart) + } + +/** Which global reordering a start-from needs on top of its seek, if any. */ +enum StartFromOrderingPlan: + /** The seek (and whatever the head-drop counters correct) already lands exactly - every mode + * except the two counting ones, and both of those on a session with a single stream, where the + * one log is already in publish order. */ + case PassThrough + + /** Drop the globally-first `n` across `streams`: a [[GlobalSkipMerge]]. `afterCut` says + * what the same layer does once the cut resolves: go inert (AsReceived), keep ordering + * best-effort, or keep ordering GUARANTEED. */ + case GlobalSkip( + n: Long, + streams: Vector[StartFromStream], + afterCut: consumer.session_config.MessageDeliveryOrder = consumer.session_config.MessageDeliveryOrder.AsReceived + ) + + /** No cut to make - a delivery-ordering layer alone, for the non-counted modes. For BEST + * EFFORT the streams carry EMPTY recorded ends, deliberately: the continuous rules need no + * ends, so that plan still costs no broker call per topic. For GUARANTEED the ends ARE the + * replay boundary - captured at session build (and re-captured by every Resume), one broker + * call per topic, which is the price of an exact replay. */ + case Ordered(streams: Vector[StartFromStream], ordering: consumer.session_config.MessageDeliveryOrder) + +/** Whether this session needs a GLOBAL reordering layer on top of its seek. + * + * ONLY "SKIP FIRST N", and only on a session with more than one delivery stream. One log is + * already in append order, so the head-drop counter is exact on its own and nothing is ever held - + * the ordinary case, a single non-partitioned topic, must stay free. + * + * "LATEST N" DELIBERATELY NEEDS NONE, at any number of streams. It used to get a bounded top-n + * heap that buffered n full message payloads and narrowed the over-fetch after delivery; its cut + * is now resolved from entry METADATA before anything is delivered (`resolveLatestN`), so every + * consumer simply starts in the right place and streams. That removed the only start-from path + * whose memory grew with a number the user typed. + * + * Every other mode reaches its position with the seek itself. Both approximate modes stay per topic + * deliberately - [[consumer.start_from.ApproximateEntryPosition]] per partition and + * [[consumer.start_from.ApproximatePublishTimePosition]] per logical topic: a count of n can be reached by + * streaming n messages and stopping, whereas a fraction of the merged stream is only known once the + * whole of it has been measured. + * + * PURE, so both the fast path and the modes that must not acquire one are pinned by test. + */ +def needsGlobalOrdering(startFrom: ConsumerSessionStartFrom, streamCount: Int): Boolean = startFrom match + case v: NthMessageAfterEarliest => v.n > 0 && streamCount > 1 + case _ => false + +/** WHICH layer this session gets - a streaming merge for "skip first n", and nothing for everything + * else, "latest n" included. + * + * The single place the mapping is made, so `handleStartFrom` cannot disagree with it, and PURE, so + * it is pinned without a broker. Resolving `streams` costs a broker call per topic, which is why + * [[needsGlobalOrdering]] stays separate: it gates that cost before the streams are asked for. + */ +def globalOrderingPlanFor( + startFrom: ConsumerSessionStartFrom, + streams: Vector[StartFromStream], + afterCut: consumer.session_config.MessageDeliveryOrder = consumer.session_config.MessageDeliveryOrder.AsReceived +): StartFromOrderingPlan = + if !needsGlobalOrdering(startFrom, streams.size) then StartFromOrderingPlan.PassThrough + else + startFrom match + case v: NthMessageAfterEarliest => StartFromOrderingPlan.GlobalSkip(v.n, streams, afterCut) + case _ => StartFromOrderingPlan.PassThrough + +/** Everything a start-from needs after its seek: what to drop off the head of each stream, and how + * to reorder what is left into the global order the counting modes are defined over. + */ +final case class StartFromPlan(discard: StartFromDiscardPlan, ordering: StartFromOrderingPlan) + +/** A delivered message the ordering layer is holding, with everything needed to resolve it later - + * possibly from a DIFFERENT topic's listener thread, since the merge releases whatever the newest + * head unblocked, not the message just offered. + */ +final case class HeldMessage( + consumer: Consumer[Array[Byte]], + message: PulsarMessage[Array[Byte]], + listener: ConsumerListener, + /** Set by the merge AT EMISSION when this delivery's order key undercuts one already emitted + * (Best effort's late emission - the message outlived its reorder window). Rides to + * [[ConsumerListener.deliverNow]], which stamps it onto the delivery's own call stack. + * Guaranteed's equivalent travels through the peek/commit pair instead. */ + deliveredOutOfOrder: Boolean = false +) + +/** The session-wide ordering layer the listeners hand every delivered message to. + * + * Armed ONCE, by `ConsumerSessionRunner.make`, and shared by every target of the session: the two + * counting modes are defined over the session's whole merged stream, so a per-target layer would + * count each target separately. + * + * Generic in what it holds so the WHOLE routing - stream identity, sort key, end of backlog - can + * be driven with plain values and real Pulsar message ids, and no broker. Production instantiates + * it with [[HeldMessage]]. + */ +final class StartFromOrdering[P] private[session_runner] ( + // Package-visible rather than class-private so a test can wire a merge with an INJECTED + // clock through the real listener/runner plumbing; production construction still goes + // through [[StartFromOrdering.make]] only. + private val merge: Option[StartFromMerge[P]], + initialStreams: Map[String, StartFromStream] +): + /** The recorded ends every offer is judged against - THE REPLAY BOUNDARY under Guaranteed. + * A var because a Resume EXTENDS it ([[extendReplayBoundary]]); written under [[inOrder]] + * like every boundary decision, volatile so the indicator-refinement task (maintenance + * thread) reads a published snapshot. */ + @volatile private var streams: Map[String, StartFromStream] = initialStreams + /** The lock that keeps the merge's DECISION and the session's ACTIONS in the same order. + * + * `offer` is itself synchronized, but that is not enough: what it answers with is a batch of + * messages that were resolved by THIS offer and now have to be handled - possibly several of + * them, possibly belonging to other topics. Releasing after `offer` let another listener thread + * resolve a later batch and process it first, so the session's stateful filters, coloring rules + * and value projections saw messages in a different order than the merge had just decided on. + */ + private val orderingLock = new Object + + /** Run `use` with this session's ordering decisions serialized. + * + * A NO-OP when nothing is being reordered. The ordinary session - one non-partitioned topic, + * or any mode that reaches its position with the seek alone - decided no order, so a + * session-wide lock per message would only serialize its partitions' deserialization for + * nothing. That fast path must stay free. + */ + def inOrder[A](use: => A): A = merge match + case None => use + // Settled is ONE-WAY: the budget is spent and nothing is held, so the merge can never + // reorder anything again - from here the lock would only serialize every partition's + // deserialization and JS evaluation for the rest of the session's life, for nothing. + case Some(layer) => if layer.isSettled then use else orderingLock.synchronized(use) + + /** Whether [[inOrder]] currently takes the session-wide ordering lock: a merge exists and has + * not settled. The delivery router reads this to keep BLOCKING work out of that lock - a + * Deliver resolved under it is enqueued to the delivery limiter's FIFO queue (drained on the + * session's timer thread) instead of running the JS lease and the gRPC write inline on the + * offering listener thread. Order is preserved by construction: the enqueue happens under + * the same lock that decided the order, the queue is FIFO with a single drainer, and the + * limiter refuses inline processing whenever a backlog or an in-flight drain exists. */ + def serializesDeliveries: Boolean = merge.exists(layer => !layer.isSettled) + + /** See [[StartFromMerge.settleIfDone]]. The listener calls this after a resolved batch is + * fully handled, still under [[inOrder]]. */ + def settleIfDone(): Unit = merge.foreach(_.settleIfDone()) + + /** See [[StartFromMerge.resetStallClock]]. The runner calls this on session RESUME. */ + def resetStallClock(): Unit = merge.foreach(_.resetStallClock()) + + /** See [[StartFromMerge.abandonedStreamIds]] - the degradation record for the progress API. */ + def abandonedStreams: Vector[String] = merge.map(_.abandonedStreamIds).getOrElse(Vector.empty) + + /** See [[StartFromMerge.stallWarningCount]] - the sweep's cue to disclose a stall. */ + def stallWarningCount: Long = merge.map(_.stallWarningCount).getOrElse(0L) + + /** See [[StartFromMerge.stalledStreamCount]] - a transient user-visible wait. */ + def stalledStreamCount: Int = merge.map(_.stalledStreamCount).getOrElse(0) + + /** See [[StartFromMerge.stalledStreamIds]] - the same wait, by name, for the debug log. */ + def stalledStreamIds: Vector[String] = merge.map(_.stalledStreamIds).getOrElse(Vector.empty) + + /** See [[StartFromMerge.isContinuousOrdering]] - the runner keeps the sweep armed for the + * session's life (and at the grace cadence) when this is set. */ + def isContinuousOrdering: Boolean = merge.exists(_.isContinuousOrdering) + + /** See [[StartFromMerge.lateDeliveryCount]] - the best-effort order's quality gauge. */ + def lateDeliveryCount: Long = merge.map(_.lateDeliveryCount).getOrElse(0L) + + /** See [[StartFromMerge.isGuaranteedOrdering]]. */ + def isGuaranteedOrdering: Boolean = merge.exists(_.isGuaranteedOrdering) + + /** See [[StartFromMerge.peekGuaranteed]]. Callers hold [[inOrder]]. */ + def peekGuaranteed(): Option[P] = merge.flatMap(_.peekGuaranteed()) + + /** See [[StartFromMerge.commitGuaranteed]]. Callers hold [[inOrder]]. */ + def commitGuaranteed(): Unit = merge.foreach(_.commitGuaranteed()) + + /** See [[StartFromMerge.abortGuaranteed]]. Callers hold [[inOrder]]. */ + def abortGuaranteed(): Unit = merge.foreach(_.abortGuaranteed()) + + /** See [[GlobalSkipMerge.relaxGuaranteedToBestEffort]]. Callers hold [[inOrder]], like every + * other decision this layer makes. A pass-through layer orders nothing and answers with + * nothing to release. */ + def relaxGuaranteedToBestEffort(): Vector[(P, StartFromOutcome)] = + merge.map(_.relaxGuaranteedToBestEffort()).getOrElse(Vector.empty) + + // FLOW CONTROL: how the merge's desired-paused set becomes consumer.pause()/resume() calls. + // Hooks are registered lazily by each listener on a stream's first delivery (a stream that + // never delivers has nothing to pause). + // + // GUARDED BY ITS OWN LOCK, deliberately not by [[inOrder]]: once the merge settles, inOrder + // stops taking the ordering lock (that is the whole point of settling) - but a topic that was + // quiet through the skip still registers its hooks on its first POST-cut delivery, and two + // listener threads doing that concurrently would race an unsynchronized map. The lock is + // uncontended and the critical sections are tiny, so this costs nothing measurable. + private val flowControlLock = new Object + // Each hook answers whether the CLIENT CALL actually took: the reconcile below only records + // transitions that succeeded, so a failed pause or resume is retried on the next reconcile + // (every handled batch, and every sweep tick) instead of being remembered as done. + private val streamPauseHooks = mutable.Map.empty[String, (() => Boolean, () => Boolean)] + private var pausedApplied: Set[String] = Set.empty + + /** Register how to pause and resume one stream's consumer; each hook answers whether the + * client call succeeded. Idempotent; first registration wins. A no-op for a pass-through + * layer, which never pauses anybody. */ + def registerStreamPauseHooks(streamId: String, pause: () => Boolean, resume: () => Boolean): Unit = + if merge.isDefined then + flowControlLock.synchronized { + if !streamPauseHooks.contains(streamId) then streamPauseHooks(streamId) = (pause, resume) + } + + /** Apply the merge's flow-control wishes: pause what it newly wants held still, resume what + * it no longer does - TRANSITIONS ONLY. The hooks land on per-consumer pause ARBITERS + * (reason Merge), so nothing outside this layer can stomp a hold or be stomped by a + * release: a user resume or the delivery pacer letting go releases only its own reason. + * That is what makes the diff safe again - the old re-assert-everything loop existed to + * heal external stomps that the arbiter now makes impossible, and it cost a pause call per + * desired stream per handled batch. Once the merge settles the desired set is empty + * forever, so the settling batch's reconcile releases everything and later calls no-op. + */ + def reconcileFlowControl(): Unit = merge.foreach { layer => + val desired = layer.desiredPausedStreams + flowControlLock.synchronized { + val toPause = (desired -- pausedApplied).filter(streamPauseHooks.contains) + val toResume = pausedApplied -- desired + // SUCCESS-ONLY bookkeeping: a transition is recorded as applied exactly when the + // client call took. A pause that threw stays un-applied and is retried on the next + // reconcile; a resume that threw stays applied so the release is retried too - + // recording either unconditionally turned one transient client error into a stream + // paused (or filling memory) forever, with the books saying everything was fine. + val paused = toPause.filter(streamId => streamPauseHooks(streamId)._1()) + val resumed = toResume.filter(streamId => streamPauseHooks.get(streamId).forall(_._2())) + pausedApplied = pausedApplied ++ paused -- resumed + } + } + + /** The counter start-from progress is read off while this layer is doing the counting. */ + def progressDiscard: Option[StartFromDiscard] = merge.flatMap(_.progressDiscard) + + /** Messages held right now - the memory profile, exposed for tests and diagnostics. */ + def heldCount: Int = merge.map(_.heldCount).getOrElse(0) + + /** See [[StartFromMerge.sweepStalled]]. Callers hold [[inOrder]], like every offer. */ + def sweepStalled(): Vector[(P, StartFromOutcome)] = merge.map(_.sweepStalled()).getOrElse(Vector.empty) + + /** Take one delivered message, and answer with everything that offer RESOLVED - which is often + * not the message just offered, and may be none at all. + */ + def offer( + consumerName: String, + topicFqn: String, + orderTime: Long, + messageId: PulsarMessageId, + payload: P, + knownFailedRetry: Boolean = false + ): Vector[(P, StartFromOutcome)] = merge match + case None => Vector(payload -> StartFromOutcome.Deliver) + case Some(layer) => + val streamId = startFromStreamId(consumerName, topicFqn) + val position = EntryPosition.of(messageId) + val stream = streams.get(streamId) + // THE REPLAY BOUNDARY, guaranteed ordering only: a message recorded STRICTLY past + // this stream's captured end belongs to the NEXT chunk. It bypasses the merge + // entirely - no order key, no duplicate watermark, no budget - so its post-Resume + // redelivery arrives as an ordinary first-time offer; its arrival is counted for + // the approximate newer-messages indicator and finishes the stream's replay + // (append order: nothing recorded can follow it). Unaddressable ids + // ([[EntryPosition.empty]]) never trip this - gt against (-1,-1) is false. + if layer.isGuaranteedOrdering && stream.exists(s => isStrictlyPastBacklogEnd(position, s.lastAtStart)) then + notePastBoundaryEntry(streamId, position) + layer.noteStreamReplayFinished(streamId) + Vector(payload -> StartFromOutcome.NextChunk) + else + // An unknown stream is treated as already drained, for the same reason an + // unanswerable one is: the merge must never wait on something it knows nothing + // about. + val atBacklogEnd = stream.forall(s => isPastBacklogEnd(position, s.lastAtStart)) + layer.offer(streamId, MessageOrderKey.of(orderTime, topicFqn, position), atBacklogEnd, payload, knownFailedRetry, position) + + // Distinct past-boundary ENTRIES seen so far, per stream the newest one - the locally-known + // lower bound behind the "~N newer" indicator on the caught-up signal (a broker refinement + // may raise it later). Approximate by design: broker redeliveries of a handed-back message + // arrive again and must not double-count, so only an entry NEWER than the stream's newest + // seen is counted - an out-of-order redelivery burst undercounts, which the "~" absorbs. + // Written under [[inOrder]] (the offer path); the counter is volatile for the report path. + private val pastBoundarySeen = mutable.Map.empty[String, (Long, Long)] + @volatile private var pastBoundaryEntries: Long = 0L + + private def notePastBoundaryEntry(streamId: String, position: EntryPosition): Unit = + val entry = (position.ledgerId, position.entryId) + if pastBoundarySeen.get(streamId).forall(seen => Ordering[(Long, Long)].gt(entry, seen)) then + pastBoundarySeen(streamId) = entry + pastBoundaryEntries += 1 + + /** Past-boundary entries observed since the current boundary was captured - see above. */ + def replayNewerEntriesSeen: Long = pastBoundaryEntries + + /** The recorded end each stream's replay is bounded by RIGHT NOW - the refinement task + * compares the broker's current ends against these. A snapshot; safe off-thread. */ + def replayBoundaryEnds: Map[String, EntryPosition] = streams.view.mapValues(_.lastAtStart).toMap + + /** EXTEND the replay boundary to freshly captured ends - what a Resume does for a Guaranteed + * session. Callers hold [[inOrder]], like every other boundary decision: serializing this + * against the offer path is exactly what makes "a past-end nack racing the re-capture" + * converge (either the offer still sees the old boundary and its nack is released by the + * resume right after, or it already sees the new one and delivers normally). */ + def extendReplayBoundary(extended: Vector[StartFromStream]): Unit = + streams = streams ++ extended.map(stream => stream.id -> stream) + pastBoundarySeen.clear() + pastBoundaryEntries = 0L + merge.foreach(_.extendReplayBoundary(extended.map(stream => stream.id -> stream.lastAtStart).toMap)) + + /** See [[StartFromMerge.isReplayCaughtUp]]. Callers hold [[inOrder]]. */ + def isReplayCaughtUp: Boolean = merge.exists(_.isReplayCaughtUp) + + /** See [[StartFromMerge.peekIsSeamViolation]]. Callers hold [[inOrder]], between a peek and + * its commit - the window in which the answer cannot move. */ + def peekIsSeamViolation: Boolean = merge.exists(_.peekIsSeamViolation) + + /** See [[StartFromMerge.replaySeamViolationCount]] - the session counter on ConsumerStats. */ + def replaySeamViolationCount: Long = merge.map(_.replaySeamViolationCount).getOrElse(0L) + +object StartFromOrdering: + /** Reorders nothing, and holds nothing. */ + def passThrough[P]: StartFromOrdering[P] = new StartFromOrdering[P](None, Map.empty) + + def make[P](plan: StartFromOrderingPlan, payloadBytesOf: P => Long = (_: P) => 0L): StartFromOrdering[P] = plan match + case StartFromOrderingPlan.PassThrough => passThrough[P] + case StartFromOrderingPlan.GlobalSkip(n, streams, afterCut) => + new StartFromOrdering[P]( + Some( + GlobalSkipMerge[P]( + streams.map(_.id), + drainedAtStart(streams), + StartFromDiscard.shared(n), + payloadBytesOf = payloadBytesOf, + policy = afterCut match + case consumer.session_config.MessageDeliveryOrder.AsReceived => OrderingPolicy.ExactCutOnly + case consumer.session_config.MessageDeliveryOrder.BestEffort => OrderingPolicy.ExactCutThenBestEffort + case consumer.session_config.MessageDeliveryOrder.Guaranteed => OrderingPolicy.ExactCutThenGuaranteed + ) + ), + byId(streams) + ) + case StartFromOrderingPlan.Ordered(streams, ordering) => + new StartFromOrdering[P]( + Some( + GlobalSkipMerge[P]( + streams.map(_.id), + // Nothing recorded at the boundary means nothing to replay: such streams + // are never waited for. Vacuous for best effort (its streams all carry + // empty ends AND its waited set is empty by policy); load-bearing for the + // guaranteed replay - Latest and the single non-persistent stream land + // here with every end empty, which is the instant caught-up. + drainedAtStart = drainedAtStart(streams), + // A shared zero budget: nothing to drop (and no skip progress - toSkip + // is 0), which is what both ordering-only policies require. + StartFromDiscard.shared(0), + payloadBytesOf = payloadBytesOf, + policy = ordering match + case consumer.session_config.MessageDeliveryOrder.Guaranteed => OrderingPolicy.GuaranteedOnly + case _ => OrderingPolicy.BestEffortOnly + ) + ), + // The recorded ends gate ONLY the guaranteed replay. A best-effort plan carries + // empty ends (never resolved - no broker call), and its offers keep computing + // atBacklogEnd = true against them, exactly as the endless map used to. + byId(streams) + ) + + private def byId(streams: Vector[StartFromStream]): Map[String, StartFromStream] = + streams.map(stream => stream.id -> stream).toMap + + /** Streams that held nothing when the session started - an empty topic, or one that would not + * say. They can never be waited for. */ + private def drainedAtStart(streams: Vector[StartFromStream]): Set[String] = + streams.filter(_.lastAtStart == EntryPosition.empty).map(_.id).toSet diff --git a/server/src/main/scala/consumer/session_runner/handleStartFrom.scala b/server/src/main/scala/consumer/session_runner/handleStartFrom.scala index 350882d13..fadba0d03 100644 --- a/server/src/main/scala/consumer/session_runner/handleStartFrom.scala +++ b/server/src/main/scala/consumer/session_runner/handleStartFrom.scala @@ -3,9 +3,14 @@ package consumer.session_runner import org.apache.pulsar.client.admin.PulsarAdmin import java.time.ZonedDateTime -import org.apache.pulsar.client.api.{Consumer, PulsarClient, Message as PulsarMessage, MessageId as PulsarMessageId} +import java.util.concurrent.{Callable, ExecutorCompletionService, Executors, ExecutionException, ThreadFactory, TimeUnit} +import java.util.concurrent.atomic.AtomicInteger +import org.apache.pulsar.client.api.{Consumer, PulsarClient, Message as PulsarMessage, MessageId as PulsarMessageId, MessageIdAdv} import _root_.topic.{TopicPartitioningType, getTopicPartitioning} -import _root_.consumer.start_from.{ConsumerSessionStartFrom, DateTime, DateTimeUnit, EarliestMessage, LatestMessage, MessageId, NthMessageAfterEarliest, NthMessageBeforeLatest, RelativeDateTime} +import _root_.consumer.start_from.{ApproximateEntryPosition, ApproximatePublishTimePosition, ConsumerSessionStartFrom, DateTime, DateTimeUnit, EarliestMessage, LatestMessage, MessageId, NthMessageAfterEarliest, NthMessageBeforeLatest, RelativeDateTime} + +import org.apache.pulsar.client.impl.MessageIdImpl +import org.apache.pulsar.common.protocol.Markers import scala.util.{Failure, Success, Try} import scala.jdk.CollectionConverters.* @@ -23,43 +28,1172 @@ def getPartitions(adminClient: PulsarAdmin, topicFqn: String): Vector[String] = .toVector partitions -def examineNonPartitionedTopicMessage(adminClient: PulsarAdmin, topicFqn: String, initialPosition: String, n: Long): Option[PulsarMessage[Array[Byte]]] = - Try(adminClient.topics.examineMessage(topicFqn, initialPosition, n)).toOption - -def examinePartitionedTopicMessage(adminClient: PulsarAdmin, topicFqn: String, initialPosition: String, n: Long): Option[PulsarMessage[Array[Byte]]] = - val partitions = getPartitions(adminClient, topicFqn) - partitions.flatMap(partitionFqn => examineNonPartitionedTopicMessage(adminClient, partitionFqn, initialPosition, n)) match - case Vector() => None - case candidates => - initialPosition match - case "earliest" => Some(candidates.minBy(msg => msg.getPublishTime)) - case "latest" => Some(candidates.maxBy(msg => msg.getPublishTime)) - -def findNthMessage(adminClient: PulsarAdmin, topicFqn: String, initialPosition: String, n: Long): Option[PulsarMessage[Array[Byte]]] = - getTopicPartitioning(adminClient, topicFqn).`type` match - case TopicPartitioningType.Partitioned => - examinePartitionedTopicMessage(adminClient, topicFqn, initialPosition, n) - case TopicPartitioningType.NonPartitioned => - examineNonPartitionedTopicMessage(adminClient, topicFqn, initialPosition, n) - -def findNthMessageMultiTopic(adminClient: PulsarAdmin, topics: Vector[String], initialPosition: String, n: Long): Option[PulsarMessage[Array[Byte]]] = - topics.flatMap(topicFqn => findNthMessage(adminClient, topicFqn, initialPosition, n)) match - case Vector() => None - case messages => - initialPosition match - case "earliest" => Some(messages.minBy(msg => msg.getPublishTime)) - case "latest" => Some(messages.maxBy(msg => msg.getPublishTime)) +/** Key the admin client uses to report how many messages the entry it just expanded holds. Set by + * `TopicsImpl.getIndividualMsgsFromBatch`; absent for a message that was not batched. + * + * VERIFIED against Pulsar 3.2.1: 100 messages sent with the default (batching) producer became ONE + * entry, and `examineMessage` answered with `X-Pulsar-num-batch-message -> 100` in + * `getProperties`. + */ +val batchSizeProperty = "X-Pulsar-num-batch-message" + +/** How many messages the entry behind `message` holds, read from a source a PRODUCER CANNOT FORGE. + * + * This count drives the backward walk's running total and the per-topic overshoot discard, so a + * value under producer control would let a crafted message overshoot the count and make the discard + * swallow real messages from the head of the stream. + * + * `X-Pulsar-num-batch-message` lives in `getProperties`, right beside the arbitrary keys a producer + * sets, and the admin client only overwrites it (from the entry's real batch metadata) for entries + * it actually expands as batches - so a forged value survives on an UNBATCHED message. The message + * id is the non-forgeable witness: the admin client returns a batch id (batch index >= 0) exactly + * for a real batch, which a producer cannot fake onto an unbatched message. + * + * - Batched id carrying its own batch size: use that - it is not a property at all. + * - Batched id without one: fall back to the verified property, but only as a POSITIVE number. + * - Unbatched id: exactly one message, whatever `getProperties` claims - the forged case. + * + * The fallback direction is deliberate: an unrecognised or out-of-range value is read as a single + * message, which makes the walk go DEEPER (over-deliver) rather than swallow. + */ +def messagesInEntryOf(message: PulsarMessage[Array[Byte]]): Int = + message.getMessageId match + case adv: MessageIdAdv if adv.getBatchIndex >= 0 => + val fromId = adv.getBatchSize + if fromId > 0 then fromId + else + message.getProperties.asScala + .get(batchSizeProperty) + .flatMap(value => Try(value.toInt).toOption) + .filter(_ >= 1) + .getOrElse(1) + case _ => 1 + +/** Strip the batch index off an id so it addresses the ENTRY. + * + * The admin client hands back a `BatchMessageIdImpl` pinned to batch index 0. Seeking to THAT id + * only delivers the whole entry while the consumer was built with `startMessageIdInclusive()` + * (without it the first message of the entry is skipped, which would silently shift every discard + * count by one). Seeking to the bare entry id behaves the same either way - VERIFIED against + * Pulsar 3.2.1 on a 3-entry x 10-message topic: both flags delivered the entry from its first + * message. + */ +def entryIdOf(messageId: PulsarMessageId): PulsarMessageId = messageId match + case id: MessageIdImpl => new MessageIdImpl(id.getLedgerId, id.getEntryId, id.getPartitionIndex) + case other => other + +/** One entry of a log, as the backward walk reads it: where it is, when it was published, and how + * many messages it holds. + * + * The publish time is what makes a MERGED backward walk possible across partitions. It costs + * nothing to carry - `examineMessage` returns a whole message - and it used to be thrown away, + * which is why narrowing the over-fetch afterwards needed a buffer of delivered messages. + * + * All messages of a batched entry share the entry's publish time (Pulsar stamps it once, on the + * batch's `MessageMetadata`), so an entry-level publish time is exact for every message in it. + */ +final case class LogEntry[A](entryId: A, publishTime: Long, messagesInEntry: Int) + +/** The k-th ENTRY counted back from the end of `topicFqn` (k = 1 is the last entry), or `None` once + * k is past the start of the log. + * + * `examineMessage` is ENTRY-addressed on both sides, but its failure modes differ: counting back + * from `latest` past the start FAILS (ManagedLedgerException "Incorrect parameter input", surfaced + * as an admin 500), while counting from `earliest` past the end silently CLAMPS to the last entry. + * Only the first is used here; the caller guards the clamping case as well. + * + * `None` means the BROKER SAID there is nothing there, and nothing else. A broker that could not + * answer - a timeout, a 401, a 404, a 500 about something else - throws, because the caller reads + * `None` as "this log is exhausted" and seeks to EARLIEST: erasing an operational failure into it + * turned "the latest 5" into the whole backlog with the session reporting success. See + * [[isEmptyLogAnswer]] for the measured classification. + * + * AN ENTRY IS NOT ALWAYS ONE MESSAGE, and not always a message at all: see [[logEntryOf]] for the + * two shapes that are neither - a chunk piece (refused) and a server-only marker (counted as + * zero). + */ +def entryFromLatest(adminClient: PulsarAdmin, topicFqn: String)(k: Long): Option[LogEntry[PulsarMessageId]] = + brokerAnswer(s"examining entry $k counted back from the end", topicFqn)(adminClient.topics.examineMessage(topicFqn, "latest", k)) + .map(message => logEntryOf(topicFqn, message)) + +/** What the backward walk makes of ONE examined entry, with the broker already out of the way - so + * the chunk refusal and the marker rule are both testable with a hand-built message rather than + * only against a live topic. + * + * Two entry shapes are NOT one user message, and they need opposite answers: + * + * - A CHUNK PIECE means the topic's entries and its messages have no fixed relationship at all, + * in either direction, so the walk REFUSES ([[isChunkPiece]]). + * - A SERVER-ONLY MARKER is an entry that holds zero messages ([[isServerOnlyMarkerEntry]]), so + * the walk simply counts it as zero and steps past it. Nothing is refused and nothing is + * miscounted: the broker never dispatches a marker, so an entry-level zero is the exact + * delivered count for it. + */ +def logEntryOf(topicFqn: String, message: PulsarMessage[Array[Byte]]): LogEntry[PulsarMessageId] = + if isChunkPiece(message) then + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: $topicFqn stores CHUNKED messages (one message split " + + "across several entries), and 'Latest n messages' counts entries - an entry count is not a message count " + + "there. Use a time-based position or 'Skip first n messages', which counts what is actually delivered.", + null + ) + val messages = if isServerOnlyMarkerEntry(message) then 0 else messagesInEntryOf(message) + LogEntry(entryIdOf(message.getMessageId), message.getPublishTime, messages) + +/** Whether this examined message is one CHUNK of a larger logical message. + * + * A chunking producer stores ONE logical message as SEVERAL ledger entries, which the consumer + * reassembles - so counting entries counts fragments, and every count the walk produces is + * silently wrong. Read from the message's own metadata behind a guarded impl cast; anything + * unreadable counts as "not chunked", so an exotic client type degrades to the old behaviour + * rather than refusing valid topics. */ +def isChunkPiece(message: PulsarMessage[Array[Byte]]): Boolean = message match + case impl: org.apache.pulsar.client.impl.MessageImpl[?] => + Try { + val metadata = impl.getMessageBuilder + metadata != null && metadata.hasNumChunksFromMsg && metadata.getNumChunksFromMsg > 1 + }.getOrElse(false) + case _ => false + +/** Whether this examined entry is a SERVER-ONLY MARKER - a transaction commit/abort record or a + * replicated-subscription snapshot - rather than a user message. + * + * SUCH AN ENTRY IS COUNTED BY THE BROKER AND DELIVERED TO NOBODY, which is exactly the asymmetry + * every entry-addressed position has to survive. Verified against Pulsar 3.2.1, end to end: + * + * - `TopicTransactionBuffer.commitTxn` / `abortTxn` build the marker with + * `Markers.newTxnCommitMarker` / `newTxnAbortMarker` (which set `MessageMetadata.marker_type`) + * and append it with `ManagedLedger.asyncAddEntry`, so it is an ORDINARY entry of the topic's + * own ledger and `getNumberOfEntries` counts it; + * - `PersistentTopicsBase.internalExamineMessageAsync` does no marker filtering whatsoever - it + * hands the entry straight to `generateResponseWithEntry`, which emits the metadata as the + * `X-Pulsar-marker-type` response header, and the admin client's `getMessagesFromHttpResponse` + * parses that header back onto the `MessageMetadata` it hands to `MessageImpl`. So a marker + * IS returned by `examineMessage`, and its type IS readable here; + * - `AbstractBaseDispatcher.filterEntriesForConsumer` drops any entry for which + * `Markers.isServerOnlyMarker` holds (it nulls and releases it) before dispatch, so no consumer + * ever receives one. + * + * Read behind the same guarded impl cast [[isChunkPiece]] uses, and through Pulsar's own + * `Markers.isServerOnlyMarker` so this cannot drift from the definition the dispatcher filters by. + * Anything unreadable counts as "not a marker", which leaves the entry counted as one message - + * the direction that over-delivers rather than swallows. + */ +def isServerOnlyMarkerEntry(message: PulsarMessage[Array[Byte]]): Boolean = message match + case impl: org.apache.pulsar.client.impl.MessageImpl[?] => + Try { + val metadata = impl.getMessageBuilder + metadata != null && Markers.isServerOnlyMarker(metadata) + }.getOrElse(false) + case _ => false + +/** The FIRST entry `topicFqn` retains right now, or `None` when it retains nothing. One lookup; + * the retention re-check reads it per contributing topic after a latest-n walk resolves. */ +def earliestRetainedEntryId(adminClient: PulsarAdmin, topicFqn: String): Option[PulsarMessageId] = + brokerAnswer("examining the first retained entry", topicFqn)(adminClient.topics.examineMessage(topicFqn, "earliest", 1)) + .map(message => entryIdOf(message.getMessageId)) + +/** How many times ONE backward step may re-read a topic whose end moved forward under it before the + * walk REFUSES. + * + * A handful of appends can land inside the milliseconds of one admin round trip, and the walk + * re-anchors past them. A log that keeps outrunning the walk for this many consecutive lookups has + * no resolvable "last n" at this moment, and both silent endings are wrong answers delivered as + * success: classifying it as exhaustion seeks EARLIEST (the whole backlog - the original defect), + * and stopping short delivers fewer than n. The walk therefore fails with + * [[StartFromUnresolvableException]], naming the topic, so the user can retry when the producer + * quietens or reach for a time position instead. + * + * A broker that CLAMPS to its last entry forever (defensive - Pulsar 3.2 FAILS past the start + * rather than clamping) must not burn this bound or be refused: it is told apart from growth by + * ONE verification lookup - re-asking the k that produced the last ACCEPTED entry. A clamped end + * never moves, so that k answers the same entry again; a grown end answers a newer one. The clamp + * therefore still resolves as `Everything`, one lookup later than the old first-repeat guard - the + * guard that could not tell a clamp from a single concurrent append. + */ +val maxLatestNReanchorSteps: Int = 64 + +/** The wall-clock budget for resolving one 'Latest n messages' request. + * + * The count cap ([[latestNMaxAccepted]]) bounds N, but N is a poor proxy for COST: the walk pays + * roughly one synchronous `examineMessage` per ENTRY, so a batched topic answers ten million in + * thousands of lookups while an unbatched one would need ten million of them - hours, inside a + * session-create call that shows no progress and cannot be cancelled. Time is the honest bound: + * a resolution that cannot finish inside this budget fails with the same "narrow the request" + * guidance the count cap gives, instead of grinding on. */ +val latestNResolveBudgetMs: Long = 30_000L + +/** Entry-position order for the backward walk: `a` is strictly older than `b` when its id sorts + * before `b`'s. `MessageIdImpl.compareTo` orders by ledger then entry then partition, so an earlier + * append is strictly less. This is what tells a genuine step back from a `latest, k` answer that + * only moved because the log grew under the walk. */ +def latestNEntryIsOlder(a: PulsarMessageId, b: PulsarMessageId): Boolean = a.compareTo(b) < 0 + +/** Where ONE physical topic has to start so that the session's topics TOGETHER deliver the last n + * messages. */ +enum LatestNSeek[+A]: + /** Everything this topic holds is older than the cut, so it contributes no history at all: seek + * it to LATEST. NOT to earliest - that would show the whole log. */ + case Nothing + + /** The walk consumed this topic's whole log: seek to EARLIEST and discard nothing. */ + case Everything + + /** Seek to `entryId` and drop the first `discard` messages delivered from it - the overshoot + * inside the entry the walk stopped on, which a seek cannot express because it can only land + * on an entry boundary. */ + case FromEntry(entryId: A, discard: Long) + +/** Resolve "deliver exactly the last `n` messages across these topics, newest first by publish + * time" into a starting position per topic - WITHOUT reading a single message. + * + * ONE MERGED BACKWARD WALK, not one walk per topic plus a buffer afterwards. Each topic gets a + * cursor stepping back through its entries; repeatedly take the cursor whose current entry has the + * LARGEST publish time, add that entry's message count to a running total, and step that cursor + * back one. Stop when the total reaches n. Because a cursor only ever moves backwards, the entries + * taken from a topic are always a contiguous suffix of its log - so "seek to the oldest entry + * taken" delivers exactly the messages the walk chose, plus everything newer, and nothing else. + * + * MEMORY IS O(NUMBER OF TOPICS): one cursor and one entry's metadata each. Nothing is buffered and + * no message payload is ever held. This replaced a bounded top-n heap of exactly n DELIVERED + * messages, which made "the latest n" the one start-from whose memory was a number the user typed, + * and which could let a live message evict a historical one because both went through the same + * heap. Neither failure mode exists here: the cut is decided before anything is delivered. + * + * COST IS O(n / batch size + topics) admin lookups - strictly cheaper than the per-topic walks it + * replaces (those cost O(topics * n / batch size)), and independent of how big the topics are. + * + * THE CUT IS BY APPEND POSITION WITHIN A TOPIC AND BY ENTRY PUBLISH TIME ACROSS TOPICS, which is + * the contract [[MessageOrderKey]] states and not a stronger one. A partition whose producer clock + * stepped backwards can hold a high-timestamp message deeper in its log than the walk ever reaches, + * and it will not be found - finding it would mean scanning the whole log, which is O(topic) at any + * n. The publish-time tie-break is the same one [[MessageOrderKey]] uses (time, then topic name), + * so an entry-level cut and a message-level order cannot disagree. + * + * AN ENTRY THAT IS NOT A MESSAGE COSTS A STEP AND NOTHING ELSE. A server-only marker (a + * transaction commit/abort record, a replicated-subscription snapshot) occupies a real entry that + * `examineMessage` returns and `getNumberOfEntries` counts, but no consumer is ever handed one - + * so `entryFromLatest` reports it as ZERO messages ([[logEntryOf]]) and the walk crosses it + * without counting it. The walk can therefore never STOP on a marker either: for n > 0, an entry + * that adds nothing cannot be the one that reaches n, so the anchor is always a real message + * entry (or, on an exhausted log, its oldest entry, which is equally safe to seek to - the broker + * filters the marker out of the dispatch). Counting a marker as one message, which is what the + * `max 1` floor used to do, made the session hold one message fewer per marker crossed. + * + * PURE: the broker sits behind `entryFromLatest`, so batched / unbatched / uneven / exhausted / + * clamped / empty logs, marker entries, and every interleaving across topics, are testable with a + * plain lambda. + * + * `entryFromLatest(topic)(k)` must answer with the k-th entry counted back from the end of that + * topic (k = 1 is the last entry), or `None` once k is past its start. + * + * THE ANCHOR MOVES WHILE THE WALK RUNS. `examineMessage(topic, "latest", k)` counts back from + * whatever the end is at the moment it is asked, so a producer appending during session creation + * shifts position k forward under the cursor: a single append makes the next step answer with the + * entry JUST taken, a burst makes it answer with a NEWER one. Neither is exhaustion. The walk tells + * a real step back from a re-anchored answer with `entryIsOlder` (a total order on entry positions + * - `MessageIdImpl.compareTo` in production), and steps k forward until the answer is strictly + * older, re-anchoring past the growth. A topic ends on `None` or on a VERIFIED clamp; a log still + * outrunning the walk at the re-anchor bound FAILS the resolution loudly instead of answering + * with the whole backlog or a short count. See [[maxLatestNReanchorSteps]]. + * + * `entryIsOlder(a, b)` must be true exactly when position `a` sits strictly BEFORE `b` in the log. + */ +def resolveLatestN[A]( + n: Long, + topicFqns: Vector[String], + entryFromLatest: String => Long => Option[LogEntry[A]], + entryIsOlder: (A, A) => Boolean, + resolveBudgetMs: Long = latestNResolveBudgetMs, + nowMs: () => Long = () => System.nanoTime() / 1_000_000L +): Map[String, LatestNSeek[A]] = + val topics = topicFqns.distinct + if n <= 0 then topics.map(_ -> LatestNSeek.Nothing).toMap + else + val startedAtMs = nowMs() + def checkBudget(): Unit = + val elapsed = nowMs() - startedAtMs + if elapsed > resolveBudgetMs then + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: 'Latest n messages' has been walking entry metadata for " + + s"${elapsed}ms, past its ${resolveBudgetMs}ms budget - the topics hold more entries than can be walked " + + "interactively (an unbatched topic costs one broker lookup per message). Ask for fewer messages, use " + + "'Skip first n messages' (which streams and reports progress), or a time-based position.", + null + ) + + val nextK = scala.collection.mutable.Map.from(topics.map(_ -> 1L)) + // The entry each cursor is currently offering. A topic missing from here has either run off + // the start of its log or had its offer taken and not yet been stepped. + val head = scala.collection.mutable.Map.empty[String, LogEntry[A]] + val previousEntryId = scala.collection.mutable.Map.empty[String, A] + // The k that ANSWERED with the accepted entry, at the moment it was accepted. Re-asking it + // is what tells a clamping broker (same answer - the end never moved) from a log that grew + // under the walk (a newer answer). See [[maxLatestNReanchorSteps]]. + val acceptedAtK = scala.collection.mutable.Map.empty[String, Long] + val oldestTaken = scala.collection.mutable.Map.empty[String, A] + val exhausted = scala.collection.mutable.Set.empty[String] + + def step(topicFqn: String): Unit = + // Advance k until the answer is STRICTLY OLDER than the entry last taken from this topic, + // re-anchoring past anything appended since the previous step. An answer that is not + // older is the moving anchor, not exhaustion: the same id means one append shifted k back + // onto the entry just taken (the old code read that as exhausted and fell back to + // EARLIEST - the whole backlog for a "latest n"); a newer id means a burst arrived (the + // old code took it, walking the cursor forward off the contiguous suffix and + // double-counting). Only `None` and a VERIFIED clamp end the topic; a log still + // outrunning the walk at the re-anchor bound fails the resolution rather than answering + // with a set nobody asked for - see [[maxLatestNReanchorSteps]] for both trades. + var reanchors = 0 + var settled = false + while !settled do + checkBudget() + entryFromLatest(topicFqn)(nextK(topicFqn)) match + case None => + exhausted += topicFqn + settled = true + case Some(entry) => + val previous = previousEntryId.get(topicFqn) + if previous.forall(prev => entryIsOlder(entry.entryId, prev)) then + head(topicFqn) = entry + previousEntryId(topicFqn) = entry.entryId + acceptedAtK(topicFqn) = nextK(topicFqn) + nextK(topicFqn) = nextK(topicFqn) + 1 + settled = true + else + // The SAME id again is ambiguous - one append per round trip and a + // clamping broker look identical from here - so it is settled by one + // verification lookup at the k that produced the accepted entry: a + // clamped end never moves and answers the same entry (exhausted); a + // grown end answers a newer one (re-anchor). `None` there means the log + // was trimmed under the walk past even the accepted entry - nothing + // older is left to take. A NEWER id needs no verification: only growth + // produces it. + val verifiedClamp = + previous.contains(entry.entryId) && { + entryFromLatest(topicFqn)(acceptedAtK(topicFqn)) match + case Some(check) => previous.contains(check.entryId) + case None => true + } + if verifiedClamp then + exhausted += topicFqn + settled = true + else if reanchors >= maxLatestNReanchorSteps then + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: 'Latest n messages' walked $topicFqn " + + s"backwards, but new messages kept arriving faster than the walk could step for " + + s"$maxLatestNReanchorSteps consecutive lookups. Retry when the topic is quieter, or use a " + + "time-based position or 'Skip first n messages' instead.", + null + ) + else + nextK(topicFqn) = nextK(topicFqn) + 1 + reanchors += 1 + + topics.foreach(step) + + var total = 0L + var lastTaken: Option[String] = None + // A max-heap of current heads keyed (publish time, topic), so selecting each entry costs + // O(log topics) instead of a scan of every topic per entry. A head changes only when its + // topic's entry is TAKEN (and the topic re-stepped), so the heap is pushed at exactly + // those points; a popped pair that no longer matches the live head map is stale and + // discarded. The tuple tie-break is the same one the scan used, so the cut is unchanged. + val selection = scala.collection.mutable.PriorityQueue.empty[(Long, String)] + head.foreach((topicFqn, entry) => selection.enqueue(entry.publishTime -> topicFqn)) + var walking = true + while walking do + var chosen: Option[String] = None + while chosen.isEmpty && selection.nonEmpty do + val (publishTime, topicFqn) = selection.dequeue() + if head.get(topicFqn).exists(_.publishTime == publishTime) then chosen = Some(topicFqn) + chosen match + case None => walking = false + case Some(topicFqn) => + val entry = head(topicFqn) + // `max 0` and NOT `max 1`: the floor is only here so a nonsensical negative + // count cannot walk the total backwards. ZERO is a real, verified answer - a + // server-only marker entry (see [[isServerOnlyMarkerEntry]]) holds no messages + // and is never dispatched - and forcing it to one made the walk stop one + // message short for every marker it crossed. `messagesInEntryOf` never returns + // zero, so nothing else is affected. + total += (entry.messagesInEntry max 0) + oldestTaken(topicFqn) = entry.entryId + lastTaken = Some(topicFqn) + head.remove(topicFqn) + if total >= n then walking = false + else + step(topicFqn) + head.get(topicFqn).foreach(next => selection.enqueue(next.publishTime -> topicFqn)) + + // The entry the walk STOPPED on may hold more messages than were still needed, and those + // are the OLDEST inside it. They cannot be seeked past - a seek lands on an entry boundary - + // so they are dropped from the head of that one topic's stream. + val overshoot = (total - n) max 0L + + topics.map { topicFqn => + topicFqn -> (oldestTaken.get(topicFqn) match + // Contributed nothing, but its tail WAS inspected: anchor there, not at seek-time + // latest. The walk already paid for this knowledge (the untaken head IS the + // inspected tail), and seeking to "latest" AT SEEK TIME silently jumped anything + // published between the inspection and the seek - a live message lost outright, + // where every contributing topic kept its concurrent appends. The anchor entry + // itself is delivered and dropped per topic (the same head-drop the overshoot + // uses), which is exactly "everything after the inspected tail". + case None => + head.get(topicFqn) match + // `max 0` for the same reason the running total uses it: an inspected tail + // that is a marker holds nothing to drop, and dropping one anyway would + // eat the first real message published after it. + case Some(tail) => LatestNSeek.FromEntry(tail.entryId, (tail.messagesInEntry max 0).toLong) + // EMPTY when inspected: everything the topic holds at seek time arrived + // after the inspection, so all of it is live traffic the session must + // show - which is what seeking EARLIEST delivers. "Latest" would race the + // same appends the anchor above exists to keep. + case None => LatestNSeek.Everything + case Some(entryId) if overshoot > 0 && lastTaken.contains(topicFqn) => LatestNSeek.FromEntry(entryId, overshoot) + // An EXHAUSTED contributor keeps its anchor: FromEntry(oldest taken) covers the + // same messages a seek-to-earliest would, and - unlike `Everything` - it is + // re-verifiable. Mapping it to Everything let retention delete a counted + // contribution between resolving and seeking with nobody noticing: the anchor + // re-check only inspects FromEntry, so the session succeeded while silently + // holding fewer than the n it promised. Everything is reserved for the one case + // with genuinely nothing to verify: a topic that held NOTHING when inspected. + case Some(entryId) if exhausted.contains(topicFqn) => LatestNSeek.FromEntry(entryId, 0L) + case Some(entryId) => LatestNSeek.FromEntry(entryId, 0L)) + }.toMap + +/** Whether `topicFqn` names a topic that stores nothing. + * + * A non-persistent topic has no backlog, no history and no entry to address: messages go straight + * from producer to whoever is connected. `PulsarAdmin.topics.examineMessage` refuses one outright + * (HTTP 405, "Examine messages on a non-persistent topic is not allowed"). + * + * Decided from the FQN and NOT from that 405: the admin call sits inside a `Try(...).toOption`, so + * catching the refusal would turn a request the session cannot satisfy into a silent fallback to + * earliest or latest - the user asks for a position in history and gets a different one, with the + * session looking like it worked. The `Try` stays as a backstop; this is the mechanism. + * + * Only the scheme counts. "persistent://public/default/non-persistent-audit" is an ordinary + * persistent topic that happens to be named after the word. + */ +def isNonPersistentTopic(topicFqn: String): Boolean = topicFqn.startsWith("non-persistent://") + +/** Whether this start-from needs messages to still be stored somewhere. + * + * Everything except "latest message" does. `EarliestMessage` is deliberately in the needs-history + * set: a seek to earliest on a non-persistent topic does not fail, it silently behaves as "from + * now" - answering a request for the start of the topic with the live tail. + * + * The catch-all treats an unrecognised mode as needing a history, so a mode added later fails + * loudly on a non-persistent topic rather than degrading quietly. + */ +def startFromNeedsRetainedHistory(startFrom: ConsumerSessionStartFrom): Boolean = startFrom match + case _: LatestMessage => false + case _: EarliestMessage => true + case _: NthMessageAfterEarliest => true + case _: NthMessageBeforeLatest => true + case _: MessageId => true + case _: DateTime => true + case _: RelativeDateTime => true + // Both approximate modes are proportions OF A HISTORY, and both need the broker to say what + // that history is - the entry count for one, the first and last publish times for the other. + // `examineMessage` answers neither on a non-persistent topic (405), so neither can compute a + // position there. + case _: ApproximateEntryPosition => true + case _: ApproximatePublishTimePosition => true + case _ => true + +/** Why this start-from cannot be honoured on these topics, or `None` if it can. + * + * Rejects only when NOTHING in the resolved set retains anything. A session may legitimately mix + * persistent and non-persistent topics, and failing all of it because one topic is live-only would + * make history positions unusable on any such session - the mixed case is handled by seeking the + * persistent topics to the requested position and the rest to "now" (see [[handleStartFrom]]). + * + * An empty resolved set is not this function's problem: `ConsumerSessionRunner.make` already + * rejects a session that resolved to no topics, with a message that points at the target. + */ +def startFromRejectionReason(startFrom: ConsumerSessionStartFrom, topicFqns: Vector[String]): Option[String] = + val allLiveOnly = topicFqns.nonEmpty && topicFqns.forall(isNonPersistentTopic) + Option.when(allLiveOnly && startFromNeedsRetainedHistory(startFrom)) { + val named = topicFqns.take(3).mkString(", ") + val andMore = if topicFqns.size > 3 then s" and ${topicFqns.size - 3} more" else "" + s"Start-from ${startFrom.getClass.getSimpleName} needs a retained message history, but every topic this session resolved to is " + + s"non-persistent ($named$andMore). A non-persistent topic stores no messages, so only LatestMessage can be used on one." + } + +/** Why this start-from's COUNT cannot be honoured, or `None` if it can. + * + * The count arrives over gRPC as a plain `int64` that any client can fill in with anything. The + * browser validates it too, so this is the TRUST BOUNDARY rather than the only guard - and a + * boundary refuses rather than clamps, because a clamp answers a different question in silence: + * "skip the first -1 messages" was read as EARLIEST (the whole topic) and "the latest -1 messages" + * as LATEST (nothing retained at all). + * + * SKIP-N HAS DELIBERATELY NO UPPER BOUND. Skipping n messages is O(n) whatever n is - Pulsar keeps + * no message-ordinal index, so the only exact way to reach message n is to stream n and throw them + * away - and the start-from progress API exists precisely so a long skip can be watched rather + * than forbidden. Any cap here would be an invented number, not a limit of the design. + * + * LATEST-N HAS ONE, and it is an OPERATIONAL bound, stated as such. The walk behind latest-n + * costs one broker lookup per entry, synchronously, while session creation holds the per-name + * lifecycle lock and reports no progress. The old boundary (Int.MaxValue, a leftover of a heap + * implementation that no longer exists) still admitted a request the server would grind on for + * hours; [[latestNMaxAccepted]] is the honest ceiling - at worst tens of thousands of entry + * lookups even on unbatched topics, i.e. minutes not hours - and the rejection says what to use + * instead. It is never NARROWED: narrowing served a different request without saying so. + * + * PURE, so every boundary is pinned by test. + */ +/** The most a 'Latest n messages' request may ask for. + * + * A sanity bound on N, NOT the cost bound: the walk pays roughly one broker lookup per ENTRY, so + * this many messages is thousands of lookups on a well-batched topic and ten million on an + * unbatched one - which no interactive request survives. The honest cost bound is + * [[latestNResolveBudgetMs]]: a walk that cannot finish in time fails with guidance, whatever N + * was. This ceiling stays to refuse the absurd outright (and to give the UI a number to mirror - + * see `latestMessageCountMax` in the frontend, pinned to this by test on both sides). */ +val latestNMaxAccepted: Long = 10_000_000L + +/** THE DUPLICATE-TARGET CONTRACT for the counted modes, stated once, and it is MODE-SPECIFIC. + * + * "Latest n": the count is per SESSION across its unique TOPICS, and when two enabled targets + * select the SAME topic each target delivers its own counted set through its own subscription. + * Two targets on one 10-message topic with "latest 3" therefore show three rows EACH (six in + * total, tagged with their target), not three split between them: a second target exists + * precisely to show a second view - its own filters, its own coloring - and starving one view to + * feed the other would make either target's output depend on the mere existence of the other. + * The metadata walk is still memoised per topic, so the broker is asked once however many + * targets share it. + * + * "Skip first n" REFUSES overlapping targets instead (see [[skipOverlapRejectionReason]]): its + * count is spent by ONE session-wide budget over the merged delivered stream, so with two + * subscriptions on one topic the budget would be spent on COPIES - a message dropped through one + * target while the other still shows it, in whichever interleaving the brokers produced. Until a + * per-source-message semantics exists (decide each unique message once, apply the decision to + * every view), refusing loudly is the only answer that means something. + */ +def startFromCountRejectionReason(startFrom: ConsumerSessionStartFrom): Option[String] = startFrom match + case v: NthMessageAfterEarliest if v.n < 0 => + Some(s"Start-from 'Skip first n messages' needs n to be zero or more, but it was ${v.n}.") + case v: NthMessageBeforeLatest if v.n < 0 => + Some(s"Start-from 'Latest n messages' needs n to be zero or more, but it was ${v.n}.") + case v: NthMessageBeforeLatest if v.n > latestNMaxAccepted => + Some( + s"Start-from 'Latest n messages' accepts at most $latestNMaxAccepted, but ${v.n} were asked for. " + + "The last n are located by walking entry metadata backwards, one broker lookup per entry, while session " + + "creation waits with no progress to show - a larger n would grind for hours. " + + "Use 'Skip first n messages' (which streams and reports progress) or a time position to reach further back." + ) + case _ => None + +/** Why a start-from derived from the RAW retained log cannot be honoured against read-compacted + * targets, or `None`. + * + * Compacted reading changes WHAT IS VISIBLE - only the newest message per key survives before + * the compaction horizon. Latest-n counts raw entries; the entry-percentage mode measures them; + * and the publish-time-percentage mode derives its range from their boundary entries. Each can be + * arbitrarily unrelated to the compacted view when keys repeat. Skip-n remains valid because it + * counts what the consumer actually delivers. */ +def readCompactedStartFromRejectionReason( + startFrom: ConsumerSessionStartFrom, + readCompactedTargetIndexes: Vector[Int] +): Option[String] = + if readCompactedTargetIndexes.isEmpty then None + else + val targets = readCompactedTargetIndexes.sorted.mkString(", ") + startFrom match + case v: NthMessageBeforeLatest if v.n > 0 => + Some( + s"Start-from 'Latest n messages' counts raw stored entries, but target(s) $targets read compacted. " + + "The stored count and visible messages can differ when keys repeat. Use 'Specific time', " + + "'Latest message', or turn off compacted reading." + ) + case _: ApproximateEntryPosition => + Some( + s"Start-from 'Approximate position (% of data)' measures raw stored entries, but target(s) $targets read compacted. " + + "That percentage does not describe the visible compacted messages. Use 'Earliest message', " + + "'Latest message', 'Specific time', or turn off compacted reading." + ) + case _: ApproximatePublishTimePosition => + Some( + s"Start-from 'Approximate position (% of time)' derives its range from the raw retained log, but target(s) " + + s"$targets read compacted. That range does not describe the visible compacted messages. Use " + + "'Earliest message', 'Latest message', 'Specific time', or turn off compacted reading." + ) + case _ => None + +/** Why a counted SKIP cannot run over these enabled targets' topic sets, or `None`. + * + * See the duplicate-target contract above: skip-n's budget is session-wide over the merged + * stream, so two subscriptions on one physical topic spend it on COPIES and neither target's + * output means "everything after the first n". Refused at creation, before any consumer exists. */ +def skipOverlapRejectionReason(startFrom: ConsumerSessionStartFrom, topicsPerEnabledTarget: Vector[Vector[String]]): Option[String] = + startFrom match + case v: NthMessageAfterEarliest if v.n > 0 => + val seenBy = topicsPerEnabledTarget.flatMap(_.distinct).groupBy(identity).view.mapValues(_.size) + val overlapping = seenBy.collect { case (topicFqn, targets) if targets > 1 => topicFqn }.toVector.sorted + Option.when(overlapping.nonEmpty) { + val named = overlapping.take(3).mkString(", ") + val andMore = if overlapping.size > 3 then s" and ${overlapping.size - 3} more" else "" + s"Start-from 'Skip first n messages' cannot run while two enabled targets select the same topic ($named$andMore): " + + "the skip counts the session's merged stream once, so it would drop a message through one target while the " + + "other still shows it. Disable one of the overlapping targets or give them disjoint topics." + } + case _ => None + +/** Why a resolved latest-n cut can no longer be applied, or `None`: an anchor entry the walk + * counted was REMOVED by retention in the gap between resolving and seeking. Seeking to a + * trimmed anchor silently lands past it, and the session then holds fewer than the n it reported + * it would - a wrong answer delivered as success. One `earliestEntry` lookup per contributing + * topic; a topic that now retains NOTHING has lost its anchor by definition. */ +def latestNAnchorRejectionReason[A]( + cut: Map[String, LatestNSeek[A]], + earliestEntry: String => Option[A], + entryIsOlder: (A, A) => Boolean +): Option[String] = + cut.toVector.sortBy(_._1).collectFirst { + case (topicFqn, LatestNSeek.FromEntry(anchor, _)) + if earliestEntry(topicFqn).map(first => entryIsOlder(anchor, first)).getOrElse(true) => + s"Could not apply the requested start-from position: retention removed the resolved anchor entry on $topicFqn " + + "between resolving 'Latest n messages' and seeking to it, so the session would silently hold fewer than the " + + "n that was asked for. Retry; if it keeps happening, the topic's retention is shorter than the time it takes " + + "to position against it." + } + +/** Split by whether the topic behind each item retains anything: `(has a history, live tail only)`. */ +def splitByRetainedHistory[A](items: Vector[A], topicOf: A => String): (Vector[A], Vector[A]) = + items.partition(item => !isNonPersistentTopic(topicOf(item))) + +/** Where an [[ApproximateEntryPosition]] lands on ONE physical topic. */ +enum ApproximateEntrySeek: + /** The very beginning of the retained log. */ + case Earliest + + /** Past the last retained message - what "latest" means, i.e. nothing retained is shown. */ + case Latest + + /** The 1-based ENTRY ordinal counted from the earliest retained entry, as + * `examineMessage(topic, "earliest", entryOrdinal)` addresses it. + * + * NOT named `ordinal`: every Scala 3 enum case already has an `ordinal` member. + */ + case Entry(entryOrdinal: Long) + +/** Reject a fraction that is not one. Shared by both approximate modes, and `modeLabel` names the mode + * that refused it - two modes now carry a fraction, and a session can only be fixed if the error + * says which control was wrong. + * + * NaN needs its own check: every comparison against it is false, so a plain range test would let it + * through and it would then floor into an entry ordinal or timestamp. + */ +private def requireFraction(fraction: Double, modeLabel: String): Unit = + if fraction.isNaN then + throw new IllegalArgumentException(s"Start-from '$modeLabel' must be a fraction between 0.0 and 1.0, but it was NaN.") + if fraction < 0.0 || fraction > 1.0 then + throw new IllegalArgumentException(s"Start-from '$modeLabel' must be a fraction between 0.0 and 1.0, but it was $fraction.") + +/** Why this start-from's FRACTION is not a fraction, or `None`. + * + * The same rule [[requireFraction]] enforces, asked at the REQUEST boundary. It is enforced twice + * on purpose: deep inside resolution it protects the arithmetic, and here it lets a create refuse + * a nonsense fraction as malformed INPUT - the class of error a client can act on - instead of + * surfacing it as a resolution failure indistinguishable from a broker that would not answer. + * PURE, like every other rejection reason here. + */ +def startFromFractionRejectionReason(startFrom: ConsumerSessionStartFrom): Option[String] = startFrom match + case v: ApproximateEntryPosition => + Try(requireFraction(v.fraction, "Approximate position (% of data)")).failed.toOption.map(_.getMessage) + case v: ApproximatePublishTimePosition => + Try(requireFraction(v.fraction, "Approximate position (% of time)")).failed.toOption.map(_.getMessage) + case _ => None + +/** One finite budget for resolving either percentage mode. The work is constant in log size but + * linear in physical-topic count, so the session-wide stream cap alone is not a latency bound. */ +val approximatePositionResolveBudgetMs: Long = 30_000L + +/** Maximum concurrent broker lookups made by ONE percentage-position resolution. This turns a + * 2,000-stream session into bounded waves without either 2,000 threads or a serial 4,000-6,000 + * request create path. */ +val approximatePositionLookupParallelism: Int = 16 + +/** Maximum concurrent broker lookups made by the WHOLE SERVER, across every session resolving a + * position or reading topic positions at once. + * + * The per-call bound above is only a local one: a pool was created per resolution, so N concurrent + * creates meant N x 16 worker threads and N x 16 admin requests in flight, with nothing anywhere + * saying no. Together with an admission check that did not cover in-flight builds, that is how a + * handful of browser tabs turned into hundreds of threads hammering the broker's admin plane. One + * shared executor makes the bound global: concurrent resolutions interleave through it instead of + * multiplying, and each still keeps its own wall-clock budget and its own cancellation. + */ +val approximatePositionLookupGlobalParallelism: Int = 32 + +private val approximateLookupThreadNumber = AtomicInteger(0) +private val approximateLookupThreadFactory: ThreadFactory = (runnable: Runnable) => + val thread = Thread(runnable, s"start-from-position-${approximateLookupThreadNumber.incrementAndGet()}") + thread.setDaemon(true) + thread + +/** THE server-wide broker-admin lookup budget - see [[approximatePositionLookupGlobalParallelism]]. + * Created on first use, so a server nobody has asked for a position owns no threads at all, and + * never shut down: it lives for the process exactly as the session maintenance scheduler does. */ +private lazy val approximateLookupExecutor: java.util.concurrent.ExecutorService = + Executors.newFixedThreadPool(approximatePositionLookupGlobalParallelism, approximateLookupThreadFactory) + +/** How many of the shared budget's workers are running a lookup right now - a test's window onto + * the global bound, and what a saturation metric would read. */ +private val approximateLookupWorkersBusy = AtomicInteger(0) + +private[session_runner] def approximateLookupWorkersInFlight: Int = approximateLookupWorkersBusy.get + +/** Resolve one value per DISTINCT physical topic with bounded concurrency and one wall-clock + * deadline for the whole operation. + * + * TWO BOUNDS, and the second is the one that protects the server. `parallelism` is how wide THIS + * operation may go; [[approximatePositionLookupGlobalParallelism]] is how wide every operation may + * go at once, and it is enforced by running every task on one shared executor. A pool per call + * bounded nothing server-wide: N concurrent resolutions meant N x 16 threads and N x 16 admin + * requests in flight. Tasks are submitted through a sliding window `parallelism` wide, so a single + * wide operation cannot monopolise the shared budget either, and concurrent operations interleave + * through it rather than multiplying. + * + * On failure or timeout nothing further is submitted, every task already submitted is cancelled, + * and running ones are interrupted. Pulsar's own request timeout remains the backstop for an HTTP + * call that does not react to interruption. The interrupt status is cleared at the START of each + * task rather than at its end: the workers are SHARED now, so one operation's cancellation must + * not be delivered to an unrelated operation's next lookup. + * + * Package-visible and generic so tests can pin deduplication, concurrency and the deadline without + * a broker or a mock PulsarAdmin. + */ +def boundedParallelTopicLookup[A]( + topicFqns: Vector[String], + operation: String, + lookup: String => A, + budgetMs: Long = approximatePositionResolveBudgetMs, + parallelism: Int = approximatePositionLookupParallelism, + // What the user can DO about a budget that ran out. The start-from modes have somewhere to + // send them; other callers of this fan-out do not, and telling a Topic Positions poll to pick + // a different start position would be nonsense. + remediation: String = + "Narrow the topic selector, use 'Earliest message', 'Latest message', or 'Specific time', and retry." +): Map[String, A] = + val topics = topicFqns.distinct + if topics.isEmpty then Map.empty + else if budgetMs <= 0 then + throw StartFromUnresolvableException( + s"Could not resolve $operation: its ${budgetMs}ms wall-clock budget was already exhausted. " + + "Narrow the topic selector or retry.", + null + ) + else + require(parallelism > 0, s"parallelism must be positive, got $parallelism") + val windowWidth = math.min(parallelism, topics.size) + val completed = ExecutorCompletionService[(String, A)](approximateLookupExecutor) + val startedAtNanos = System.nanoTime() + val budgetNanos = TimeUnit.MILLISECONDS.toNanos(budgetMs) + val deadlineNanos = + if Long.MaxValue - startedAtNanos < budgetNanos then Long.MaxValue else startedAtNanos + budgetNanos + + val futures = scala.collection.mutable.ArrayBuffer.empty[java.util.concurrent.Future[(String, A)]] + var nextToSubmit = 0 + def submitNext(): Unit = + if nextToSubmit < topics.size then + val topicFqn = topics(nextToSubmit) + nextToSubmit += 1 + futures += completed.submit(new Callable[(String, A)]: + override def call(): (String, A) = + // A cancelled sibling's interrupt must not land on this task: the worker is + // shared, and its previous occupant may have been interrupted on the way out. + Thread.interrupted() + approximateLookupWorkersBusy.incrementAndGet() + try topicFqn -> lookup(topicFqn) + catch + case err: StartFromUnresolvableException => throw err + case err: Throwable => + throw StartFromUnresolvableException( + s"Could not resolve $operation for $topicFqn. ${Option(err.getMessage).getOrElse(err.getClass.getSimpleName)}", + err + ) + finally approximateLookupWorkersBusy.decrementAndGet() + ) + () + (0 until windowWidth).foreach(_ => submitNext()) + + def timedOut(): Nothing = + throw StartFromUnresolvableException( + s"Could not resolve $operation across ${topics.size} physical topic(s) within its ${budgetMs}ms " + + s"wall-clock budget. $remediation", + null + ) + + try + val answers = Map.newBuilder[String, A] + var remaining = topics.size + while remaining > 0 do + val waitNanos = deadlineNanos - System.nanoTime() + if waitNanos <= 0 then timedOut() + val future = completed.poll(waitNanos, TimeUnit.NANOSECONDS) + if future == null then timedOut() + try answers += future.get() + catch + case err: ExecutionException => throw Option(err.getCause).getOrElse(err) + remaining -= 1 + // One out, one in: the window stays exactly as wide as this operation is allowed to be. + submitNext() + answers.result() + catch + case err: InterruptedException => + Thread.currentThread.interrupt() + throw StartFromUnresolvableException(s"Could not resolve $operation because the request was interrupted.", err) + finally + // Nothing further is submitted, and what was submitted is cancelled. The executor is + // SHARED and therefore never shut down - cancellation is the only thing this operation + // is entitled to do to it. + nextToSubmit = topics.size + futures.foreach(_.cancel(true)) + +/** Resolve a retained-entry fraction for one physical topic. + * + * Interior values leave `floor(fraction * numberOfEntries)` entries behind and seek to the next + * 1-based entry. This is an ENTRY fraction, not a message percentile: producer batches can put + * different numbers of messages in different entries. The exact endpoints use Pulsar's Earliest + * and Latest positions and an empty topic falls back to Earliest for interior values. + * + * SERVER-ONLY MARKERS ARE IN THE DENOMINATOR AND ARE NOT REMOVED - stated plainly rather than + * implied to be handled. On a topic that uses transactions or geo-replication, every commit, + * abort and replicated-subscription snapshot is a real entry that `getNumberOfEntries` counts + * ([[retainedEntryCount]]) and that no consumer is ever handed, so "50%" is 50% of the entries + * INCLUDING those. The skew is the marker share of the log: a stream of small transactions can + * make it large (a two-message transaction adds one marker per two messages, so 50% of entries is + * roughly 50% of messages only because the markers are spread evenly; a topic whose markers + * cluster - a long transaction-free warm-up followed by heavy transactional traffic - lands the + * position further into the messages than the percentage says). It is NOT detected: finding the + * markers means examining entries, which is O(topic) and defeats the point of a mode that costs + * one counter read. The modes that are IMMUNE and are the remedy: 'Skip first n messages' counts + * what is actually delivered, and every time-based position ('Specific time', 'N units ago', + * 'Approximate position (% of time)') is addressed by timestamp and never by count. 'Latest n + * messages' is immune too, but for a different reason - it examines the entries it crosses and so + * can and does recognise a marker ([[logEntryOf]]). + * + * @throws IllegalArgumentException + * for NaN, an infinity, or a fraction outside [0.0, 1.0] - see [[requireFraction]]. + */ +def resolveApproximateEntryPosition(fraction: Double, numberOfEntries: Long): ApproximateEntrySeek = + requireFraction(fraction, "Approximate position (% of data)") + + if fraction <= 0.0 then ApproximateEntrySeek.Earliest + else if fraction >= 1.0 then ApproximateEntrySeek.Latest + else if numberOfEntries <= 0 then ApproximateEntrySeek.Earliest + else + // No clamp needed, and none added: the guards above leave 0 < fraction < 1 and + // numberOfEntries > 0, so floor(fraction * numberOfEntries) is between 0 and + // numberOfEntries - 1 and the ordinal lands inside [1, numberOfEntries] on its own. The + // range is pinned by test instead - a clamp here would have hidden a rounding change rather + // than caught it. + val entriesToLeaveBehind = math.floor(fraction * numberOfEntries).toLong + ApproximateEntrySeek.Entry(entriesToLeaveBehind + 1) + +/** How many entries `topicFqn` still holds. Entry-addressed, and answered from the managed ledger's + * own counters, so it costs the same on a topic of ten messages and on one of ten billion. + * + * ENTRIES, NOT MESSAGES, AND NOT EVEN ONLY USER ENTRIES: `PersistentTopic.getNumberOfEntries` + * counts a batched entry once however many messages it holds, and counts server-only marker + * entries (transaction commit/abort, replicated-subscription snapshots) that no consumer is ever + * handed. Its one caller is the entry-percentage mode, which is approximate by construction - see + * [[resolveApproximateEntryPosition]] for exactly what that skews and which modes are immune. + */ +def retainedEntryCount(adminClient: PulsarAdmin, topicFqn: String): Long = + adminClient.topics.getInternalStats(topicFqn).numberOfEntries + +/** The k-th ENTRY counted from the START of `topicFqn` (k = 1 is the first retained entry), as an + * entry-addressed id to seek to. + * + * `None` when the BROKER SAYS there is nothing there - an empty topic, which fails rather than + * answering. A broker that could not answer at all throws instead: the caller falls back to + * EARLIEST on `None`, which is honest for an entry that has aged out from under a stale entry + * count and dishonest for a transient 500. Counting from "earliest" past the END does not fail: it + * silently CLAMPS to the last entry, which is why the caller must never hand it an ordinal larger + * than the entry count. + */ +def entryFromEarliest(adminClient: PulsarAdmin, topicFqn: String)(entryOrdinal: Long): Option[PulsarMessageId] = + brokerAnswer(s"examining entry $entryOrdinal counted from the start", topicFqn)( + adminClient.topics.examineMessage(topicFqn, "earliest", entryOrdinal) + ).map(message => entryIdOf(message.getMessageId)) + +/** Resolve one entry-percentage seek per DISTINCT physical topic. + * + * The endpoints short-circuit before [[boundedParallelTopicLookup]], making 0% and 100% exactly as + * cheap and as available as Earliest and Latest. Interior positions are memoised by FQN: duplicate + * target consumers still each seek, but they share one broker-derived position and cannot drift + * apart while retention or appends move the log. + * + * THE RETENTION RE-CHECK LOOKS AT THE LOG'S FRONT, NOT AT ITS SIZE. Re-reading the entry COUNT + * answers only one of the two ways the log can move under a resolution: a topic that trimmed five + * entries off the front and appended five to the tail between the examine and the check has + * exactly the count it started with, an entirely different entry at that ordinal, and an anchor id + * that has been DELETED - and the count-only check accepted it. Comparing the resolved anchor with + * where the log now BEGINS catches that: an anchor older than the first retained entry is gone, + * and so is every anchor on a topic that now retains nothing. Both cases fall back to Earliest - + * the same answer this function already gives when the examine finds nothing, and the honest one: + * everything the topic still holds is newer than the position that was asked for. + * + * @param earliestRetainedEntryOf + * where the topic's retained log begins RIGHT NOW, or None when it retains nothing. + * @param entryIsOlder + * strict entry order - `entryIsOlder(anchor, first)` means the anchor has been trimmed away. + */ +def resolveApproximateEntrySeeks[A]( + fraction: Double, + topicFqns: Vector[String], + earliest: A, + latest: A, + retainedEntryCountOf: String => Long, + entryFromEarliestOf: String => Long => Option[A], + earliestRetainedEntryOf: String => Option[A], + entryIsOlder: (A, A) => Boolean, + budgetMs: Long = approximatePositionResolveBudgetMs, + parallelism: Int = approximatePositionLookupParallelism +): Map[String, A] = + requireFraction(fraction, "Approximate position (% of data)") + val topics = topicFqns.distinct + + if fraction <= 0.0 then topics.map(_ -> earliest).toMap + else if fraction >= 1.0 then topics.map(_ -> latest).toMap + else + boundedParallelTopicLookup( + topics, + operation = "'Approximate position (% of data)'", + budgetMs = budgetMs, + parallelism = parallelism, + lookup = topicFqn => + resolveApproximateEntryPosition(fraction, retainedEntryCountOf(topicFqn)) match + case ApproximateEntrySeek.Earliest => earliest + case ApproximateEntrySeek.Latest => latest + case ApproximateEntrySeek.Entry(entryOrdinal) => + entryFromEarliestOf(topicFqn)(entryOrdinal) match + case None => earliest + case Some(answer) => + // Retention can trim between the count and examine calls. Counting + // from earliest past the new end clamps to newest, so the ordinal + // must still exist - AND the entry it named must still be retained, + // which the count alone cannot say. See the note above. + val ordinalStillExists = retainedEntryCountOf(topicFqn) >= entryOrdinal + val anchorStillRetained = + earliestRetainedEntryOf(topicFqn).exists(first => !entryIsOlder(answer, first)) + if ordinalStillExists && anchorStillRetained then answer else earliest + ) + +/** Publish times observed on one physical topic's first and last retained entries. */ +final case class TopicPublishTimeSpan(firstPublishTimeMs: Long, lastPublishTimeMs: Long) + +/** Where an [[ApproximatePublishTimePosition]] lands. Unlike [[ApproximateEntrySeek]] this is one answer for + * a whole LOGICAL topic: every partition is seeked to the same instant. + */ +enum ApproximatePublishTimeSeek: + /** The very beginning of the retained log. */ + case Earliest + + /** The first message published at or after this epoch millisecond, i.e. an ordinary + * `Consumer.seek(timestamp)` - the same broker path the "Specific time" mode uses. + */ + case Timestamp(publishTimeMs: Long) + +/** Resolve a publish-time fraction over a set of physical topics into ONE instant. + * + * The inexpensive approximation inspects only each physical topic's first and last retained + * entries. It interpolates from the smallest observed first-boundary time to the largest observed + * last-boundary time, then seeks every topic to that timestamp. It does not scan interior + * messages, so producer clock changes can make the observed boundaries differ from the true + * minimum and maximum publish times. + * + * THE SET IT IS GIVEN IS THE SESSION'S WHOLE SELECTION, not one logical topic's partitions - see + * [[resolveApproximatePublishTimeSeeks]] for why the boundary is drawn there. + * + * 0% uses Earliest without a lookup. Empty or backwards boundary ranges also use Earliest; 100% + * seeks to the largest observed last-boundary time. Timestamp seeks are millisecond-granular and + * may include several messages with the same timestamp. + * + * @throws IllegalArgumentException + * for NaN, an infinity, or a fraction outside [0.0, 1.0] - see [[requireFraction]]. + */ +def resolveApproximatePublishTimePosition( + fraction: Double, + partitionFqns: Vector[String], + timeSpanOf: String => Option[TopicPublishTimeSpan] +): ApproximatePublishTimeSeek = + requireFraction(fraction, "Approximate position (% of time)") + + if fraction <= 0.0 then ApproximatePublishTimeSeek.Earliest + else + val spans = partitionFqns.distinct.flatMap(timeSpanOf) + if spans.isEmpty then ApproximatePublishTimeSeek.Earliest + else + val earliest = spans.map(_.firstPublishTimeMs).min + val latest = spans.map(_.lastPublishTimeMs).max + if fraction >= 1.0 then ApproximatePublishTimeSeek.Timestamp(latest) + else if latest <= earliest then ApproximatePublishTimeSeek.Earliest + else ApproximatePublishTimeSeek.Timestamp(earliest + math.floor(fraction * (latest - earliest)).toLong) + +/** The publish times observed on the first and last retained entries of `topicFqn`, or `None` when the + * BROKER SAYS it holds nothing - an empty topic, where `examineMessage` fails rather than + * answering. A non-persistent one (405) is ruled out by the caller before this is reached. + * + * A broker that could not answer THROWS rather than answering `None`. The range this feeds is + * min(first) .. max(last) across every partition, and a partition silently dropped out of it + * produced a confident cutoff over a narrower range - a different position, reported as success. + * + * This is deliberately a boundary lookup, not a scan for the true minimum or maximum timestamp. + * Producer clock changes can make interior timestamps fall outside the observed range. + * + * A boundary entry that is a SERVER-ONLY MARKER is not special-cased and does not need to be: the + * result of this lookup is a pair of TIMESTAMPS, and a marker carries an ordinary broker-stamped + * publish time from inside the log's own range. The mode built on it seeks by timestamp, so it + * never counts anything and cannot be skewed by how many entries are markers - unlike the + * entry-percentage mode, which is (see [[resolveApproximateEntryPosition]]). + */ +def publishTimeSpan(adminClient: PulsarAdmin, topicFqn: String): Option[TopicPublishTimeSpan] = + for + first <- brokerAnswer("reading the first retained message", topicFqn)(adminClient.topics.examineMessage(topicFqn, "earliest", 1L)) + last <- brokerAnswer("reading the last retained message", topicFqn)(adminClient.topics.examineMessage(topicFqn, "latest", 1L)) + yield TopicPublishTimeSpan(first.getPublishTime, last.getPublishTime) + +/** The LOGICAL topic a physical one belongs to: a partition's parent topic, or the topic itself. + * + * Only the `-partition-N` suffix Pulsar itself mints is stripped. A non-partitioned topic literally + * named `orders-partition-3` would be grouped under `orders` - the same ambiguity Pulsar's own + * `TopicName` carries, and the reason the broker refuses to create such a name. + */ +def logicalTopicOf(topicFqn: String): String = + val partitionOf = """^(.*)-partition-\d+$""".r + topicFqn match + case partitionOf(parent) => parent + case _ => topicFqn + +/** Group selected physical topics by logical topic, preserving first-seen order and removing + * duplicates inside each group. Kept separate from broker lookup so partition grouping is directly + * testable and cannot silently regress into one range per partition. */ +def groupPhysicalTopicsByLogicalTopic(topicFqns: Vector[String]): Vector[(String, Vector[String])] = + val groups = scala.collection.mutable.LinkedHashMap.empty[String, Vector[String]] + topicFqns.distinct.foreach { topicFqn => + val logical = logicalTopicOf(topicFqn) + groups.update(logical, groups.getOrElse(logical, Vector.empty) :+ topicFqn) + } + groups.toVector + +/** Resolve the session's publish-time-percentage position: ONE instant, keyed by logical topic so + * every caller can look it up the way it already does. Every physical span is looked up at most + * once, with bounded concurrency and one deadline, before any caller applies a seek. + * + * ONE CUTOFF FOR THE WHOLE SELECTION, and that is the point of the mode. The range used to be + * pooled per LOGICAL topic, so a session over a one-hour topic and a thirty-day topic answered + * "50%" with two wall-clock instants a fortnight apart: each topic began half way through ITS OWN + * history, the merged view was a slice of no particular period, and nothing on screen could be + * compared with anything else. A history percentage over a set of topics only means something if + * the history it is a percentage OF is the set's. + * + * The ENTRY-percentage mode stays deliberately per physical topic: it answers a different question + * - where in each topic's stored data to begin - which has no cross-topic instant to share. + * + * A topic whose boundaries the broker would not describe (it retains nothing) contributes no + * boundary and receives the session's cutoff like every other: dropping out of the range would + * narrow it, and being given a different position would reintroduce exactly what this fixes. + */ +def resolveApproximatePublishTimeSeeks( + fraction: Double, + topicFqns: Vector[String], + timeSpanOf: String => Option[TopicPublishTimeSpan], + budgetMs: Long = approximatePositionResolveBudgetMs, + parallelism: Int = approximatePositionLookupParallelism +): Map[String, ApproximatePublishTimeSeek] = + requireFraction(fraction, "Approximate position (% of time)") + val groups = groupPhysicalTopicsByLogicalTopic(topicFqns) + + if fraction <= 0.0 then groups.map((logical, _) => logical -> ApproximatePublishTimeSeek.Earliest).toMap + else + val physicalTopics = groups.flatMap(_._2) + val spans = boundedParallelTopicLookup( + physicalTopics, + operation = "'Approximate position (% of time)'", + budgetMs = budgetMs, + parallelism = parallelism, + lookup = timeSpanOf + ) + val sessionSeek = resolveApproximatePublishTimePosition(fraction, physicalTopics, spans.apply) + groups.map((logical, _) => logical -> sessionSeek).toMap def getIsSingleNonPartitionedTopic(adminClient: PulsarAdmin, topics: Vector[String]): Boolean = - topics.size == 1 && getTopicPartitioning(adminClient, topics.head) == TopicPartitioningType.NonPartitioned + // `.type` is essential: getTopicPartitioning returns a TopicPartitioning record, and comparing + // the whole record to a TopicPartitioningType was ALWAYS false - which silently disabled the + // single-topic fast path, so every start-from seeked by publishTime instead of by message id. + topics.size == 1 && getTopicPartitioning(adminClient, topics.head).`type` == TopicPartitioningType.NonPartitioned + +/** Resolve a "N units ago" start position against a supplied `now`. + * + * Split out of the seek path (and given an explicit `now`) so all unit x rounding combinations are + * testable with a frozen clock. `ZonedDateTime.truncatedTo` REJECTS units larger than a day, so + * Week/Month/Year must be rounded with date adjusters - passing them to truncatedTo threw + * UnsupportedTemporalTypeException, i.e. "1 month ago, rounded to the start of the month" - an + * ordinary UI selection - failed the whole session with a generic error. + */ +def resolveRelativeDateTime(v: RelativeDateTime, now: ZonedDateTime): ZonedDateTime = + import java.time.temporal.ChronoUnit + v.unit match + case DateTimeUnit.Year => + val dt = now.minusYears(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1) else dt + case DateTimeUnit.Month => + val dt = now.minusMonths(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1) else dt + case DateTimeUnit.Week => + val dt = now.minusWeeks(v.value) + if v.isRoundedToUnitStart then + dt.truncatedTo(ChronoUnit.DAYS).`with`(java.time.temporal.TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY)) + else dt + case DateTimeUnit.Day => + val dt = now.minusDays(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.DAYS) else dt + case DateTimeUnit.Hour => + val dt = now.minusHours(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.HOURS) else dt + case DateTimeUnit.Minute => + val dt = now.minusMinutes(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.MINUTES) else dt + case DateTimeUnit.Second => + val dt = now.minusSeconds(v.value) + if v.isRoundedToUnitStart then dt.truncatedTo(ChronoUnit.SECONDS) else dt +/** The message `messageId` names in `topicFqn`, or `None` if that topic genuinely does not hold it. + * + * `None` MEANS "NOT THERE", AND NOTHING ELSE. Every operational failure used to collapse into the + * same `None` - a message id that could not be parsed at all, a reader the broker refused to + * create, an unreachable topic, a read that timed out - and the caller then reported "Message with + * such ID not found", which is a diagnosis of the user's input for what was a fault in the server + * or the broker. Worse, on a multi-topic session it made an unreachable topic indistinguishable + * from one that simply does not hold the id, so the session could be positioned from whichever + * topics happened to answer. + * + * A malformed id is an INVALID ARGUMENT (the client sent something that is not a message id at + * all); everything else that prevents an answer is a [[StartFromUnresolvableException]]. + */ def getMessageById(pulsarClient: PulsarClient, topicFqn: String, messageId: Array[Byte]): Option[PulsarMessage[Array[Byte]]] = val subscriptionName = s"dekaf_${java.util.UUID.randomUUID.toString}" val resolvedMessageId = Try(PulsarMessageId.fromByteArrayWithTopic(messageId, topicFqn)) match case Success(messageId) => messageId - case Failure(_) => - return None + case Failure(err) => + throw new IllegalArgumentException( + s"Start-from message id could not be read: it is not a Pulsar message id (${messageId.length} bytes). ${err.getMessage}", + err + ) + + def unresolvable(what: String, err: Throwable): Nothing = + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: $what failed for $topicFqn. ${err.getMessage}", + err + ) val reader = Try { pulsarClient @@ -70,21 +1204,62 @@ def getMessageById(pulsarClient: PulsarClient, topicFqn: String, messageId: Arra .subscriptionName(subscriptionName) .create() } match - case Success(reader) => reader - case Failure(err) => - logger.error(s"Failed to create reader for topic $topicFqn", err) - return None + case Success(reader) => reader + case Failure(err) => unresolvable("opening a reader at the requested message id", err) try - if reader.hasMessageAvailable then - val message = reader.readNext(5, java.util.concurrent.TimeUnit.SECONDS) - if message.getMessageId.toByteArray sameElements messageId then Some(message) - else None - else None - catch { - case _: Throwable => None - } finally - reader.close() + val hasMessage = Try(reader.hasMessageAvailable) match + case Success(available) => available + case Failure(err) => unresolvable("asking whether the requested message id is still retained", err) + + if !hasMessage then None + else + Try(reader.readNext(5, java.util.concurrent.TimeUnit.SECONDS)) match + // `readNext` answers with null when the wait ran out. The broker said the message + // was there and then did not hand it over, which is a fault and not an absence. + case Success(null) => unresolvable("reading the message at the requested message id", new java.util.concurrent.TimeoutException("the read timed out after 5s")) + case Success(message) => Option.when(message.getMessageId.toByteArray sameElements messageId)(message) + case Failure(err) => unresolvable("reading the message at the requested message id", err) + finally Try(reader.close()) + +/** How ONE consumer is positioned for a Message-ID start-from, once the message has been found. */ +enum MessageIdSeek: + /** The exact message. Only the topic that OWNS the id can be positioned this way - a Pulsar + * message id addresses a ledger and entry of one topic and means nothing in another. */ + case ById(messageId: PulsarMessageId) + + /** The message's publish INSTANT - the only position that exists across topics, and only + * approximately the same place: everything published in that same millisecond is included. */ + case ByPublishTime(atMs: Long) + +/** Where each of a session's consumers starts, for a start-from that names one message id. + * + * THE OWNING TOPIC IS SEEKED BY ID, EXACTLY. It used to be seeked by publish time along with every + * other topic as soon as the session covered more than one, so "start from this message" silently + * became "start from this millisecond" even on the topic the user picked the message from - and any + * earlier message sharing that millisecond, or sharing its producer batch, came with it. + * + * EVERY OTHER TOPIC IS SEEKED BY PUBLISH TIME, and that is a real approximation rather than an + * oversight: a message id is unique only within one topic, so there is no exact corresponding + * position in the others. The instant is the closest thing that exists, and it is inclusive of + * everything stamped with the same millisecond. + * + * PURE, so both halves are pinned without a broker. Two targets on the owning topic each have their + * own consumer and both get the exact id. + */ +def messageIdSeeks[C]( + consumers: Vector[C], + topicOf: C => String, + ownerTopicFqn: String, + messageId: PulsarMessageId, + publishTime: Long +): Vector[(C, MessageIdSeek)] = + consumers.map { consumer => + val seek = + if topicOf(consumer) == ownerTopicFqn then MessageIdSeek.ById(messageId) + else MessageIdSeek.ByPublishTime(publishTime) + consumer -> seek + } def topicsToNonPartitionedTopic(pulsarAdmin: PulsarAdmin, topics: Vector[String]) = topics.flatMap { topicFqn => @@ -93,95 +1268,320 @@ def topicsToNonPartitionedTopic(pulsarAdmin: PulsarAdmin, topics: Vector[String] case TopicPartitioningType.Partitioned => getPartitions(pulsarAdmin, topicFqn) } +/** The one message a Message-ID start-from resolves to, looked up across the physical topics the + * session covers. + * + * DISTINCT, and that is the whole point: a session's topic vector is the CONCATENATION of every + * enabled target's resolved topics, and two targets may legitimately select the same topic - each + * has its own consumer, filters and colouring. Looking the id up once per NAME rather than once + * per physical topic read the same message twice and refused a valid session with "Multiple + * messages found for the same message id". Every target's consumer is still seeked; only the + * LOOKUP is deduplicated. + * + * Two GENUINELY different topics answering is still refused: a message id is only unique within + * one topic, so there is no way to know which of them the user meant. + * + * PURE: the broker sits behind `lookup`, so the shapes that matter - nothing found, one found, the + * same physical topic named twice, two genuinely different topics answering - are all testable + * with a plain lambda. + */ +def resolveMessageIdAcrossTopics[M](topicFqns: Vector[String], lookup: String => Option[M]): Option[M] = + topicFqns.distinct.flatMap(topicFqn => lookup(topicFqn)) match + case Vector() => None + case Vector(msg) => Some(msg) + case _ => throw new RuntimeException("Multiple messages found for the same message id") + +/** The one message the id names, together with the topic that OWNS it - which is what lets that + * topic be seeked exactly while the rest are seeked by publish time (see [[messageIdSeeks]]). */ def getMessageByIdMultiTopic( pulsarAdmin: PulsarAdmin, pulsarClient: PulsarClient, nonPartitionedTopicFqns: Vector[String], messageId: Array[Byte] -): Option[PulsarMessage[Array[Byte]]] = - val messageIds = nonPartitionedTopicFqns.flatMap(topicFqn => getMessageById(pulsarClient, topicFqn, messageId)) - - messageIds match - case Vector() => None - case Vector(msg) => Some(msg) - case _ => throw new RuntimeException("Multiple messages found for the same message id") +): Option[(String, PulsarMessage[Array[Byte]])] = + resolveMessageIdAcrossTopics( + nonPartitionedTopicFqns, + topicFqn => getMessageById(pulsarClient, topicFqn, messageId).map(message => topicFqn -> message) + ) +/** Seek every consumer of a session to its start position, and report what the seek could not + * express exactly: messages to discard from the head of a stream, and the global reordering the two + * counting modes need on top. + * + * The two counting modes are EXACT under batching and across partitions, and both are GLOBAL: they + * count the session's whole merged stream, ordered by publish time. Neither can be done with a seek + * alone, because a seek only ever lands on an entry boundary - see [[StartFromDiscardPlan]]. + * + * "EXACT" IS ABOUT THE COUNT WITHOUT QUALIFICATION, and about WHICH messages only as far as the + * logs really are in publish-time order: publish time is stamped by the producer, and Pulsar + * preserves append order within a partition rather than clock order. [[MessageOrderKey]] states + * precisely what is and is not guaranteed, including that delivery SEQUENCE after a skip's cut is + * the brokers' order rather than the global one. + * + * - "Skip first n messages" ([[NthMessageAfterEarliest]]): seek everything to the very beginning, + * then drop the GLOBALLY-FIRST n by publish time across every physical topic and deliver the + * rest. On a single log that is exactly "start at message n+1" and costs nothing but a head-drop + * counter. Across partitions it is a streaming k-way merge over one held message per topic + * ([[GlobalSkipMerge]]) - the count is exactly n, the choice of WHICH n is exact as far as the + * logs are in publish-time order, and n is never buffered, because "skip first n" has + * deliberately no cap. + * + * - "Latest n messages" ([[NthMessageBeforeLatest]]): ONE MERGED BACKWARD WALK over every + * physical topic's entry metadata ([[resolveLatestN]]) - take whichever topic's current entry + * was published latest, count its messages, step that topic back one entry, until n messages + * are accounted for. Each topic is then seeked to the oldest entry the walk took from it (or to + * LATEST if it contributed none), and the overshoot inside the single entry the walk stopped on + * is dropped from that topic's head. NOTHING IS BUFFERED and no delivered message takes part in + * the decision: the cut is known before the consumers are resumed. Memory is O(number of + * topics) and cost is O(n / batch size + topics) admin lookups. A session over p partitions + * shows exactly n messages and not n * p, and it starts showing them immediately rather than + * waiting for every partition to drain. + * + * - "Approximate position (% of data)" ([[ApproximateEntryPosition]]): PER PHYSICAL TOPIC, and deliberately + * stays that way. The fraction is resolved against that topic's own entry count and seeked to, + * so a session 60% of the way into a 4-partition topic is 60% into each of the four logs, and a + * session over several enabled targets is 60% into every topic they resolve to. That is not the + * same position as "60% of the merged stream" unless the logs are the same size. Unlike the two + * counting modes it cannot be made global cheaply: a count of n can be reached by streaming n + * messages and stopping, whereas a FRACTION of the merged stream is only known once the whole of + * it has been measured, which is O(topic) at any n. Rounding, endpoints, why the position is + * only approximate, and empty topics: see [[resolveApproximateEntryPosition]]. + * + * - "Approximate position (% of time)" ([[ApproximatePublishTimePosition]]): PER LOGICAL TOPIC. The topic's + * partitions contribute their first- and last-entry boundary timestamps, and every partition + * is seeked to the single instant selected from that observed range. Still per topic and not + * per session: two enabled targets position against their own backlogs, not against a merged + * view. The seek is BY TIMESTAMP, so it rides the same broker path as [[DateTime]]. Endpoints, + * clock-skew limits and empty topics: see [[resolveApproximatePublishTimePosition]]. + * + * The two approximate modes answer different questions: one uses retained-entry position and + * the other interpolates between observed publish-time boundaries. + * + * NON-PERSISTENT TOPICS keep nothing, so no history position exists on one. If EVERY topic the + * session resolved to is non-persistent, a history mode is REJECTED up front (see + * [[startFromRejectionReason]]) rather than degrading into a seek that silently means "from now". + * A MIXED session is not rejected: the persistent topics get the position that was asked for and + * the non-persistent ones are seeked to latest, which is the only position they have. One wrinkle + * follows from that - "skip first n" counts the MERGED DELIVERED stream, so on a mixed session + * anything the non-persistent topics deliver live counts towards its n as well. Such a topic holds + * no backlog, so it never holds the merge up waiting for a head it will not produce. "Latest n" + * counts stored entries instead of delivered messages, so a non-persistent topic contributes + * nothing to its n and simply streams alongside the historical tail. + */ def handleStartFrom( startFrom: ConsumerSessionStartFrom, consumers: Vector[Consumer[Array[Byte]]], adminClient: PulsarAdmin, pulsarClient: PulsarClient, - nonPartitionedTopicFqns: Vector[String] -): Unit = - consumers.foreach(_.resume()) + nonPartitionedTopicFqns: Vector[String], + // The session-level delivery-order choice. It rides on the + // start-from plan because the ordering layer is built exactly once, here - a counted skip + // keeps its exact merge and simply does not let go of it after the cut; every other mode + // gains an order-only layer over the same machinery. + deliveryOrdering: consumer.session_config.MessageDeliveryOrder = consumer.session_config.MessageDeliveryOrder.AsReceived +): StartFromPlan = + // Before anything is resumed or seeked: a position that cannot exist on these topics is a + // validation error, not something to discover halfway through seeking them. + startFromRejectionReason(startFrom, nonPartitionedTopicFqns).foreach(reason => throw new IllegalArgumentException(reason)) + startFromCountRejectionReason(startFrom).foreach(reason => throw new IllegalArgumentException(reason)) - startFrom match + // The consumers STAY PAUSED for all of this. They used to be resumed here, which opened a + // delivery window over every broker round trip below - the backward entry walk, reading each + // topic's last message id - while the session was still being built: its counters and ordering + // layer are armed by the caller AFTER this returns, and its message handler is still a no-op. + // Whatever arrived in that window was therefore consumed by nobody, and a session could swallow + // its own backlog and then deliver nothing. Seeking does not need a running consumer; pausing + // only withholds flow permits. + + // Everything below positions the topics that HAVE a history. A non-persistent topic can only + // start from now, whatever the session asked for. + val (historyConsumers, liveOnlyConsumers) = splitByRetainedHistory(consumers, _.getTopic) + // DISTINCT: two enabled targets may select the same topic, and the session's FQN vector is the + // concatenation of what each resolved to. The CONSUMERS are deliberately not deduplicated - + // every one of them still has to be seeked - but a topic that is asked ABOUT twice pushed the + // single-topic fast path off (`size == 1`) and made the Message-ID lookup read one physical + // message once per name. + val historyTopicFqns = nonPartitionedTopicFqns.filterNot(isNonPersistentTopic).distinct + liveOnlyConsumers.foreach(_.seek(PulsarMessageId.latest)) + + // Resolving the streams costs one broker call per topic, so the gate is checked FIRST and the + // layer is only built for a session that will actually use one. + val isGlobal = needsGlobalOrdering(startFrom, consumers.size) + lazy val globalOrdering: StartFromOrderingPlan = + if isGlobal then globalOrderingPlanFor(startFrom, startFromStreamsAt(consumers), afterCut = deliveryOrdering) + else StartFromOrderingPlan.PassThrough + + // Topics this start-from positions AT THE LIVE EDGE - nothing retained behind the cursor, so + // under the GUARANTEED replay they have nothing to replay and their boundary is EMPTY (the + // instant-caught-up input). Populated by the branches whose seek decision already knows it + // for free: Latest (everything), latest-n (its non-contributing topics), and the approximate + // entry position's endpoint fast path. A time-based seek past a topic's end cannot be told + // apart without extra broker reads and is deliberately not classified: an idle such stream + // holds its replay and is disclosed through the stall machinery instead. + val seekedToLiveEdge = scala.collection.mutable.Set.empty[String] + + val plan: StartFromPlan = startFrom match case _: EarliestMessage => - consumers.foreach(_.seek(PulsarMessageId.earliest)) + historyConsumers.foreach(_.seek(PulsarMessageId.earliest)) + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) case _: LatestMessage => - consumers.foreach(_.seek(PulsarMessageId.latest)) + historyConsumers.foreach(_.seek(PulsarMessageId.latest)) + seekedToLiveEdge ++= historyTopicFqns + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) case v: NthMessageAfterEarliest => - val n = v.n - if getIsSingleNonPartitionedTopic(adminClient, nonPartitionedTopicFqns) then - findNthMessage(adminClient, nonPartitionedTopicFqns.head, "earliest", n) match - case Some(message) => consumers.foreach(_.seek(message.getMessageId)) - case None => consumers.foreach(_.seek(PulsarMessageId.latest)) - else - findNthMessageMultiTopic(adminClient, nonPartitionedTopicFqns, "earliest", n) match - case Some(message) => consumers.foreach(_.seek(message.getPublishTime)) - case None => consumers.foreach(_.seek(PulsarMessageId.latest)) + // Seek + discard, NOT examineMessage. examineMessage is ENTRY-addressed and clamps past + // the end without failing, so "skip 5" on a topic whose 100 messages are one batched + // entry used to ask for entry 6, get the last entry, and skip fifty. Streaming n + // messages and throwing them away is O(n) - and n is a number the user typed. + historyConsumers.foreach(_.seek(PulsarMessageId.earliest)) + if isGlobal then + // The merge owns the counter, so the listener must NOT also hold one: two counters + // over the same messages would drop 2n. It is the progress source as well - the + // number the client is shown is the merge's own budget draining. + StartFromPlan(StartFromDiscardPlan.Nothing, globalOrdering) + else if v.n > 0 then StartFromPlan(StartFromDiscardPlan.SharedTotal(v.n), StartFromOrderingPlan.PassThrough) + else StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) case v: NthMessageBeforeLatest => - val n = v.n - if getIsSingleNonPartitionedTopic(adminClient, nonPartitionedTopicFqns) then - findNthMessage(adminClient, nonPartitionedTopicFqns.head, "latest", n) match - case Some(message) => consumers.foreach(_.seek(message.getMessageId)) - case None => consumers.foreach(_.seek(PulsarMessageId.earliest)) + if v.n <= 0 then + // "the last 0 messages" is nothing at all - the same position "Latest message" uses. + historyConsumers.foreach(_.seek(PulsarMessageId.latest)) + seekedToLiveEdge ++= historyTopicFqns + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) else - findNthMessageMultiTopic(adminClient, nonPartitionedTopicFqns, "latest", n) match - case Some(message) => consumers.foreach(_.seek(message.getPublishTime)) - case None => consumers.foreach(_.seek(PulsarMessageId.earliest)) + // RESOLVE EVERY TOPIC BEFORE SEEKING ANY OF THEM - the same discipline both + // approximate modes follow, and for the same reason: a broker that stops answering + // half way through must not leave part of the session sitting at a position nobody + // asked for. + val cut = resolveLatestN(v.n, historyConsumers.map(_.getTopic).distinct, entryFromLatest(adminClient, _), latestNEntryIsOlder) + // RETENTION RE-CHECK, after resolving and before any seek: see + // [[latestNAnchorRejectionReason]]. Every topic verifies before any topic seeks - + // the same all-or-nothing discipline the resolution itself follows. + latestNAnchorRejectionReason(cut, earliestRetainedEntryId(adminClient, _), latestNEntryIsOlder) + .foreach(reason => throw StartFromUnresolvableException(reason, null)) + val discards = scala.collection.mutable.Map.empty[NonPartitionedTopicFqn, Long] + historyConsumers.foreach { consumer => + val topicFqn = consumer.getTopic + cut.getOrElse(topicFqn, LatestNSeek.Nothing) match + // Everything this topic holds is older than the cut. LATEST, not earliest: + // seeking a non-contributing partition to the beginning would show all of it. + case LatestNSeek.Nothing => + consumer.seek(PulsarMessageId.latest) + seekedToLiveEdge += topicFqn + case LatestNSeek.Everything => consumer.seek(PulsarMessageId.earliest) + case LatestNSeek.FromEntry(entryId, discard) => + consumer.seek(entryId) + if discard > 0 then discards(topicFqn) = discard + } + // A PER-TOPIC head-drop and never a session-wide one: this is the overshoot inside + // the single entry the walk stopped on, not a skip the user asked for, so it must + // not be reported as progress. Two targets on that topic each drop their own copy. + StartFromPlan(StartFromDiscardPlan.PerTopic(discards.toMap), StartFromOrderingPlan.PassThrough) case v: MessageId => - if getIsSingleNonPartitionedTopic(adminClient, nonPartitionedTopicFqns) then + if getIsSingleNonPartitionedTopic(adminClient, historyTopicFqns) then val messageId = MessageId.toPulsar(v).getOrElse(throw new RuntimeException(s"Failed to parse message ID.")) - val topicFqn = nonPartitionedTopicFqns.head + val topicFqn = historyTopicFqns.head getMessageById(pulsarClient, topicFqn, messageId.toByteArray) match - case Some(message) => consumers.foreach(_.seek(message.getMessageId)) + case Some(message) => historyConsumers.foreach(_.seek(message.getMessageId)) case None => throw new RuntimeException(s"Message with such ID not found in the topic: $topicFqn.") else - getMessageByIdMultiTopic(adminClient, pulsarClient, nonPartitionedTopicFqns, v.messageIdBytes) match - case Some(message) => - consumers.foreach(_.seek(message.getPublishTime)) + getMessageByIdMultiTopic(adminClient, pulsarClient, historyTopicFqns, v.messageIdBytes) match + case Some((ownerTopicFqn, message)) => + // The topic the id belongs to gets the EXACT message; the others get the + // instant it was published at, which is the only cross-topic position a + // message id has. See [[messageIdSeeks]]. + messageIdSeeks(historyConsumers, _.getTopic, ownerTopicFqn, message.getMessageId, message.getPublishTime) + .foreach { (consumer, seekTo) => + seekTo match + case MessageIdSeek.ById(messageId) => consumer.seek(messageId) + case MessageIdSeek.ByPublishTime(atMs) => consumer.seek(atMs) + } case None => throw new RuntimeException(s"Message with such ID not found.") + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) case v: DateTime => val timestamp = v.dateTime.toEpochMilli - consumers.foreach(_.seek(timestamp)) + historyConsumers.foreach(_.seek(timestamp)) + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) case v: RelativeDateTime => - val now = ZonedDateTime.now() - val dateTime = v.unit match - case DateTimeUnit.Year => - val dt = now.minusYears(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.YEARS) else dt - case DateTimeUnit.Month => - val dt = now.minusMonths(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.MONTHS) else dt - case DateTimeUnit.Week => - val dt = now.minusWeeks(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.WEEKS) else dt - case DateTimeUnit.Day => - val dt = now.minusDays(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.DAYS) else dt - case DateTimeUnit.Hour => - val dt = now.minusHours(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.HOURS) else dt - case DateTimeUnit.Minute => - val dt = now.minusMinutes(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.MINUTES) else dt - case DateTimeUnit.Second => - val dt = now.minusSeconds(v.value) - if v.isRoundedToUnitStart then dt.truncatedTo(java.time.temporal.ChronoUnit.SECONDS) else dt - consumers.foreach(_.seek(dateTime.toInstant.toEpochMilli)) + // Resolve ONCE, outside the loop: evaluating now() per consumer seeks each partition of + // a multi-topic session to a slightly different boundary. + val startAt = resolveRelativeDateTime(v, ZonedDateTime.now()).toInstant.toEpochMilli + historyConsumers.foreach(_.seek(startAt)) + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) + + case v: ApproximateEntryPosition => + // Resolve EVERY DISTINCT physical topic before seeking any consumer. Duplicate target + // consumers share one answer, and the endpoint fast paths ask the broker nothing. + val seeksByPhysicalTopic = resolveApproximateEntrySeeks( + fraction = v.fraction, + topicFqns = historyConsumers.map(_.getTopic), + earliest = PulsarMessageId.earliest, + latest = PulsarMessageId.latest, + retainedEntryCountOf = retainedEntryCount(adminClient, _), + entryFromEarliestOf = entryFromEarliest(adminClient, _), + // The same lookup and the same order the latest-n anchor re-check uses - one more + // O(1) admin call per interior position, which is what it costs to know that the + // entry chosen a moment ago has not been trimmed away since. + earliestRetainedEntryOf = earliestRetainedEntryId(adminClient, _), + entryIsOlder = latestNEntryIsOlder + ) + historyConsumers.foreach(consumer => consumer.seek(seeksByPhysicalTopic(consumer.getTopic))) + // The endpoint fast path answers LATEST for fraction 1.0 (and for an empty topic): + // such a topic replays nothing under Guaranteed, and the resolver already said so. + seeksByPhysicalTopic.foreach((topicFqn, seekTo) => if seekTo == PulsarMessageId.latest then seekedToLiveEdge += topicFqn) + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) + + case v: ApproximatePublishTimePosition => + // ONE bounded, memoised lookup per physical topic; then ONE instant for the whole + // session, which every topic seeks to. Every answer exists before any consumer seeks, + // preserving all-or-nothing positioning. + val seeksByTopic = resolveApproximatePublishTimeSeeks( + fraction = v.fraction, + topicFqns = historyConsumers.map(_.getTopic), + timeSpanOf = publishTimeSpan(adminClient, _) + ) + + historyConsumers.foreach { consumer => + seeksByTopic(logicalTopicOf(consumer.getTopic)) match + case ApproximatePublishTimeSeek.Earliest => consumer.seek(PulsarMessageId.earliest) + case ApproximatePublishTimeSeek.Timestamp(atMs) => consumer.seek(atMs) + } + StartFromPlan(StartFromDiscardPlan.Nothing, StartFromOrderingPlan.PassThrough) + + // Add continuous delivery ordering for modes that decided no ordering of their own. BEST + // EFFORT keeps its old shape: a layer over stream ids with EMPTY ends (no broker calls), and + // only when there is something to interleave - a single stream is already in order and stays + // on the free pass-through path. GUARANTEED builds its layer at ANY stream count (the replay + // boundary, the auto-pause and the caught-up signal live in it) and resolves the REAL + // recorded ends - one broker call per topic, except for the streams the seek itself already + // proved are at the live edge, whose boundary is empty without asking. + val planWithMessageDeliveryOrder = + val guaranteed = deliveryOrdering == consumer.session_config.MessageDeliveryOrder.Guaranteed + if deliveryOrdering != consumer.session_config.MessageDeliveryOrder.AsReceived + && plan.ordering == StartFromOrderingPlan.PassThrough + && (consumers.size > 1 || guaranteed) + then + val orderedStreams = + if guaranteed then + // Skip the per-topic end lookups entirely when every stream's boundary is + // already known to be empty - Latest costs no broker call at all. + val boundaryNeeded = consumers.exists(c => !isNonPersistentTopic(c.getTopic) && !seekedToLiveEdge.contains(c.getTopic)) + val capturedEnds: Map[String, EntryPosition] = + if boundaryNeeded then startFromStreamsAt(consumers).map(s => s.id -> s.lastAtStart).toMap + else Map.empty + consumers.map { c => + val id = startFromStreamId(c.getConsumerName, c.getTopic) + val end = + if seekedToLiveEdge.contains(c.getTopic) then EntryPosition.empty + else capturedEnds.getOrElse(id, EntryPosition.empty) + StartFromStream(id, end) + } + else consumers.map(c => StartFromStream(startFromStreamId(c.getConsumerName, c.getTopic), EntryPosition.empty)) + plan.copy(ordering = StartFromOrderingPlan.Ordered(orderedStreams, deliveryOrdering)) + else plan + // Belt and braces: they should never have been resumed, and the caller resumes them itself. consumers.foreach(_.pause()) + planWithMessageDeliveryOrder diff --git a/server/src/main/scala/consumer/session_runner/messageConverters.scala b/server/src/main/scala/consumer/session_runner/messageConverters.scala index 02b940f69..623c2cd65 100644 --- a/server/src/main/scala/consumer/session_runner/messageConverters.scala +++ b/server/src/main/scala/consumer/session_runner/messageConverters.scala @@ -57,11 +57,11 @@ object converters: val messageId = Option(msg.getMessageId.toByteArray) val sequenceId = Option(msg.getSequenceId) val producerName = Option(msg.getProducerName) - val key = Option(msg.getKey).flatMap(key => { - parseJson(s"""\"$key\"""") match - case Left(_) => None - case Right(k) => Some(k) - }) + // Encode via circe rather than splicing the raw key into a JSON string literal: a key + // containing a quote/backslash/newline produced invalid JSON, which parsed to Left and + // silently DROPPED the key - so key filters and key projections quietly missed those + // messages. (The value path already goes through primitiveConv.bytesToJsonString.) + val key = Option(msg.getKey).map(_.asJson) val size = Option(msg.size) val orderingKey = Option(msg.getOrderingKey) val topic = Option(msg.getTopicName) diff --git a/server/src/main/scala/consumer/session_runner/startFromLookups.scala b/server/src/main/scala/consumer/session_runner/startFromLookups.scala new file mode 100644 index 000000000..32f45bc03 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/startFromLookups.scala @@ -0,0 +1,92 @@ +package consumer.session_runner + +import org.apache.pulsar.client.admin.PulsarAdminException + +import scala.util.{Failure, Success, Try} + +/** The broker could not answer a question the requested start position depends on. + * + * Distinct from "the log holds nothing there", which is an ANSWER. A session that cannot be put + * where the user asked for it must fail to be created, rather than start somewhere else and report + * success. + */ +final class StartFromUnresolvableException(message: String, cause: Throwable) extends RuntimeException(message, cause) + +/** The two phrases the broker answers "there is nothing there" with. Lower-cased at the point of + * comparison, so a version that changes the capitalisation still matches. */ +private val emptyLogPhrases: Vector[String] = Vector("total message is zero", "incorrect parameter input") + +/** HTTP 412: `examinemessage`'s only precondition is that the log holds something. */ +private val preconditionFailed: Int = 412 + +private def causeChain(err: Throwable): Vector[Throwable] = + // Bounded: a cause chain that references itself is not unheard of, and this runs on the session + // creation path. + Iterator.iterate(err)(_.getCause).takeWhile(_ != null).take(10).toVector + +private def saysEmptyLog(message: String): Boolean = + Option(message).map(_.toLowerCase).exists(text => emptyLogPhrases.exists(text.contains)) + +/** Whether an admin failure is the broker SAYING THERE IS NOTHING THERE, rather than failing to + * answer at all. + * + * `PulsarAdmin.topics.examineMessage` has no "empty" answer: both of its legitimate "no such + * entry" outcomes arrive as failures, and everything else that can go wrong arrives the same way. + * Erasing all of them into one `None` is what turned a transient broker error into a DIFFERENT + * VALID START POSITION - "Latest 5" quietly showed the entire backlog, and creation reported + * success. + * + * MEASURED against the Pulsar this repo runs its e2e against (3.2.1), through `PulsarAdmin` + * itself, on a non-partitioned persistent topic holding 3 unbatched messages: + * + * - EMPTY TOPIC, `earliest/1`: `PreconditionFailedException`, statusCode 412, message and + * httpError both "Could not examine messages due to the total message is zero". + * - PAST THE START, `latest/99`: `ServerSideErrorException`, statusCode 500, message and + * httpError both "... Message: Incorrect parameter input error code: -14 ... + * org.apache.bookkeeper.mledger.ManagedLedgerException ...". + * - MISSING TOPIC: `NotFoundException`, statusCode 404, "Topic ... not found". + * - UNREACHABLE BROKER: plain `PulsarAdminException`, **statusCode 500**, httpError NULL, + * message "...RetryException: Could not complete the operation. Number of retries has been + * exhausted...". + * - PAST THE END, `earliest/4` and `earliest/99`: HTTP 200, silently CLAMPED to the last entry. + * That is why only the `latest` side is ever walked, and why [[resolveLatestN]] guards the + * clamp with a repeated-entry check rather than relying on a failure. + * + * THE STATUS CODE ALONE CANNOT CLASSIFY, and that last measurement is why: a broker that cannot be + * reached at all reports the same 500 as a walk that ran off the start of the log. Only 412 is + * unambiguous; the 500 has to be told apart by what it says. Both `getMessage` and `getHttpError` + * are checked because the admin client fills them from the same server reason but leaves + * `httpError` null when the failure never reached the server. + * + * EVERYTHING ELSE IS AN OPERATIONAL FAILURE: a timeout, a 401/403, a 404, a broker restarting + * mid-request, a 500 that says something else. None of those means the log is empty, and none of + * them may be answered with a position the user did not ask for. + * + * PURE, so every one of these shapes is pinned by test. + */ +def isEmptyLogAnswer(err: Throwable): Boolean = + causeChain(err).exists { + case admin: PulsarAdminException => + admin.getStatusCode == preconditionFailed || saysEmptyLog(admin.getMessage) || saysEmptyLog(admin.getHttpError) + case other => saysEmptyLog(other.getMessage) + } + +/** Ask the broker one question about a topic's log, keeping "there is nothing there" and "it could + * not say" apart. + * + * `Some(answer)` - the broker answered. `None` - the broker answered, and the answer is that there + * is nothing at that position. A throw - the broker could not answer, and the start position the + * user asked for cannot be resolved, so the session must not be created. + * + * `question` and `topicFqn` are for the message the client is shown; they are the only way a user + * can tell "your topic is empty" from "the broker is unwell". + */ +def brokerAnswer[A](question: String, topicFqn: String)(lookup: => A): Option[A] = + Try(lookup) match + case Success(value) => Some(value) + case Failure(err) if isEmptyLogAnswer(err) => None + case Failure(err) => + throw StartFromUnresolvableException( + s"Could not resolve the requested start-from position: $question failed for $topicFqn. ${err.getMessage}", + err + ) diff --git a/server/src/main/scala/consumer/session_runner/topicPositions.scala b/server/src/main/scala/consumer/session_runner/topicPositions.scala new file mode 100644 index 000000000..92a3b1b62 --- /dev/null +++ b/server/src/main/scala/consumer/session_runner/topicPositions.scala @@ -0,0 +1,342 @@ +package consumer.session_runner + +import com.google.protobuf.ByteString +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.MessageId as PulsarMessageId +import org.apache.pulsar.client.impl.MessageIdImpl + +/** The per-topic debug view behind the session's "Topic Positions" tab: where each physical topic + * begins and ends, and how far the session has read through it. + * + * EVERYTHING HERE IS PURE. The three broker lookups a row needs (first entry, last entry, internal + * stats) sit behind plain arguments, so every arrangement worth reasoning about - an empty topic, a + * topic occupying one instant, a cursor whose ledger has aged out, a clock that ran backwards - is + * a table test rather than a fixture. The impure part is one adapter in ConsumerServiceImpl. + * + * WHY A HIGH-WATER MARK AND NOT "THE LAST MESSAGE ON SCREEN". The cursor is recorded where messages + * are ACKNOWLEDGED, so it counts what the session read, not what survived its filters. A filter + * that drops 99% of a topic would otherwise make both progress figures read almost zero while the + * session was in fact nearly finished. + */ + +/** How far this session has read into one physical topic. */ +final case class TopicCursor(messageId: PulsarMessageId, publishTime: Long) + +/** The oldest and newest log positions this session has processed from one physical topic. + * + * Both endpoints move only OUTWARD in Pulsar message-id order. That makes an older redelivery + * harmless and keeps the range stable even when producer publish clocks move backwards. When two + * observations have the same id (notably non-persistent topics, whose ids may all be `0:0`), the + * first observation is retained and the newest observation refreshes `last`. + */ +final case class TopicConsumedBounds(first: TopicCursor, last: TopicCursor) + +/** Widen consumed bounds with one newly processed message. */ +def widenConsumedBounds(existing: TopicConsumedBounds, incoming: TopicCursor): TopicConsumedBounds = + val firstComparison = incoming.messageId.compareTo(existing.first.messageId) + val lastComparison = incoming.messageId.compareTo(existing.last.messageId) + val first = + if firstComparison < 0 then incoming + else existing.first + val last = + if lastComparison > 0 || (lastComparison == 0 && incoming.publishTime != existing.last.publishTime) then incoming + else existing.last + // A redelivery inside the already-known range is the common no-op case. Reuse the immutable + // value instead of allocating another one on the hot listener path. + if (first eq existing.first) && (last eq existing.last) then existing + else TopicConsumedBounds(first, last) + +/** One ledger of a topic's managed ledger, reduced to what an ordinal needs. */ +final case class LedgerSpan(ledgerId: Long, entries: Long) + +/** An endpoint of a retained log - its first or last message. */ +final case class LogEndpoint(messageId: PulsarMessageId, publishTime: Long) + +/** Everything one row of the table is computed from. + * + * `first`/`last` are absent for an EMPTY topic, which is not a failure: `examineMessage` answers + * "latest" by throwing there and "earliest" with a 412, and both are classified as "the log is + * empty" rather than as broker trouble (see [[isEmptyLogAnswer]]). `unavailableReason` is for a + * topic that could not be asked at all - a non-persistent one, which Pulsar refuses to examine with + * a 405 - and is the difference between "nothing to report" and "nothing is known". + */ +final case class TopicPositionInputs( + topicFqn: String, + first: Option[LogEndpoint], + last: Option[LogEndpoint], + firstConsumed: Option[TopicCursor], + cursor: Option[TopicCursor], + ledgers: Vector[LedgerSpan], + currentLedgerEntries: Long, + retainedEntries: Long, + unavailableReason: Option[String] +) + +/** One assembled row. Every figure is optional because every one of them can be genuinely unknown, + * and a blank cell is honest where a zero would be a lie. + */ +final case class TopicPositionRow( + topicFqn: String, + first: Option[LogEndpoint], + last: Option[LogEndpoint], + firstConsumed: Option[TopicCursor], + cursor: Option[TopicCursor], + cursorTimeFraction: Option[Double], + cursorEntryFraction: Option[Double], + cursorEntryOrdinal: Option[Long], + retainedEntries: Option[Long], + unavailableReason: Option[String] +) + +/** The ledger list with Pulsar's open-ledger hole filled in. + * + * THE HOLE: `getInternalStats` reports the CURRENT (still open) ledger with `entries: 0` and + * `size: 0` - the real count lives in `currentLedgerEntries` alongside the list, not in the entry + * itself. Measured on a 6060-entry single-ledger topic: `ledgers: [{ledgerId: 7057, entries: 0}]` + * with `currentLedgerEntries: 6060`. Walking the list as reported therefore puts EVERY cursor in + * the open ledger at ordinal 1, i.e. "0% through", for the whole life of that ledger - which on a + * topic that has never rolled is the whole topic. + * + * The correction is positional, not value-based: the open ledger is the LAST one, so only the last + * element is patched, and only when it reports nothing. A closed ledger that genuinely holds zero + * entries in the middle of the list is left alone. + */ +def retainedLedgerSpans(ledgers: Vector[LedgerSpan], currentLedgerEntries: Long): Vector[LedgerSpan] = + if ledgers.isEmpty then ledgers + else + val last = ledgers.last + if last.entries > 0 || currentLedgerEntries <= 0 then ledgers + else ledgers.init :+ last.copy(entries = currentLedgerEntries) + +/** The cursor's 1-BASED ordinal among the entries the topic still retains. + * + * `None` when the cursor's ledger is not in the retained list at all, which is what a cursor that + * has aged out from under retention looks like: the session read entries that have since been + * trimmed. Reporting 0, or clamping it to the first retained entry, would both claim the session is + * at the beginning when it is in fact past the beginning - so it reports nothing. + * + * ENTRIES, NOT MESSAGES. A batched entry counts once here however many messages it carries; see + * [[cursorEntryFractionOf]] for why the distinction is named rather than hidden. + */ +def entryOrdinalOf(ledgers: Vector[LedgerSpan], cursorLedgerId: Long, cursorEntryId: Long): Option[Long] = + val index = ledgers.indexWhere(_.ledgerId == cursorLedgerId) + if index < 0 then None + else + val before = ledgers.take(index).map(_.entries).sum + // `cursorEntryId` is 0-based within its ledger, so +1 makes the whole thing 1-based: the + // very first retained entry is ordinal 1, matching how `examineMessage` addresses entries. + Some(before + cursorEntryId + 1) + +/** Where the cursor sits in the topic's TIME range, in [0.0, 1.0]. + * + * `None` unless there is a cursor AND both endpoints AND a range with an interior: + * + * - FIRST == LAST - everything the topic holds was published inside one millisecond - has no + * interior to place anything in, and the same rule ApproximatePublishTimePosition follows applies + * here: no position separates the messages, so no fraction describes one. Reporting 0.0 or 1.0 + * would both be inventions. + * - A RANGE REPORTED BACKWARDS (first > last) is not a range. Publish time is stamped by the + * PRODUCER, so a clock that stepped back can produce one; a negative denominator would hand + * back a nonsense fraction that looks like a real measurement. + * + * CLAMPED into [0.0, 1.0] because the three lookups are not atomic: the last entry is read before + * the cursor is, so a message published in between puts the cursor past the recorded end. That is a + * stale denominator, not a cursor that overran the topic, and 1.0 is the honest rendering of it. + */ +def cursorTimeFractionOf(first: Option[LogEndpoint], last: Option[LogEndpoint], cursor: Option[TopicCursor]): Option[Double] = + for + f <- first + l <- last + c <- cursor + if l.publishTime > f.publishTime + yield ((c.publishTime - f.publishTime).toDouble / (l.publishTime - f.publishTime).toDouble).max(0.0).min(1.0) + +/** Where the cursor sits among the topic's STORED ENTRIES, in [0.0, 1.0]. + * + * ENTRIES, NOT MESSAGES, and the name says so. Pulsar addresses stored data by entry, and a batched + * entry holds many messages, so this tracks a message-count percentage only as closely as batch + * sizes stayed uniform across the topic's life - exactly the approximation ApproximateEntryPosition + * documents for the same reason. Calling it "% of messages" would be a number the broker cannot + * actually produce without a message-ordinal index, which needs an operator opt-in that is off by + * default in every Pulsar version. + * + * `None` on a topic retaining nothing: there is no denominator, and 0/0 is not 0%. + */ +def cursorEntryFractionOf(cursorEntryOrdinal: Option[Long], retainedEntries: Long): Option[Double] = + if retainedEntries <= 0 then None + else cursorEntryOrdinal.map(ordinal => (ordinal.toDouble / retainedEntries.toDouble).max(0.0).min(1.0)) + +/** The ledger and entry of a message id, with any BATCH INDEX stripped. + * + * A batched message's id carries a third coordinate, and the ordinal walk is entry-addressed - two + * messages of one batch share an entry and must land on one ordinal, not two. + */ +def ledgerAndEntryOf(messageId: PulsarMessageId): Option[(Long, Long)] = entryIdOf(messageId) match + case id: MessageIdImpl => Some((id.getLedgerId, id.getEntryId)) + case _ => None + +/** Assemble one row from what the broker said and what the session has read. */ +def buildTopicPositionRow(inputs: TopicPositionInputs): TopicPositionRow = + val spans = retainedLedgerSpans(inputs.ledgers, inputs.currentLedgerEntries) + val ordinal = for + c <- inputs.cursor + (ledgerId, entryId) <- ledgerAndEntryOf(c.messageId) + o <- entryOrdinalOf(spans, ledgerId, entryId) + yield o + + TopicPositionRow( + topicFqn = inputs.topicFqn, + first = inputs.first, + last = inputs.last, + firstConsumed = inputs.firstConsumed, + cursor = inputs.cursor, + cursorTimeFraction = cursorTimeFractionOf(inputs.first, inputs.last, inputs.cursor), + cursorEntryFraction = cursorEntryFractionOf(ordinal, inputs.retainedEntries), + cursorEntryOrdinal = ordinal, + // A topic that could not be asked reports no denominator either - `retainedEntries` is 0 + // there only because nothing filled it in, and a "0 entries" cell would read as an empty + // topic rather than an unavailable one. + retainedEntries = Option.when(inputs.unavailableReason.isEmpty && inputs.retainedEntries >= 0)(inputs.retainedEntries), + unavailableReason = inputs.unavailableReason + ) + +/** Collapse the per-listener consumed ranges of one session into one range per physical topic. + * + * Overlapping enabled targets may each consume the same topic. The table has one row for that + * topic, so its range is the union: the earliest first and furthest last message-id endpoints. + */ +def mergeConsumedBounds(perListener: Iterable[Map[String, TopicConsumedBounds]]): Map[String, TopicConsumedBounds] = + perListener.flatten.foldLeft(Map.empty[String, TopicConsumedBounds]) { case (acc, (topicFqn, bounds)) => + val merged = acc.get(topicFqn) match + case None => bounds + case Some(existing) => + val firstComparison = bounds.first.messageId.compareTo(existing.first.messageId) + val lastComparison = bounds.last.messageId.compareTo(existing.last.messageId) + // Equal-id tie breakers make aggregation deterministic and useful for + // non-persistent topics, whose messages may all carry the same id. They never + // reorder distinct persistent log positions by an untrusted producer clock. + val first = + if firstComparison < 0 || (firstComparison == 0 && bounds.first.publishTime < existing.first.publishTime) + then bounds.first + else existing.first + val last = + if lastComparison > 0 || (lastComparison == 0 && bounds.last.publishTime > existing.last.publishTime) + then bounds.last + else existing.last + TopicConsumedBounds(first, last) + acc.updated(topicFqn, merged) + } + +/** Put one row on the wire. + * + * ABSENT STAYS ABSENT. Every optional here maps to an unset protobuf wrapper rather than to a zero, + * because the client renders a blank cell for "not known" and a real figure for "known to be zero", + * and the two are different answers: the first of N stored entries has position 1/N, while a topic + * whose session position has aged out is not known. + */ +def topicPositionToPb(row: TopicPositionRow): consumerPb.TopicPosition = + consumerPb.TopicPosition( + topicFqn = row.topicFqn, + firstMessageId = row.first.map(e => ByteString.copyFrom(e.messageId.toByteArray)), + firstPublishTime = row.first.map(_.publishTime), + lastMessageId = row.last.map(e => ByteString.copyFrom(e.messageId.toByteArray)), + lastPublishTime = row.last.map(_.publishTime), + cursorMessageId = row.cursor.map(c => ByteString.copyFrom(c.messageId.toByteArray)), + cursorPublishTime = row.cursor.map(_.publishTime), + cursorTimeFraction = row.cursorTimeFraction, + cursorEntryFraction = row.cursorEntryFraction, + retainedEntries = row.retainedEntries, + cursorEntryOrdinal = row.cursorEntryOrdinal, + unavailableReason = row.unavailableReason, + firstConsumedMessageId = row.firstConsumed.map(c => ByteString.copyFrom(c.messageId.toByteArray)), + firstConsumedPublishTime = row.firstConsumed.map(_.publishTime) + ) + +/** How many of one session's physical topics the Topic Positions poll may ask the broker about at + * once. Deliberately narrower than the start-from position lookups: this one is POLLED, so its + * cost recurs for as long as the tab is open, while a position resolution happens once per Play. + * Every task still runs on the server-wide budget shared with those lookups + * ([[approximatePositionLookupGlobalParallelism]]), so the two cannot multiply. */ +val topicPositionLookupParallelism: Int = 8 + +/** The wall-clock budget for one scan. Past it the poll answers with what it could not do rather + * than holding the broker and the client indefinitely; the tab simply asks again. */ +val topicPositionScanBudgetMs: Long = 20_000 + +/** How many Topic Positions scans may be in progress across the whole server. Each of these threads + * only WAITS - it dispatches the per-topic lookups onto the shared budget and collects them - so + * this bounds pile-up, not broker load; the coalescing below already limits a session to one scan + * at a time however fast its tab polls. */ +private val topicPositionScanConcurrency: Int = 8 + +private val topicPositionScanThreadNumber = java.util.concurrent.atomic.AtomicInteger(0) + +private lazy val topicPositionScanExecutor: java.util.concurrent.ExecutorService = + java.util.concurrent.Executors.newFixedThreadPool( + topicPositionScanConcurrency, + (runnable: Runnable) => + val thread = Thread(runnable, s"topic-positions-scan-${topicPositionScanThreadNumber.incrementAndGet()}") + thread.setDaemon(true) + thread + ) + +/** ONE Topic Positions scan per session at a time, however fast its tab polls. + * + * The tab polls once a second and a session may hold up to 2,000 physical topics, each costing two + * `examineMessage` calls plus `getInternalStats`. Run as the RPC used to run them - serially, on + * the gRPC thread, before returning an already-completed Future - a single browser could hold a + * service thread for a sweep of thousands of admin calls, and every tick started ANOTHER sweep on + * top of the one still running. So: the scan leaves the caller's thread, its per-topic lookups go + * through the server-wide bounded budget with a wall-clock deadline and cancellation, and a poll + * arriving while a scan is in flight JOINS it instead of starting a second one. Joining hands back + * an answer at most one scan old, which is exactly what a once-a-second poll of a slow sweep can + * honestly show. + * + * The scan-registry entry is removed BEFORE its promise completes, so the next poll after a scan + * finishes always starts a fresh one. + */ +final class TopicPositionScanner( + parallelism: Int = topicPositionLookupParallelism, + budgetMs: Long = topicPositionScanBudgetMs, + execute: Runnable => Unit = topicPositionScanExecutor.execute +): + private val inFlight = java.util.concurrent.ConcurrentHashMap[String, scala.concurrent.Future[Vector[TopicPositionRow]]]() + + /** Scans running right now - a test's window, and what a saturation metric would read. */ + private[consumer] def scansInFlight: Int = inFlight.size + + def scan( + sessionName: String, + topicFqns: Vector[String], + lookup: String => TopicPositionRow + ): scala.concurrent.Future[Vector[TopicPositionRow]] = + val topics = topicFqns.distinct.sorted + if topics.isEmpty then scala.concurrent.Future.successful(Vector.empty) + else + val promise = scala.concurrent.Promise[Vector[TopicPositionRow]]() + val joined = inFlight.putIfAbsent(sessionName, promise.future) + if joined != null then joined + else + val started = scala.util.Try(execute(() => + val outcome = scala.util.Try { + val rows = boundedParallelTopicLookup( + topics, + operation = s"the topic positions of consumer session $sessionName", + lookup = lookup, + budgetMs = budgetMs, + parallelism = parallelism, + remediation = "Narrow the session's topic selector, or close the Topic Positions tab while it reads." + ) + topics.flatMap(rows.get) + } + // Out of the registry FIRST: a caller that sees this promise complete and polls + // again must get a new scan, never a finished one it can never leave. + inFlight.remove(sessionName, promise.future) + promise.complete(outcome) + () + )) + started.failed.foreach { err => + inFlight.remove(sessionName, promise.future) + promise.failure(err) + } + promise.future diff --git a/server/src/main/scala/consumer/session_target/ConsumerSessionTarget.scala b/server/src/main/scala/consumer/session_target/ConsumerSessionTarget.scala index a2707f4ab..184932bf9 100644 --- a/server/src/main/scala/consumer/session_target/ConsumerSessionTarget.scala +++ b/server/src/main/scala/consumer/session_target/ConsumerSessionTarget.scala @@ -19,15 +19,18 @@ case class ConsumerSessionTarget( ) object ConsumerSessionTarget: + /** Each field under its own name, so the caller's `targets[n]` prefix composes into a full path + * - see [[_root_.consumer.atProtoField]]. */ def fromPb(v: pb.ConsumerSessionTarget): ConsumerSessionTarget = ConsumerSessionTarget( isEnabled = v.isEnabled, - consumptionMode = ConsumerSessionTargetConsumptionMode.fromPb(v.getConsumptionMode), - messageValueDeserializer = Deserializer.fromPb(v.getMessageValueDeserializer), - topicSelector = TopicSelector.fromPb(v.getTopicSelector), - messageFilterChain = MessageFilterChain.fromPb(v.getMessageFilterChain), - coloringRuleChain = ColoringRuleChain.fromPb(v.getColoringRuleChain), - valueProjectionList = ValueProjectionList.fromPb(v.getValueProjectionList) + consumptionMode = _root_.consumer.atProtoField("consumption_mode")(ConsumerSessionTargetConsumptionMode.fromPb(v.getConsumptionMode)), + messageValueDeserializer = + _root_.consumer.atProtoField("message_value_deserializer")(Deserializer.fromPb(v.getMessageValueDeserializer)), + topicSelector = _root_.consumer.atProtoField("topic_selector")(TopicSelector.fromPb(v.getTopicSelector)), + messageFilterChain = _root_.consumer.atProtoField("message_filter_chain")(MessageFilterChain.fromPb(v.getMessageFilterChain)), + coloringRuleChain = _root_.consumer.atProtoField("coloring_rule_chain")(ColoringRuleChain.fromPb(v.getColoringRuleChain)), + valueProjectionList = _root_.consumer.atProtoField("value_projection_list")(ValueProjectionList.fromPb(v.getValueProjectionList)) ) def toPb(v: ConsumerSessionTarget): pb.ConsumerSessionTarget = diff --git a/server/src/main/scala/consumer/session_target/topic_selector/MultiTopicSelector.scala b/server/src/main/scala/consumer/session_target/topic_selector/MultiTopicSelector.scala index 59471c532..a9308140e 100644 --- a/server/src/main/scala/consumer/session_target/topic_selector/MultiTopicSelector.scala +++ b/server/src/main/scala/consumer/session_target/topic_selector/MultiTopicSelector.scala @@ -14,9 +14,14 @@ case class MultiTopicSelector(topicFqns: Vector[String]): val partitions = getTopicPartitions(adminClient, topicFqn) partitions case TopicPartitioningType.NonPartitioned => Vector(topicFqn) - case Failure(_) => - println(s"Failed to get topic partitioning for topic $topicFqn") - Vector.empty + case Failure(err) => + // Swallowing this DROPPED the topic from the selection. The user named these + // FQNs explicitly, so silently consuming from a subset is wrong - and when + // every topic was unresolvable (unreachable broker, topic deleted between the + // picker and the session) the selector returned an empty vector, which the + // session runner accepted as a consumer-less session reported to the UI as OK. + // The sibling NamespacedRegexTopicSelector has always let these propagate. + throw new RuntimeException(s"Failed to resolve topic $topicFqn. ${err.getMessage}", err) }.distinct object MultiTopicSelector: diff --git a/server/src/main/scala/consumer/start_from/ApproximateEntryPosition.scala b/server/src/main/scala/consumer/start_from/ApproximateEntryPosition.scala new file mode 100644 index 000000000..3888a858b --- /dev/null +++ b/server/src/main/scala/consumer/start_from/ApproximateEntryPosition.scala @@ -0,0 +1,20 @@ +package consumer.start_from + +import com.tools.teal.pulsar.ui.api.v1.consumer as pb + +/** Start approximately `fraction` of the way through the retained ENTRIES of each physical topic. + * 0.0 is the earliest retained entry and 1.0 is past the latest. + * + * Pulsar can address an entry ordinal without scanning the log. An entry can contain a producer + * batch, so this is deliberately an entry position rather than a message percentile. The rounding, + * endpoint, empty-topic and retention-race behavior lives in + * `consumer.session_runner.resolveApproximateEntryPosition`. + */ +case class ApproximateEntryPosition(fraction: Double) + +object ApproximateEntryPosition: + def fromPb(v: pb.ApproximateEntryPosition): ApproximateEntryPosition = + ApproximateEntryPosition(fraction = v.fraction) + + def toPb(v: ApproximateEntryPosition): pb.ApproximateEntryPosition = + pb.ApproximateEntryPosition(fraction = v.fraction) diff --git a/server/src/main/scala/consumer/start_from/ApproximatePublishTimePosition.scala b/server/src/main/scala/consumer/start_from/ApproximatePublishTimePosition.scala new file mode 100644 index 000000000..cc3eae11f --- /dev/null +++ b/server/src/main/scala/consumer/start_from/ApproximatePublishTimePosition.scala @@ -0,0 +1,20 @@ +package consumer.start_from + +import com.tools.teal.pulsar.ui.api.v1.consumer as pb + +/** Interpolate `fraction` between a logical topic's observed first- and final-entry publish times. + * 0.0 is the earliest retained position; 1.0 seeks to the largest observed final-entry publish + * time. + * + * The range uses the selected partitions' boundary timestamps, and every partition seeks to the + * same timestamp. Interpolation, endpoint and empty-topic behavior lives in + * `consumer.session_runner.resolveApproximatePublishTimePosition`. + */ +case class ApproximatePublishTimePosition(fraction: Double) + +object ApproximatePublishTimePosition: + def fromPb(v: pb.ApproximatePublishTimePosition): ApproximatePublishTimePosition = + ApproximatePublishTimePosition(fraction = v.fraction) + + def toPb(v: ApproximatePublishTimePosition): pb.ApproximatePublishTimePosition = + pb.ApproximatePublishTimePosition(fraction = v.fraction) diff --git a/server/src/main/scala/consumer/start_from/ConsumerSessionStartFrom.scala b/server/src/main/scala/consumer/start_from/ConsumerSessionStartFrom.scala index f60608f91..402d81e6a 100644 --- a/server/src/main/scala/consumer/start_from/ConsumerSessionStartFrom.scala +++ b/server/src/main/scala/consumer/start_from/ConsumerSessionStartFrom.scala @@ -6,8 +6,14 @@ import com.google.protobuf.timestamp.Timestamp import java.time.Instant -type ConsumerSessionStartFrom = EarliestMessage | LatestMessage | NthMessageAfterEarliest | NthMessageBeforeLatest | MessageId | DateTime | RelativeDateTime +type ConsumerSessionStartFrom = EarliestMessage | LatestMessage | NthMessageAfterEarliest | NthMessageBeforeLatest | MessageId | DateTime | + RelativeDateTime | ApproximateEntryPosition | ApproximatePublishTimePosition +/** A UNION type, so these matches are NOT checked for exhaustiveness: a mode missing from either + * direction compiles fine and throws at runtime, surfacing as a generic failure on saving or + * loading a session. `startFromConversionsTest` sweeps every mode through both directions for that + * reason - it is the only thing standing in for the missing compiler check. + */ object ConsumerSessionStartFrom: def fromPb(startFrom: pb.ConsumerSessionStartFrom): ConsumerSessionStartFrom = startFrom.startFrom match @@ -18,6 +24,8 @@ object ConsumerSessionStartFrom: case pb.ConsumerSessionStartFrom.StartFrom.StartFromMessageId(v) => MessageId.fromPb(v) case pb.ConsumerSessionStartFrom.StartFrom.StartFromDateTime(v) => DateTime.fromPb(v) case pb.ConsumerSessionStartFrom.StartFrom.StartFromRelativeDateTime(v) => RelativeDateTime.fromPb(v) + case pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition(v) => ApproximateEntryPosition.fromPb(v) + case pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximatePublishTimePosition(v) => ApproximatePublishTimePosition.fromPb(v) case _ => throw IllegalArgumentException("Unknown ConsumerSessionStartFrom type.") def toPb(startFrom: ConsumerSessionStartFrom): pb.ConsumerSessionStartFrom = @@ -26,10 +34,31 @@ object ConsumerSessionStartFrom: pb.ConsumerSessionStartFrom(startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromEarliestMessage(EarliestMessage.toPb(v))) case v: LatestMessage => pb.ConsumerSessionStartFrom(startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromLatestMessage(LatestMessage.toPb(v))) + // Both Nth modes were readable but not writable: saving a session that used one threw + // "Unknown ConsumerSessionStartFrom type" from here. + case v: NthMessageAfterEarliest => + pb.ConsumerSessionStartFrom(startFrom = + pb.ConsumerSessionStartFrom.StartFrom.StartFromNthMessageAfterEarliest(NthMessageAfterEarliest.toPb(v)) + ) + case v: NthMessageBeforeLatest => + pb.ConsumerSessionStartFrom(startFrom = + pb.ConsumerSessionStartFrom.StartFrom.StartFromNthMessageBeforeLatest(NthMessageBeforeLatest.toPb(v)) + ) case v: MessageId => pb.ConsumerSessionStartFrom(startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromMessageId(MessageId.toPb(v))) case v: DateTime => pb.ConsumerSessionStartFrom(startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromDateTime(DateTime.toPb(v))) case v: RelativeDateTime => pb.ConsumerSessionStartFrom(startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromRelativeDateTime(RelativeDateTime.toPb(v))) + // The two approximate modes carry the SAME payload - one double - so a branch that + // reached for the other one's oneof case would still round-trip a fraction and look + // right; the only symptom would be a session positioned by the wrong rule. + case v: ApproximateEntryPosition => + pb.ConsumerSessionStartFrom(startFrom = + pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition(ApproximateEntryPosition.toPb(v)) + ) + case v: ApproximatePublishTimePosition => + pb.ConsumerSessionStartFrom(startFrom = + pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximatePublishTimePosition(ApproximatePublishTimePosition.toPb(v)) + ) case _ => throw IllegalArgumentException("Unknown ConsumerSessionStartFrom type.") diff --git a/server/src/main/scala/library/Library.scala b/server/src/main/scala/library/Library.scala index cc3881e41..8d2191ccc 100644 --- a/server/src/main/scala/library/Library.scala +++ b/server/src/main/scala/library/Library.scala @@ -3,6 +3,7 @@ package library import scalapb.json4s.JsonFormat import com.typesafe.scalalogging.Logger import com.tools.teal.pulsar.ui.library.v1.library as pb +import scala.concurrent.blocking import scala.util.{Failure, Success, Try} type FileName = String @@ -31,20 +32,42 @@ object Library: // (e.g. `../../…`) can never escape the library directory. Real ids are UUIDs. private val SafeItemId = "^[A-Za-z0-9_-]{1,200}$".r + private def isSafeItemId(itemId: LibraryItemId): Boolean = + SafeItemId.findFirstIn(itemId).isDefined + + // The single file name an item may live under. Write, delete and scan must all agree on it. + private def fileNameOf(itemId: LibraryItemId): FileName = s"$itemId.binpb" + class Library: private var rootDir = "./data" - private var db = LibraryDb(itemsById = Map.empty) + @volatile private var db = LibraryDb(itemsById = Map.empty) private val logger: Logger = Logger(getClass.getName) + // A Library instance is shared by every gRPC call (LibraryServiceImpl holds exactly one), so + // saveLibraryItem/deleteLibraryItem run concurrently against it. Each mutation is + // "touch the file, rescan the dir, publish the snapshot" - three steps that MUST NOT interleave: + // - two writers could interleave so that an OLDER scan publishes last, dropping a + // just-written item from the snapshot even though its file is on disk; + // - a delete racing a scan could remove a file between os.list and os.read.bytes, blowing up + // the unrelated writer with NoSuchFileException; + // - deleteItem's exists-then-remove could let two concurrent deletes both report success. + // One lock over the whole sequence makes each mutation atomic with respect to the others. + // + // Each mutation is additionally wrapped in scala.concurrent.blocking: LibraryServiceImpl is + // bound on ExecutionContext.global (see GrpcServer), a bounded ForkJoinPool that only spawns + // compensation threads for blocking it is TOLD about - unmarked, the O(N) directory rescan + // below, serialized under this lock, would starve unrelated compute tasks pool-wide. + private val mutationLock = new Object + def size: Int = db.itemsById.size private def requireSafeItemId(itemId: LibraryItemId): Unit = - if Library.SafeItemId.findFirstIn(itemId).isEmpty then + if !Library.isSafeItemId(itemId) then throw new IllegalArgumentException( s"Invalid library item id - only alphanumerics, '_' and '-' are allowed." ) - def writeItem(item: LibraryItem): Unit = + def writeItem(item: LibraryItem): Unit = blocking(mutationLock.synchronized { val itemId = item.spec.metadata.id requireSafeItemId(itemId) @@ -55,7 +78,7 @@ class Library: s"Library item $itemId must be available in at least one context; an item without contexts would be unreachable." ) - val fileName = s"$itemId.binpb" + val fileName = Library.fileNameOf(itemId) val filePath = os.Path(fileName, os.Path(rootDir, os.pwd)) val itemAsBinary = LibraryItem.toPb(item).toByteArray @@ -65,15 +88,27 @@ class Library: ) refreshDb() + }) - def deleteItem(itemId: LibraryItemId): Unit = + def deleteItem(itemId: LibraryItemId): Unit = blocking(mutationLock.synchronized { requireSafeItemId(itemId) - val fileName = s"$itemId.binpb" + val fileName = Library.fileNameOf(itemId) val filePath = os.Path(fileName, os.Path(rootDir, os.pwd)) + // os.remove delegates to Files.deleteIfExists, which returns false rather than throwing - + // so deleting an id that was never there reported OK, indistinguishable from a real delete. + // Check the FILE, not the cached db: the db is a snapshot and a stale entry would otherwise + // decide the outcome. NoSuchElementException stays out of the IllegalArgumentException + // (INVALID_ARGUMENT) channel used for malformed ids; the service maps it to NOT_FOUND. + // The exists check and the remove are only meaningful together, hence the surrounding lock - + // unserialized, two concurrent deletes both saw the file and both reported success. + if !os.exists(filePath) then + throw new NoSuchElementException(s"No library item with id: $itemId") + os.remove(filePath) refreshDb() + }) def getItemById(itemId: LibraryItemId): Option[LibraryItem] = db.itemsById.get(itemId) @@ -108,17 +143,53 @@ class Library: .map { path => val fileName = path.last val fileContent = os.read.bytes(path) - val scanResultEntryA = Try(LibraryItem.fromPb(pb.LibraryItem.parseFrom(fileContent))) + val scanResultEntryA = Try { + val itemPb = pb.LibraryItem.parseFrom(fileContent) + // A stored item carrying the unimplemented AllNamespaceMatcher.namespace_regex + // must keep LOADING. The conversion below REFUSES a set regex - right for + // request data, where accepting it would silently widen the caller's scope - + // but on this path the refusal made the whole item vanish from the library + // with only a WARN and no way to repair it through the product. Strip the + // FIELD (it was never applied by any version that wrote it), keep the ITEM, + // and name what was ignored. + val lenientPb = itemPb.copy(metadata = itemPb.metadata.map(md => + md.copy(availableForContexts = md.availableForContexts.map(matcher => + resourceMatcherWithoutNamespaceRegex( + matcher, + pattern => + logger.warn( + s"Library file $fileName: ignoring unimplemented AllNamespaceMatcher.namespace_regex '$pattern'; the matcher covers every namespace of the matching tenant." + ) + ) + )) + )) + LibraryItem.fromPb(lenientPb) + } val scanResultEntry = scanResultEntryA.toEither - val libraryItemIdFromFileName = fileName.split('.').head scanResultEntry match case Left(err) => logger.warn(s"Failed to parse library item from file $fileName: $err") fileName -> scanResultEntry case Right(item) => val itemId = item.spec.metadata.id - if itemId != libraryItemIdFromFileName then + // Whatever the scan surfaces must be ADDRESSABLE: writeItem and deleteItem + // both derive `$itemId.binpb` from the id and reject ids outside the safe + // charset, so the scan has to hold itself to the same two rules. Deriving + // the id with `fileName.split('.').head` instead accepted + // `id.extra.binpb` as item `id` - listed and gettable, but deleting it hit + // NOT_FOUND and saving it created a SECOND file; and skipping + // requireSafeItemId surfaced ids like `bad+id` that every write and delete + // then rejected with INVALID_ARGUMENT. + if !Library.isSafeItemId(itemId) then + logger.warn(s"Skipping library file $fileName: item id $itemId is not a valid item id") + fileName -> Left( + new Exception( + s"Library item id $itemId in file $fileName is not a valid item id" + ) + ) + else if fileName != Library.fileNameOf(itemId) then + logger.warn(s"Skipping library file $fileName: item id $itemId does not match the file name") fileName -> Left( new Exception( s"File name $fileName does not match library item id $itemId" @@ -128,7 +199,14 @@ class Library: } .toMap - private def refreshDb(): Unit = + // Every mutation rebuilds the snapshot by rescanning the WHOLE directory rather than patching + // the one key it touched. Deliberate: the scan is the single reconciliation point (it also + // picks up files an operator adds or removes out of band), and an incremental update would + // have to re-prove every scan invariant on its own (safe-id filter, name/id agreement, + // parse-failure skips) - drift between the two paths would surface as phantom or missing + // items. N is the library size (hundreds of small files), mutations are user-click rare, and + // the I/O is marked `blocking`, so the O(N) rescan is cheap for what it buys. + private def refreshDb(): Unit = blocking(mutationLock.synchronized { val scanResult = scan() val itemsById = scanResult.collect { case (_, Right(item)) => val itemId = item.spec.metadata.id @@ -137,3 +215,4 @@ class Library: logger.info(s"Library refreshed. Found ${itemsById.size} items in library") db = LibraryDb(itemsById = itemsById) + }) diff --git a/server/src/main/scala/library/LibraryServiceImpl.scala b/server/src/main/scala/library/LibraryServiceImpl.scala index 36ae23d6c..236e83d3b 100644 --- a/server/src/main/scala/library/LibraryServiceImpl.scala +++ b/server/src/main/scala/library/LibraryServiceImpl.scala @@ -35,9 +35,13 @@ val config = Await.result(readConfigAsync, Duration(10, SECONDS)) // DEKAF_DATA_DIR), so per-connection isolation lives at the deployment layer, not in here. val libraryRoot = s"${config.dataDir.get}/library" -class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: +/** The store is a constructor parameter defaulting to the process-wide library directory - same + * reason as `pulsarAuthToCookie`/`PulsarAuthRoutes.routesWith`: `libraryRoot` is derived from a + * config val loaded once per JVM, so nothing could otherwise observe what these RPCs answer without + * writing into the running instance's own data directory. The production call site in `GrpcServer` + * is unchanged. */ +class LibraryServiceImpl(val library: Library = Library.createAndRefreshDb(libraryRoot)) extends pb.LibraryServiceGrpc.LibraryService: val logger: Logger = Logger(getClass.getName) - val library: Library = Library.createAndRefreshDb(libraryRoot) override def saveLibraryItem(request: SaveLibraryItemRequest): Future[SaveLibraryItemResponse] = logger.debug(s"Updating library item: ${request.item}") @@ -48,16 +52,16 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: val libraryItem = LibraryItem.fromPb(request.item.get) library.writeItem(libraryItem) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.SaveLibraryItemResponse(status = Some(status))) } catch { case err: IllegalArgumentException => logger.warn(s"Rejected library item save: ${err.getMessage}") - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = s"Unable to save library item. ${err.getMessage}") + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = s"Unable to save library item. ${err.getMessage}") Future.successful(pb.SaveLibraryItemResponse(status = Some(status))) case err: Exception => logger.warn(s"Failed to save library item: ${err.getMessage}") - val status: Status = Status(code = Code.INTERNAL.index, message = s"Unable to save library item. ${err.getMessage}}") + val status: Status = Status(code = Code.INTERNAL.value, message = s"Unable to save library item. ${err.getMessage}") Future.successful(pb.SaveLibraryItemResponse(status = Some(status))) } @@ -69,16 +73,22 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: library.deleteItem(request.id) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.DeleteLibraryItemResponse(status = Some(status))) } catch { case e: IllegalArgumentException => logger.warn(s"Rejected library item delete: ${e.getMessage}") - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = s"Unable to delete library item. ${e.getMessage}") + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = s"Unable to delete library item. ${e.getMessage}") + Future.successful(pb.DeleteLibraryItemResponse(status = Some(status))) + // Deleting something that isn't there is a client-visible NOT_FOUND, not a 500 - it + // reaches here whenever the id has no file (previously this reported OK silently). + case e: NoSuchElementException => + logger.warn(s"Library item to delete not found: ${e.getMessage}") + val status: Status = Status(code = Code.NOT_FOUND.value, message = s"Unable to delete library item. ${e.getMessage}") Future.successful(pb.DeleteLibraryItemResponse(status = Some(status))) case e: Exception => logger.warn(s"Failed to delete library item: ${e.getMessage}") - val status: Status = Status(code = Code.INTERNAL.index, message = "Unable to delete library item") + val status: Status = Status(code = Code.INTERNAL.value, message = "Unable to delete library item") Future.successful(pb.DeleteLibraryItemResponse(status = Some(status))) } @@ -91,17 +101,17 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: val libraryItem = library.getItemById(request.id) if libraryItem.isEmpty then - val status: Status = Status(code = Code.NOT_FOUND.index) + val status: Status = Status(code = Code.NOT_FOUND.value) return Future.successful(pb.GetLibraryItemResponse(status = Some(status))) val libraryItemPb = LibraryItem.toPb(libraryItem.get) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetLibraryItemResponse(status = Some(status), item = Some(libraryItemPb))) } catch { case e: Exception => logger.warn(s"Failed to get library item: ${e.getMessage}") - val status: Status = Status(code = Code.INTERNAL.index, message = s"Unable to get library item. ${e.getMessage}") + val status: Status = Status(code = Code.INTERNAL.value, message = s"Unable to get library item. ${e.getMessage}") Future.successful(pb.GetLibraryItemResponse(status = Some(status))) } @@ -117,16 +127,16 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: val libraryItemsPb = libraryItems.map(LibraryItem.toPb) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ListLibraryItemsResponse(status = Some(status), items = libraryItemsPb)) } catch { case e: IllegalArgumentException => logger.warn(s"Rejected library items list: ${e.getMessage}") - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = s"Unable to list library items. ${e.getMessage}") + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = s"Unable to list library items. ${e.getMessage}") Future.successful(pb.ListLibraryItemsResponse(status = Some(status))) case e: Exception => logger.warn(s"Failed to list library items: ${e.getMessage}") - val status: Status = Status(code = Code.INTERNAL.index, message = s"Unable to list library items. ${e.getMessage}") + val status: Status = Status(code = Code.INTERNAL.value, message = s"Unable to list library items. ${e.getMessage}") Future.successful(pb.ListLibraryItemsResponse(status = Some(status))) } @@ -150,7 +160,7 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: ) ).toVector - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetLibraryItemsCountResponse( status = Some(status), itemCountPerType = itemCountPerType @@ -158,10 +168,10 @@ class LibraryServiceImpl extends pb.LibraryServiceGrpc.LibraryService: } catch { case e: IllegalArgumentException => logger.warn(s"Rejected library items count: ${e.getMessage}") - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = s"Unable to get library items count. ${e.getMessage}") + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = s"Unable to get library items count. ${e.getMessage}") Future.successful(pb.GetLibraryItemsCountResponse(status = Some(status))) case e: Exception => logger.warn(s"Failed to get library items count: ${e.getMessage}") - val status: Status = Status(code = Code.INTERNAL.index, message = s"Unable to get library items count. ${e.getMessage}") + val status: Status = Status(code = Code.INTERNAL.value, message = s"Unable to get library items count. ${e.getMessage}") Future.successful(pb.GetLibraryItemsCountResponse(status = Some(status))) } diff --git a/server/src/main/scala/library/managed_items/ManagedConsumerSessionConfig.scala b/server/src/main/scala/library/managed_items/ManagedConsumerSessionConfig.scala index ef4f3a804..ca15394d8 100644 --- a/server/src/main/scala/library/managed_items/ManagedConsumerSessionConfig.scala +++ b/server/src/main/scala/library/managed_items/ManagedConsumerSessionConfig.scala @@ -1,6 +1,7 @@ package library.managed_items import com.tools.teal.pulsar.ui.library.v1.managed_items as pb +import com.tools.teal.pulsar.ui.api.v1.consumer as apiPb import library.{ManagedItemMetadata, ManagedItemReference, ManagedItemTrait} case class ManagedConsumerSessionConfigSpec( @@ -10,7 +11,13 @@ case class ManagedConsumerSessionConfigSpec( pauseTriggerChain: ManagedConsumerSessionPauseTriggerChainValOrRef, coloringRuleChain: ManagedColoringRuleChainValOrRef, valueProjectionList: ManagedValueProjectionListValOrRef, - numDisplayItems: Option[Long] + numDisplayItems: Option[Long], + // Missing and UNSPECIFIED both use GUARANTEED, the product default - owner decision + // (2026-08-11, direct instruction), the third move of this default (the plan file's decision + // log is the record). Pre-existing saved items carry no field at all and run the default; + // explicit Best effort and Fastest are choices and are preserved. + messageDeliveryOrder: Option[apiPb.MessageDeliveryOrder] = Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED), + deliveryOrderKey: Option[apiPb.DeliveryOrderKey] = None ) object ManagedConsumerSessionConfigSpec: @@ -22,7 +29,9 @@ object ManagedConsumerSessionConfigSpec: pauseTriggerChain = ManagedConsumerSessionPauseTriggerChainValOrRef.fromPb(v.getPauseTriggerChain), coloringRuleChain = ManagedColoringRuleChainValOrRef.fromPb(v.getColoringRuleChain), valueProjectionList = ManagedValueProjectionListValOrRef.fromPb(v.getValueProjectionList), - numDisplayItems = v.numDisplayItems + numDisplayItems = v.numDisplayItems, + messageDeliveryOrder = Some(normalizeMessageDeliveryOrder(v.messageDeliveryOrder)), + deliveryOrderKey = Option.when(v.deliveryOrderKey != apiPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_UNSPECIFIED)(v.deliveryOrderKey) ) def toPb(v: ManagedConsumerSessionConfigSpec): pb.ManagedConsumerSessionConfigSpec = @@ -33,9 +42,22 @@ object ManagedConsumerSessionConfigSpec: pauseTriggerChain = Some(ManagedConsumerSessionPauseTriggerChainValOrRef.toPb(v.pauseTriggerChain)), coloringRuleChain = Some(ManagedColoringRuleChainValOrRef.toPb(v.coloringRuleChain)), valueProjectionList = Some(ManagedValueProjectionListValOrRef.toPb(v.valueProjectionList)), - numDisplayItems = v.numDisplayItems + numDisplayItems = v.numDisplayItems, + messageDeliveryOrder = v.messageDeliveryOrder + .map(normalizeMessageDeliveryOrder) + .getOrElse(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED), + deliveryOrderKey = v.deliveryOrderKey.getOrElse(apiPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_UNSPECIFIED) ) + private def normalizeMessageDeliveryOrder(v: apiPb.MessageDeliveryOrder): apiPb.MessageDeliveryOrder = v match + case apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED => v + case apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME => v + case apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED => v + case apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED => + apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + case _: apiPb.MessageDeliveryOrder.Unrecognized => + apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + case class ManagedConsumerSessionConfig( metadata: ManagedItemMetadata, spec: ManagedConsumerSessionConfigSpec diff --git a/server/src/main/scala/library/managed_items/ManagedConsumerSessionStartFrom.scala b/server/src/main/scala/library/managed_items/ManagedConsumerSessionStartFrom.scala index 80a6ecd6b..c30ebfd71 100644 --- a/server/src/main/scala/library/managed_items/ManagedConsumerSessionStartFrom.scala +++ b/server/src/main/scala/library/managed_items/ManagedConsumerSessionStartFrom.scala @@ -3,11 +3,18 @@ package library.managed_items import com.tools.teal.pulsar.ui.library.v1.managed_items as pb import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb import library.{ManagedItemMetadata, ManagedItemReference, ManagedItemTrait} -import _root_.consumer.start_from.{EarliestMessage, LatestMessage, NthMessageAfterEarliest, NthMessageBeforeLatest} +import _root_.consumer.start_from.{ + ApproximateEntryPosition, + ApproximatePublishTimePosition, + EarliestMessage, + LatestMessage, + NthMessageAfterEarliest, + NthMessageBeforeLatest +} case class ManagedConsumerSessionStartFromSpec( - startFrom: EarliestMessage | LatestMessage | ManagedConsumerSessionStartFromValOrRef | ManagedMessageIdValOrRef | ManagedDateTimeValOrRef | - ManagedRelativeDateTimeValOrRef | NthMessageBeforeLatest | NthMessageAfterEarliest + startFrom: EarliestMessage | LatestMessage | ManagedMessageIdValOrRef | ManagedDateTimeValOrRef | + ManagedRelativeDateTimeValOrRef | NthMessageBeforeLatest | NthMessageAfterEarliest | ApproximateEntryPosition | ApproximatePublishTimePosition ) object ManagedConsumerSessionStartFromSpec: @@ -26,6 +33,10 @@ object ManagedConsumerSessionStartFromSpec: ManagedConsumerSessionStartFromSpec(startFrom = NthMessageAfterEarliest.fromPb(sf.value)) case sf: pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromNthMessageBeforeLatest => ManagedConsumerSessionStartFromSpec(startFrom = NthMessageBeforeLatest.fromPb(sf.value)) + case sf: pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromApproximateEntryPosition => + ManagedConsumerSessionStartFromSpec(startFrom = ApproximateEntryPosition.fromPb(sf.value)) + case sf: pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromApproximatePublishTimePosition => + ManagedConsumerSessionStartFromSpec(startFrom = ApproximatePublishTimePosition.fromPb(sf.value)) case _ => throw new IllegalArgumentException("Invalid ManagedConsumerSessionStartFromSpec type") @@ -59,6 +70,14 @@ object ManagedConsumerSessionStartFromSpec: pb.ManagedConsumerSessionStartFromSpec( startFrom = pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromNthMessageBeforeLatest(NthMessageBeforeLatest.toPb(v)) ) + case v: ApproximateEntryPosition => + pb.ManagedConsumerSessionStartFromSpec( + startFrom = pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromApproximateEntryPosition(ApproximateEntryPosition.toPb(v)) + ) + case v: ApproximatePublishTimePosition => + pb.ManagedConsumerSessionStartFromSpec( + startFrom = pb.ManagedConsumerSessionStartFromSpec.StartFrom.StartFromApproximatePublishTimePosition(ApproximatePublishTimePosition.toPb(v)) + ) case _ => throw new IllegalArgumentException("Invalid ManagedConsumerSessionStartFromSpec type") diff --git a/server/src/main/scala/library/managed_items/ManagedRelativeDateTime.scala b/server/src/main/scala/library/managed_items/ManagedRelativeDateTime.scala index f0d92a669..cc33ca3dc 100644 --- a/server/src/main/scala/library/managed_items/ManagedRelativeDateTime.scala +++ b/server/src/main/scala/library/managed_items/ManagedRelativeDateTime.scala @@ -12,6 +12,17 @@ case class ManagedRelativeDateTimeSpec( object ManagedRelativeDateTimeSpec: def fromPb(v: pb.ManagedRelativeDateTimeSpec): ManagedRelativeDateTimeSpec = + // Trust boundary: managed_items.proto stores `value` as int64, but a relative date-time can + // only ever be resolved into the api form (consumer.proto RelativeDateTime.value is int32). + // A library file written by a non-Dekaf client could persist a value that is negative or + // outside int32 range; accepting it would let the later narrowing to int32 silently truncate. + // Reject here (IllegalArgumentException -> INVALID_ARGUMENT on save, skipped file on scan), + // matching the other malformed-input guards, rather than storing a value that can never + // become a valid session request. + if v.value < 0 || v.value > Int.MaxValue then + throw new IllegalArgumentException( + s"Managed relative date-time value ${v.value} is out of range; it must be a non-negative int32 (0..${Int.MaxValue})." + ) ManagedRelativeDateTimeSpec( value = v.value, unit = DateTimeUnit.fromPb(v.unit), diff --git a/server/src/main/scala/library/resourceMatchersConversions.scala b/server/src/main/scala/library/resourceMatchersConversions.scala index 794746adf..3d2aeb83e 100644 --- a/server/src/main/scala/library/resourceMatchersConversions.scala +++ b/server/src/main/scala/library/resourceMatchersConversions.scala @@ -33,14 +33,45 @@ def tenantMatcherToPb(v: TenantMatcher): pb.TenantMatcher = case v: AllTenantMatcher => pb.TenantMatcher(matcher = pb.TenantMatcher.Matcher.All(allTenantMatcherToPb(v))) +/** Nested matcher messages are proto3 optional, so an older/partial client can leave them unset. + * `.get` turned that into NoSuchElementException, which LibraryServiceImpl maps to INTERNAL (a + * 500 for a client-side mistake); IllegalArgumentException is the INVALID_ARGUMENT channel used + * by the oneof guards in this same file. */ +private def required[A](field: Option[A], name: String): A = + field.getOrElse(throw new IllegalArgumentException(s"Missing required field: $name")) + def exactNamespaceMatcherFromPb(v: pb.ExactNamespaceMatcher): ExactNamespaceMatcher = - ExactNamespaceMatcher(tenant = tenantMatcherFromPb(v.tenant.get), namespace = v.namespace) + ExactNamespaceMatcher(tenant = tenantMatcherFromPb(required(v.tenant, "ExactNamespaceMatcher.tenant")), namespace = v.namespace) def exactNamespaceMatcherToPb(v: ExactNamespaceMatcher): pb.ExactNamespaceMatcher = pb.ExactNamespaceMatcher(tenant = Some(tenantMatcherToPb(v.tenant)), namespace = v.namespace) +/** `AllNamespaceMatcher.namespace_regex` (proto field 2) is NOT implemented, and the model has no + * field for it - "all namespaces of a matching tenant" is the whole meaning of this matcher. + * + * It stays unimplemented deliberately rather than by oversight: `test` compares one MATCHER against + * another, not a matcher against a concrete namespace, so an AllNamespaceMatcher tested against + * another AllNamespaceMatcher would have to decide whether one regex subsumes another - undecidable + * in general. There is no honest semantics to give it for that case. + * + * So a set regex is REFUSED. Accepting it and dropping it returned a matcher covering every + * namespace of the tenant to a caller that asked for a subset - success plus a silently WIDER + * access scope. A server-side warning does not reach that caller; only an error does. + * IllegalArgumentException is the INVALID_ARGUMENT channel used by the other malformed-input guards + * in this file. An UNSET regex (the proto default) still converts, so every stored item and every + * request the UI makes are untouched - nothing writes the field. + * + * The refusal is for REQUEST data only. For items already ON DISK it made the whole item vanish + * from the library, so the scan strips a set regex before converting - see + * `resourceMatcherWithoutNamespaceRegex` below. + */ def allNamespaceMatcherFromPb(v: pb.AllNamespaceMatcher): AllNamespaceMatcher = - AllNamespaceMatcher(tenant = tenantMatcherFromPb(v.tenant.get)) + if v.namespaceRegex.nonEmpty then + throw new IllegalArgumentException( + s"AllNamespaceMatcher.namespace_regex is not implemented (got '${v.namespaceRegex}'). " + + "This matcher covers EVERY namespace of the matching tenant; use ExactNamespaceMatcher to narrow the scope." + ) + AllNamespaceMatcher(tenant = tenantMatcherFromPb(required(v.tenant, "AllNamespaceMatcher.tenant"))) def allNamespaceMatcherToPb(v: AllNamespaceMatcher): pb.AllNamespaceMatcher = pb.AllNamespaceMatcher(tenant = Some(tenantMatcherToPb(v.tenant))) @@ -62,7 +93,7 @@ def namespaceMatcherToPb(v: NamespaceMatcher): pb.NamespaceMatcher = def exactTopicMatcherFromPb(v: pb.ExactTopicMatcher): ExactTopicMatcher = ExactTopicMatcher( - namespace = namespaceMatcherFromPb(v.namespace.get), + namespace = namespaceMatcherFromPb(required(v.namespace, "ExactTopicMatcher.namespace")), topic = v.topic ) @@ -74,7 +105,7 @@ def exactTopicMatcherToPb(v: ExactTopicMatcher): pb.ExactTopicMatcher = def allTopicMatcherFromPb(v: pb.AllTopicMatcher): AllTopicMatcher = AllTopicMatcher( - namespace = namespaceMatcherFromPb(v.namespace.get) + namespace = namespaceMatcherFromPb(required(v.namespace, "AllTopicMatcher.namespace")) ) def allTopicMatcherToPb(v: AllTopicMatcher): pb.AllTopicMatcher = @@ -111,3 +142,35 @@ def resourceMatcherToPb(v: ResourceMatcher): pb.ResourceMatcher = pb.ResourceMatcher(matcher = pb.ResourceMatcher.Matcher.Namespace(namespaceMatcherToPb(v))) case v: TopicMatcher => pb.ResourceMatcher(matcher = pb.ResourceMatcher.Matcher.Topic(topicMatcherToPb(v))) + +/** READ-path counterpart to the refusal in `allNamespaceMatcherFromPb`: return `v` with any SET + * `AllNamespaceMatcher.namespace_regex` cleared, reporting each dropped pattern via `onDrop`. + * + * A REQUEST carrying the unimplemented field must be refused - accepting it would silently widen + * the caller's scope. A STORED item is different: refusing it on the scan path made the whole + * item fail to parse and vanish from the library, with only a server-side WARN and no way to + * repair it through the product. Dropping the FIELD (never applied by any version that wrote it) + * keeps the item loadable and addressable, and `onDrop` lets the caller name the item and the + * ignored pattern. The field nests in two places: directly under a namespace matcher, and under + * the namespace of either topic matcher variant. + */ +def resourceMatcherWithoutNamespaceRegex(v: pb.ResourceMatcher, onDrop: String => Unit): pb.ResourceMatcher = + def cleanNamespace(ns: pb.NamespaceMatcher): pb.NamespaceMatcher = + ns.matcher match + case pb.NamespaceMatcher.Matcher.All(all) if all.namespaceRegex.nonEmpty => + onDrop(all.namespaceRegex) + ns.copy(matcher = pb.NamespaceMatcher.Matcher.All(all.copy(namespaceRegex = ""))) + case _ => ns + + v.matcher match + case pb.ResourceMatcher.Matcher.Namespace(ns) => + v.copy(matcher = pb.ResourceMatcher.Matcher.Namespace(cleanNamespace(ns))) + case pb.ResourceMatcher.Matcher.Topic(topic) => + val cleaned = topic.matcher match + case pb.TopicMatcher.Matcher.Exact(exact) => + topic.copy(matcher = pb.TopicMatcher.Matcher.Exact(exact.copy(namespace = exact.namespace.map(cleanNamespace)))) + case pb.TopicMatcher.Matcher.All(all) => + topic.copy(matcher = pb.TopicMatcher.Matcher.All(all.copy(namespace = all.namespace.map(cleanNamespace)))) + case _ => topic + v.copy(matcher = pb.ResourceMatcher.Matcher.Topic(cleaned)) + case _ => v diff --git a/server/src/main/scala/metrics/MetricsServiceImpl.scala b/server/src/main/scala/metrics/MetricsServiceImpl.scala index 55382eced..11acad97b 100644 --- a/server/src/main/scala/metrics/MetricsServiceImpl.scala +++ b/server/src/main/scala/metrics/MetricsServiceImpl.scala @@ -41,14 +41,14 @@ class MetricsServiceImpl extends MetricsServiceGrpc.MetricsService: (namespace, getOptionalNamespaceMetricsPb(metricsEntries, namespace)) ).toMap - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetNamespacesMetricsResponse( status = Some(status), namespacesMetrics = optionalNamespacesMetrics )) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetNamespacesMetricsResponse(status = Some(status))) } @@ -58,14 +58,14 @@ class MetricsServiceImpl extends MetricsServiceGrpc.MetricsService: (namespace, getOptionalNamespacePersistentMetricsPb(metricsEntries, namespace)) ).toMap - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(GetNamespacesPersistentMetricsResponse( status = Some(status), namespacesPersistentMetrics = optionalNamespacesPersistentMetrics )) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetNamespacesPersistentMetricsResponse(status = Some(status))) } @@ -76,14 +76,14 @@ class MetricsServiceImpl extends MetricsServiceGrpc.MetricsService: // (namespace, getOptionalTenMetricsPb(metricsEntries, namespace)) // ).toMap // -// val status: Status = Status(code = Code.OK.index) +// val status: Status = Status(code = Code.OK.value) // Future.successful(GetTenantsMetricsResponse( // status = Some(status), // namespacesMetrics = optionalNamespacesMetrics // )) // } catch { // case err => -// val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) +// val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) // Future.successful(GetTenantsMetricsResponse(status = Some(status))) // } diff --git a/server/src/main/scala/namespace/NamespaceServiceImpl.scala b/server/src/main/scala/namespace/NamespaceServiceImpl.scala index 6bd6aa30e..22a4f779a 100644 --- a/server/src/main/scala/namespace/NamespaceServiceImpl.scala +++ b/server/src/main/scala/namespace/NamespaceServiceImpl.scala @@ -89,11 +89,11 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: try { adminClient.namespaces.createNamespace(request.namespaceName, policies) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(CreateNamespaceResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateNamespaceResponse(status = Some(status))) } @@ -104,11 +104,11 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: try { adminClient.namespaces.deleteNamespace(request.namespaceName, request.force) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(DeleteNamespaceResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteNamespaceResponse(status = Some(status))) } @@ -121,11 +121,11 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: adminClient.namespaces.getNamespaces(request.tenant).asScala catch { case err: Exception => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.ListNamespacesResponse(status = Some(status))) } - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ListNamespacesResponse(status = Some(status), namespaces = namespaces.toSeq)) override def getTopicsCount(request: GetTopicsCountRequest): Future[GetTopicsCountResponse] = @@ -194,7 +194,7 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: (ns, count) ).toMap - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetTopicsCountResponse( status = Some(status), @@ -206,7 +206,7 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetTopicsCountResponse(status = Some(status))) } @@ -225,19 +225,19 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: try { val permissions = Option(adminClient.namespaces.getPermissions(request.namespace).asScala.toMap) match case None => - val status = Status(code = Code.INTERNAL.index) + val status = Status(code = Code.INTERNAL.value) return Future.successful(GetPermissionsResponse(status = Some(status))) case Some(v) => v.map(x => x._1 -> new pb.AuthActions(authActions = x._2.asScala.toList.map(authActionToPb))) Future.successful( GetPermissionsResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), permissions ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetPermissionsResponse(status = Some(status))) } override def grantPermissions(request: GrantPermissionsRequest): Future[GrantPermissionsResponse] = @@ -258,7 +258,7 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: if permissions.exists(_._1 == request.role && request.existenceCheck) then val status = Status( - code = Code.FAILED_PRECONDITION.index, + code = Code.FAILED_PRECONDITION.value, message = s"There are already granted permissions for this role: ${request.role}. Please choose another role name." ) return Future.successful(GrantPermissionsResponse(status = Some(status))) @@ -267,12 +267,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( GrantPermissionsResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GrantPermissionsResponse(status = Some(status))) } @@ -285,12 +285,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( RevokePermissionsResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RevokePermissionsResponse(status = Some(status))) } @@ -301,7 +301,7 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: try { val permissions = Option(adminClient.namespaces.getPermissionOnSubscription(request.namespace).asScala.toMap) match case None => - val status = Status(code = Code.INTERNAL.index) + val status = Status(code = Code.INTERNAL.value) return Future.successful(GetPermissionOnSubscriptionResponse(status = Some(status))) case Some(v) => v.collect { @@ -312,14 +312,14 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( GetPermissionOnSubscriptionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), permissions, roles ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetPermissionOnSubscriptionResponse(status = Some(status))) } @@ -332,7 +332,7 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: if permissions.exists(_._1 == request.subscription && request.existenceCheck) then val status = Status( - code = Code.FAILED_PRECONDITION.index, + code = Code.FAILED_PRECONDITION.value, message = s"There are already assigned roles for this subscription: ${request.subscription}. Please choose another subscription name." ) return Future.successful(GrantPermissionOnSubscriptionResponse(status = Some(status))) @@ -340,12 +340,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: adminClient.namespaces.grantPermissionOnSubscription(request.namespace, request.subscription, request.roles.toSet.asJava) Future.successful( GrantPermissionOnSubscriptionResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GrantPermissionOnSubscriptionResponse(status = Some(status))) } @@ -358,12 +358,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( RevokePermissionOnSubscriptionResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RevokePermissionOnSubscriptionResponse(status = Some(status))) } @@ -382,13 +382,13 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( GetPropertiesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), properties ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetPropertiesResponse(status = Some(status))) } override def setProperties(request: SetPropertiesRequest): Future[SetPropertiesResponse] = @@ -405,12 +405,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( SetPropertiesResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetPropertiesResponse(status = Some(status))) } @@ -423,12 +423,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( UnloadNamespaceResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(UnloadNamespaceResponse(status = Some(status))) } @@ -441,12 +441,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( UnloadNamespaceBundleResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(UnloadNamespaceBundleResponse(status = Some(status))) } @@ -459,12 +459,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( ClearNamespaceBacklogResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(ClearNamespaceBacklogResponse(status = Some(status))) } @@ -477,12 +477,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( ClearBundleBacklogResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(ClearBundleBacklogResponse(status = Some(status))) } @@ -502,12 +502,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( SplitNamespaceBundleResponse( - status = Some(Status(code = Code.OK.index)) + status = Some(Status(code = Code.OK.value)) ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SplitNamespaceBundleResponse(status = Some(status))) } @@ -520,12 +520,12 @@ class NamespaceServiceImpl extends NamespaceServiceGrpc.NamespaceService: Future.successful( GetBundlesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), bundles = Option(bundles.getBoundaries).map(_.asScala.toSeq).getOrElse(Seq.empty).sliding(2).map { case List(a, b) => s"${a}_$b" }.toSeq ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetBundlesResponse(status = Some(status))) } diff --git a/server/src/main/scala/namespace_policies/NamespacePoliciesServiceImpl.scala b/server/src/main/scala/namespace_policies/NamespacePoliciesServiceImpl.scala index da4a26a06..5f4a3330e 100644 --- a/server/src/main/scala/namespace_policies/NamespacePoliciesServiceImpl.scala +++ b/server/src/main/scala/namespace_policies/NamespacePoliciesServiceImpl.scala @@ -24,7 +24,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val isAllowAutoUpdateSchema = adminClient.namespaces.getIsAllowAutoUpdateSchema(request.namespace) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetIsAllowAutoUpdateSchemaResponse( status = Some(status), @@ -33,7 +33,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetIsAllowAutoUpdateSchemaResponse(status = Some(status))) } @@ -43,11 +43,11 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { adminClient.namespaces.setIsAllowAutoUpdateSchema(request.namespace, request.isAllowAutoUpdateSchema) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(SetIsAllowAutoUpdateSchemaResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetIsAllowAutoUpdateSchemaResponse(status = Some(status))) } @@ -56,7 +56,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val strategy = adminClient.namespaces.getSchemaCompatibilityStrategy(request.namespace) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetSchemaCompatibilityStrategyResponse( status = Some(status), @@ -65,7 +65,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSchemaCompatibilityStrategyResponse(status = Some(status))) } @@ -80,11 +80,11 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac request.namespace, schemaCompatibilityStrategyFromPb(request.strategy) ) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(SetSchemaCompatibilityStrategyResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSchemaCompatibilityStrategyResponse(status = Some(status))) } @@ -94,7 +94,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val schemaValidationEnforced = adminClient.namespaces.getSchemaValidationEnforced(request.namespace) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetSchemaValidationEnforceResponse( status = Some(status), @@ -103,7 +103,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSchemaValidationEnforceResponse(status = Some(status))) } @@ -113,11 +113,11 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting schema validation enforce policy for namespace ${request.namespace}") adminClient.namespaces.setSchemaValidationEnforced(request.namespace, request.schemaValidationEnforced) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(SetSchemaValidationEnforceResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSchemaValidationEnforceResponse(status = Some(status))) } @@ -133,13 +133,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac Future.successful( GetAutoSubscriptionCreationResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), autoSubscriptionCreation = autoSubscriptionCreationPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetAutoSubscriptionCreationResponse(status = Some(status))) } @@ -155,14 +155,14 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac case pb.AutoSubscriptionCreation.AUTO_SUBSCRIPTION_CREATION_DISABLED => AutoSubscriptionCreationOverride.builder.allowAutoSubscriptionCreation(false).build() case _ => - val status = Status(code = Code.INVALID_ARGUMENT.index, message = "Wrong allow subscription creation argument received") + val status = Status(code = Code.INVALID_ARGUMENT.value, message = "Wrong allow subscription creation argument received") return Future.successful(SetAutoSubscriptionCreationResponse(status = Some(status))) adminClient.namespaces.setAutoSubscriptionCreation(request.namespace, autoSubscriptionCreationOverride) - Future.successful(SetAutoSubscriptionCreationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetAutoSubscriptionCreationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetAutoSubscriptionCreationResponse(status = Some(status))) } @@ -173,10 +173,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing auto subscription creation policy for namespace ${request.namespace}") adminClient.namespaces.removeAutoSubscriptionCreation(request.namespace) - Future.successful(RemoveAutoSubscriptionCreationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveAutoSubscriptionCreationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveAutoSubscriptionCreationResponse(status = Some(status))) } @@ -205,14 +205,14 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac Future.successful( GetAutoTopicCreationResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), autoTopicCreation = autoTopicCreationPb, autoTopicCreationOverride = autoTopicCreationOverridePb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetAutoTopicCreationResponse(status = Some(status))) } @@ -221,13 +221,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac val adminClient = RequestContext.pulsarAdmin.get() if !request.autoTopicCreation.isAutoTopicCreationSpecified then - val status = Status(code = Code.FAILED_PRECONDITION.index) + val status = Status(code = Code.FAILED_PRECONDITION.value) return Future.successful(SetAutoTopicCreationResponse(status = Some(status))) val autoTopicCreationOverridePb = request.autoTopicCreationOverride match case Some(v) => v case _ => - val status = Status(code = Code.FAILED_PRECONDITION.index) + val status = Status(code = Code.FAILED_PRECONDITION.value) return Future.successful(SetAutoTopicCreationResponse(status = Some(status))) try { @@ -250,10 +250,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .build adminClient.namespaces.setAutoTopicCreation(request.namespace, autoTopicCreation) - Future.successful(SetAutoTopicCreationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetAutoTopicCreationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetAutoTopicCreationResponse(status = Some(status))) } @@ -263,10 +263,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { adminClient.namespaces.removeAutoTopicCreation(request.namespace) - Future.successful(RemoveAutoTopicCreationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveAutoTopicCreationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveAutoTopicCreationResponse(status = Some(status))) } @@ -307,14 +307,14 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac Future.successful( GetBacklogQuotasResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), destinationStorage = destinationStorageBacklogQuotaPb, messageAge = messageAgeBacklogQuotaPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetBacklogQuotasResponse(status = Some(status))) } @@ -361,10 +361,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac adminClient.namespaces.setBacklogQuota(request.namespace, backlogQuota, BacklogQuotaType.message_age) case None => - Future.successful(SetBacklogQuotasResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetBacklogQuotasResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetBacklogQuotasResponse(status = Some(status))) } @@ -380,13 +380,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac logger.info(s"Removing backlog quota (message age) on namespace ${request.namespace}") adminClient.namespaces.removeBacklogQuota(request.namespace, BacklogQuotaType.message_age) case _ => - val status = Status(code = Code.INVALID_ARGUMENT.index, message = "Backlog quota type should be specified") + val status = Status(code = Code.INVALID_ARGUMENT.value, message = "Backlog quota type should be specified") return Future.successful(RemoveBacklogQuotaResponse(status = Some(status))) - Future.successful(RemoveBacklogQuotaResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveBacklogQuotaResponse(status = Some(Status(code = Code.OK.value)))) catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveBacklogQuotaResponse(status = Some(status))) } @@ -398,13 +398,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac Future.successful( GetNamespaceAntiAffinityGroupResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), namespaceAntiAffinityGroup = namespaceAntiAffinityGroup ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetNamespaceAntiAffinityGroupResponse(status = Some(status))) } @@ -413,10 +413,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try adminClient.namespaces.setNamespaceAntiAffinityGroup(request.namespace, request.namespaceAntiAffinityGroup) - Future.successful(SetNamespaceAntiAffinityGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetNamespaceAntiAffinityGroupResponse(status = Some(Status(code = Code.OK.value)))) catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetNamespaceAntiAffinityGroupResponse(status = Some(status))) } @@ -425,10 +425,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try adminClient.namespaces.deleteNamespaceAntiAffinityGroup(request.namespace) - Future.successful(RemoveNamespaceAntiAffinityGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveNamespaceAntiAffinityGroupResponse(status = Some(Status(code = Code.OK.value)))) catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveNamespaceAntiAffinityGroupResponse(status = Some(status))) } @@ -446,13 +446,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .toList Future.successful( GetAntiAffinityNamespacesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), namespaces ) ) catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetAntiAffinityNamespacesResponse(status = Some(status))) } @@ -468,13 +468,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac case None => None Future.successful( GetBookieAffinityGroupResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), groupData = groupData ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetBookieAffinityGroupResponse(status = Some(status))) } @@ -491,10 +491,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac logger.info(s"Setting bookie affinity group for namespace ${request.namespace}. $groupData") adminClient.namespaces.setBookieAffinityGroup(request.namespace, groupData) - Future.successful(SetBookieAffinityGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetBookieAffinityGroupResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetBookieAffinityGroupResponse(status = Some(status))) } @@ -504,10 +504,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing bookie affinity group policy for namespace ${request.namespace}") adminClient.namespaces.deleteBookieAffinityGroup(request.namespace) - Future.successful(RemoveBookieAffinityGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveBookieAffinityGroupResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveBookieAffinityGroupResponse(status = Some(status))) } @@ -519,12 +519,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac case None => pb.GetCompactionThresholdResponse.Threshold.Disabled(new pb.CompactionThresholdDisabled()) case Some(v) => pb.GetCompactionThresholdResponse.Threshold.Enabled(new CompactionThresholdEnabled(threshold = v)) Future.successful(GetCompactionThresholdResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), threshold )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetCompactionThresholdResponse(status = Some(status))) } @@ -534,10 +534,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting compaction threshold policy for namespace ${request.namespace}. ${request.threshold}") adminClient.namespaces.setCompactionThreshold(request.namespace, request.threshold) - Future.successful(SetCompactionThresholdResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetCompactionThresholdResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetCompactionThresholdResponse(status = Some(status))) } @@ -547,10 +547,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing compaction threshold policy for namespace ${request.namespace}") adminClient.namespaces.removeCompactionThreshold(request.namespace) - Future.successful(RemoveCompactionThresholdResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveCompactionThresholdResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveCompactionThresholdResponse(status = Some(status))) } @@ -565,12 +565,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac pb.GetDeduplicationSnapshotIntervalResponse.Interval.Enabled(new DeduplicationSnapshotIntervalEnabled(interval = v)) Future.successful(GetDeduplicationSnapshotIntervalResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), interval )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetDeduplicationSnapshotIntervalResponse(status = Some(status))) } @@ -580,10 +580,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting deduplication snapshot interval policy for namespace ${request.namespace}. ${request.interval}") adminClient.namespaces.setDeduplicationSnapshotInterval(request.namespace, request.interval) - Future.successful(SetDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetDeduplicationSnapshotIntervalResponse(status = Some(status))) } @@ -593,10 +593,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing deduplication snapshot interval policy for namespace ${request.namespace}") adminClient.namespaces.removeDeduplicationSnapshotInterval(request.namespace) - Future.successful(RemoveDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveDeduplicationSnapshotIntervalResponse(status = Some(status))) } @@ -611,12 +611,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac pb.GetDeduplicationResponse.Deduplication.Specified(new DeduplicationSpecified(enabled = v)) Future.successful(GetDeduplicationResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), deduplication )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetDeduplicationResponse(status = Some(status))) } @@ -626,10 +626,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting deduplication policy for namespace ${request.namespace}") adminClient.namespaces.setDeduplicationStatus(request.namespace, request.enabled) - Future.successful(SetDeduplicationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetDeduplicationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetDeduplicationResponse(status = Some(status))) } @@ -639,10 +639,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing deduplication policy for namespace ${request.namespace}") adminClient.namespaces.removeDeduplicationStatus(request.namespace) - Future.successful(RemoveDeduplicationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveDeduplicationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveDeduplicationResponse(status = Some(status))) } @@ -660,12 +660,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetDelayedDeliveryResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), delayedDelivery = delayedDeliveryPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetDelayedDeliveryResponse(status = Some(status))) } @@ -680,10 +680,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .build() adminClient.namespaces.setDelayedDeliveryMessages(request.namespace, delayedDeliveryPolicies) - Future.successful(SetDelayedDeliveryResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetDelayedDeliveryResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetDelayedDeliveryResponse(status = Some(status))) } @@ -693,10 +693,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing delayed delivery policy for namespace ${request.namespace}") adminClient.namespaces.removeDelayedDeliveryMessages(request.namespace) - Future.successful(RemoveDelayedDeliveryResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveDelayedDeliveryResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveDelayedDeliveryResponse(status = Some(status))) } @@ -716,12 +716,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), dispatchRate = dispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetDispatchRateResponse(status = Some(status))) } @@ -738,10 +738,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .build adminClient.namespaces.setDispatchRate(request.namespace, dispatchRate) - Future.successful(SetDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetDispatchRateResponse(status = Some(status))) } @@ -751,10 +751,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing dispatch rate policy for namespace ${request.namespace}") adminClient.namespaces.removeDispatchRate(request.namespace) - Future.successful(RemoveDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveDispatchRateResponse(status = Some(status))) } @@ -764,17 +764,17 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val encryptionRequired = Option(adminClient.namespaces.getEncryptionRequiredStatus(request.namespace)) match case None => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = "Can't fetch encryption status from broker") + val status = Status(code = Code.FAILED_PRECONDITION.value, message = "Can't fetch encryption status from broker") return Future.successful(GetEncryptionRequiredResponse(status = Some(status))) case Some(v) => v Future.successful(GetEncryptionRequiredResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), encryptionRequired )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetEncryptionRequiredResponse(status = Some(status))) } @@ -784,10 +784,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting encryption required policy for namespace ${request.namespace}") adminClient.namespaces.setEncryptionRequiredStatus(request.namespace, request.encryptionRequired) - Future.successful(SetEncryptionRequiredResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetEncryptionRequiredResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetEncryptionRequiredResponse(status = Some(status))) } @@ -812,12 +812,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetInactiveTopicPoliciesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), inactiveTopicPolicies = inactiveTopicPoliciesPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetInactiveTopicPoliciesResponse(status = Some(status))) } @@ -840,10 +840,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac throw new IllegalArgumentException("Invalid inactiveTopicDeleteMode mode") adminClient.namespaces.setInactiveTopicPolicies(request.namespace, inactiveTopicPolicies) - Future.successful(SetInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetInactiveTopicPoliciesResponse(status = Some(status))) } @@ -853,10 +853,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing inactive topic policies policy for namespace ${request.namespace}") adminClient.namespaces.removeInactiveTopicPolicies(request.namespace) - Future.successful(RemoveInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveInactiveTopicPoliciesResponse(status = Some(status))) } @@ -873,12 +873,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxConsumersPerSubscriptionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxConsumersPerSubscription = maxConsumersPerSubscriptionPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxConsumersPerSubscriptionResponse(status = Some(status))) } @@ -888,10 +888,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max consumers per subscription policy for namespace ${request.namespace}") adminClient.namespaces.setMaxConsumersPerSubscription(request.namespace, request.maxConsumersPerSubscription) - Future.successful(SetMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxConsumersPerSubscriptionResponse(status = Some(status))) } @@ -901,10 +901,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max consumers per subscription policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxConsumersPerSubscription(request.namespace) - Future.successful(RemoveMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxConsumersPerSubscriptionResponse(status = Some(status))) } @@ -921,12 +921,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxConsumersPerTopicResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxConsumersPerTopic = maxConsumersPerTopicPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxConsumersPerTopicResponse(status = Some(status))) } @@ -936,10 +936,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max consumers per topic policy for namespace ${request.namespace}") adminClient.namespaces.setMaxConsumersPerTopic(request.namespace, request.maxConsumersPerTopic) - Future.successful(SetMaxConsumersPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxConsumersPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxConsumersPerTopicResponse(status = Some(status))) } @@ -949,10 +949,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max consumers per topic policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxConsumersPerTopic(request.namespace) - Future.successful(RemoveMaxConsumersPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxConsumersPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxConsumersPerTopicResponse(status = Some(status))) } @@ -969,12 +969,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxProducersPerTopicResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxProducersPerTopic = maxProducersPerTopicPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxProducersPerTopicResponse(status = Some(status))) } @@ -984,10 +984,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max producers per topic policy for namespace ${request.namespace}") adminClient.namespaces.setMaxProducersPerTopic(request.namespace, request.maxProducersPerTopic) - Future.successful(SetMaxProducersPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxProducersPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxProducersPerTopicResponse(status = Some(status))) } @@ -997,10 +997,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max producers per topic policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxProducersPerTopic(request.namespace) - Future.successful(RemoveMaxProducersPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxProducersPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxProducersPerTopicResponse(status = Some(status))) } @@ -1017,12 +1017,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxSubscriptionsPerTopicResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxSubscriptionsPerTopic = maxSubscriptionsPerTopicPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxSubscriptionsPerTopicResponse(status = Some(status))) } @@ -1032,10 +1032,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max subscriptions per topic policy for namespace ${request.namespace}") adminClient.namespaces.setMaxSubscriptionsPerTopic(request.namespace, request.maxSubscriptionsPerTopic) - Future.successful(SetMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxSubscriptionsPerTopicResponse(status = Some(status))) } @@ -1045,10 +1045,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max subscriptions per topic policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxSubscriptionsPerTopic(request.namespace) - Future.successful(RemoveMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxSubscriptionsPerTopicResponse(status = Some(status))) } @@ -1065,12 +1065,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxTopicsPerNamespaceResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxTopicsPerNamespace = maxTopicsPerNamespacePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxTopicsPerNamespaceResponse(status = Some(status))) } @@ -1080,10 +1080,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max topics per namespace policy for namespace ${request.namespace}") adminClient.namespaces.setMaxTopicsPerNamespace(request.namespace, request.maxTopicsPerNamespace) - Future.successful(SetMaxTopicsPerNamespaceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxTopicsPerNamespaceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxTopicsPerNamespaceResponse(status = Some(status))) } @@ -1093,10 +1093,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max topics per namespace policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxTopicsPerNamespace(request.namespace) - Future.successful(RemoveMaxTopicsPerNamespaceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxTopicsPerNamespaceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxTopicsPerNamespaceResponse(status = Some(status))) } @@ -1113,12 +1113,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxUnackedMessagesPerConsumerResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxUnackedMessagesPerConsumer = maxUnackedMessagesPerConsumerPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxUnackedMessagesPerConsumerResponse(status = Some(status))) } @@ -1128,10 +1128,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max unacked messages per consumer policy for namespace ${request.namespace}") adminClient.namespaces.setMaxUnackedMessagesPerConsumer(request.namespace, request.maxUnackedMessagesPerConsumer) - Future.successful(SetMaxUnackedMessagesPerConsumerResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxUnackedMessagesPerConsumerResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxUnackedMessagesPerConsumerResponse(status = Some(status))) } @@ -1141,10 +1141,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max unacked messages per consumer policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxUnackedMessagesPerConsumer(request.namespace) - Future.successful(RemoveMaxUnackedMessagesPerConsumerResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxUnackedMessagesPerConsumerResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxUnackedMessagesPerConsumerResponse(status = Some(status))) } @@ -1161,12 +1161,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMaxUnackedMessagesPerSubscriptionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxUnackedMessagesPerSubscription = maxUnackedMessagesPerSubscriptionPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMaxUnackedMessagesPerSubscriptionResponse(status = Some(status))) } @@ -1176,10 +1176,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting max unacked messages per subscription policy for namespace ${request.namespace}") adminClient.namespaces.setMaxUnackedMessagesPerSubscription(request.namespace, request.maxUnackedMessagesPerSubscription) - Future.successful(SetMaxUnackedMessagesPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMaxUnackedMessagesPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMaxUnackedMessagesPerSubscriptionResponse(status = Some(status))) } @@ -1189,10 +1189,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing max unacked messages per subscription policy for namespace ${request.namespace}") adminClient.namespaces.removeMaxUnackedMessagesPerSubscription(request.namespace) - Future.successful(RemoveMaxUnackedMessagesPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMaxUnackedMessagesPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMaxUnackedMessagesPerSubscriptionResponse(status = Some(status))) } @@ -1209,12 +1209,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetMessageTtlResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), messageTtl = messageTtlPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetMessageTtlResponse(status = Some(status))) } @@ -1224,10 +1224,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting message TTL policy for namespace ${request.namespace}") adminClient.namespaces.setNamespaceMessageTTL(request.namespace, request.messageTtlSeconds) - Future.successful(SetMessageTtlResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetMessageTtlResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetMessageTtlResponse(status = Some(status))) } @@ -1237,10 +1237,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing message TTL policy for namespace ${request.namespace}") adminClient.namespaces.removeNamespaceMessageTTL(request.namespace) - Future.successful(RemoveMessageTtlResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveMessageTtlResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveMessageTtlResponse(status = Some(status))) } @@ -1257,12 +1257,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetOffloadDeletionLagResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), offloadDeletionLag = offloadDeletionLagPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetOffloadDeletionLagResponse(status = Some(status))) } @@ -1272,10 +1272,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting offload deletion lag policy for namespace ${request.namespace}") adminClient.namespaces.setOffloadDeleteLag(request.namespace, request.offloadDeletionLagMs, TimeUnit.MILLISECONDS) - Future.successful(SetOffloadDeletionLagResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetOffloadDeletionLagResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetOffloadDeletionLagResponse(status = Some(status))) } @@ -1285,10 +1285,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing offload deletion lag policy for namespace ${request.namespace}") adminClient.namespaces.clearOffloadDeleteLag(request.namespace) - Future.successful(RemoveOffloadDeletionLagResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveOffloadDeletionLagResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveOffloadDeletionLagResponse(status = Some(status))) } @@ -1298,7 +1298,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val offloadThresholdPb = Option(adminClient.namespaces.getOffloadThreshold(request.namespace)) match case None => - val status = Status(code = Code.FAILED_PRECONDITION.index) + val status = Status(code = Code.FAILED_PRECONDITION.value) return Future.successful(GetOffloadThresholdResponse(status = Some(status))) case Some(v) => pb.GetOffloadThresholdResponse.OffloadThreshold.Specified(new OffloadThresholdSpecified( @@ -1306,12 +1306,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetOffloadThresholdResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), offloadThreshold = offloadThresholdPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetOffloadThresholdResponse(status = Some(status))) } @@ -1321,10 +1321,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting offload threshold policy for namespace ${request.namespace}") adminClient.namespaces.setOffloadThreshold(request.namespace, request.offloadThresholdBytes) - Future.successful(SetOffloadThresholdResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetOffloadThresholdResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetOffloadThresholdResponse(status = Some(status))) } @@ -1344,12 +1344,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetPersistenceResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), persistence = persistencePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetPersistenceResponse(status = Some(status))) } @@ -1360,10 +1360,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac logger.info(s"Setting persistence policy for namespace ${request.namespace}") val persistencePolicies = PersistencePolicies(request.bookkeeperEnsemble, request.bookkeeperWriteQuorum, request.bookkeeperAckQuorum, request.managedLedgerMaxMarkDeleteRate) adminClient.namespaces.setPersistence(request.namespace, persistencePolicies) - Future.successful(SetPersistenceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetPersistenceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetPersistenceResponse(status = Some(status))) } @@ -1373,10 +1373,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing persistence policy for namespace ${request.namespace}") adminClient.namespaces.removePersistence(request.namespace) - Future.successful(RemovePersistenceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemovePersistenceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemovePersistenceResponse(status = Some(status))) } @@ -1389,12 +1389,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .getOrElse(Seq.empty[String]) Future.successful(GetReplicationClustersResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), replicationClusters )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetReplicationClustersResponse(status = Some(status))) } @@ -1404,10 +1404,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting replication clusters for namespace ${request.namespace}") adminClient.namespaces.setNamespaceReplicationClusters(request.namespace, request.replicationClusters.toSet.asJava) - Future.successful(SetReplicationClustersResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetReplicationClustersResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetReplicationClustersResponse(status = Some(status))) } @@ -1427,12 +1427,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetReplicatorDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), replicatorDispatchRate = replicatorDispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetReplicatorDispatchRateResponse(status = Some(status))) } @@ -1449,10 +1449,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .build adminClient.namespaces.setReplicatorDispatchRate(request.namespace, dispatchRate) - Future.successful(SetReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetReplicatorDispatchRateResponse(status = Some(status))) } @@ -1462,10 +1462,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing replicator dispatch rate for namespace ${request.namespace}") adminClient.namespaces.removeReplicatorDispatchRate(request.namespace) - Future.successful(RemoveReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveReplicatorDispatchRateResponse(status = Some(status))) } @@ -1485,12 +1485,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetSubscriptionDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionDispatchRate = subscriptionDispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionDispatchRateResponse(status = Some(status))) } @@ -1507,10 +1507,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac .build adminClient.namespaces.setSubscriptionDispatchRate(request.namespace, dispatchRate) - Future.successful(SetSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscriptionDispatchRateResponse(status = Some(status))) } @@ -1520,10 +1520,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing subscription dispatch rate for namespace ${request.namespace}") adminClient.namespaces.removeSubscriptionDispatchRate(request.namespace) - Future.successful(RemoveSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveSubscriptionDispatchRateResponse(status = Some(status))) } @@ -1541,12 +1541,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetRetentionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), retention = retentionPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetRetentionResponse(status = Some(status))) } @@ -1558,10 +1558,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac val retention = new RetentionPolicies(request.retentionTimeInMinutes, request.retentionSizeInMb) adminClient.namespaces.setRetention(request.namespace, retention) - Future.successful(SetRetentionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetRetentionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetRetentionResponse(status = Some(status))) } @@ -1571,10 +1571,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing retention for namespace ${request.namespace}") adminClient.namespaces.removeRetention(request.namespace) - Future.successful(RemoveRetentionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveRetentionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveRetentionResponse(status = Some(status))) } @@ -1592,12 +1592,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetSubscribeRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscribeRate = subscribeRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscribeRateResponse(status = Some(status))) } @@ -1609,10 +1609,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac val subscribeRate = new SubscribeRate(request.subscribeThrottlingRatePerConsumer, request.ratePeriodInSeconds) adminClient.namespaces.setSubscribeRate(request.namespace, subscribeRate) - Future.successful(SetSubscribeRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetSubscribeRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscribeRateResponse(status = Some(status))) } @@ -1622,10 +1622,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing subscribe rate policy for namespace ${request.namespace}") adminClient.namespaces.removeSubscribeRate(request.namespace) - Future.successful(RemoveSubscribeRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveSubscribeRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveSubscribeRateResponse(status = Some(status))) } @@ -1638,12 +1638,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac case SubscriptionAuthMode.Prefix => pb.SubscriptionAuthMode.SUBSCRIPTION_AUTH_MODE_PREFIX Future.successful(GetSubscriptionAuthModeResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionAuthMode = subscriptionAuthModePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionAuthModeResponse(status = Some(status))) } @@ -1656,13 +1656,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac case pb.SubscriptionAuthMode.SUBSCRIPTION_AUTH_MODE_NONE => SubscriptionAuthMode.None case pb.SubscriptionAuthMode.SUBSCRIPTION_AUTH_MODE_PREFIX => SubscriptionAuthMode.Prefix case _ => - return Future.successful(SetSubscriptionAuthModeResponse(status = Some(Status(code = Code.INVALID_ARGUMENT.index, message = "Invalid subscription auth mode")))) + return Future.successful(SetSubscriptionAuthModeResponse(status = Some(Status(code = Code.INVALID_ARGUMENT.value, message = "Invalid subscription auth mode")))) adminClient.namespaces.setSubscriptionAuthMode(request.namespace, subscriptionAuthMode) - Future.successful(SetSubscriptionAuthModeResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetSubscriptionAuthModeResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscriptionAuthModeResponse(status = Some(status))) } @@ -1679,12 +1679,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetSubscriptionExpirationTimeResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionExpirationTime = subscriptionExpirationTimePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionExpirationTimeResponse(status = Some(status))) } @@ -1694,10 +1694,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting subscription expiration time policy for namespace ${request.namespace}") adminClient.namespaces.setSubscriptionExpirationTime(request.namespace, request.subscriptionExpirationTimeInMinutes) - Future.successful(SetSubscriptionExpirationTimeResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetSubscriptionExpirationTimeResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscriptionExpirationTimeResponse(status = Some(status))) } @@ -1707,10 +1707,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing subscription expiration time policy for namespace ${request.namespace}") adminClient.namespaces.removeSubscriptionExpirationTime(request.namespace) - Future.successful(RemoveSubscriptionExpirationTimeResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveSubscriptionExpirationTimeResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveSubscriptionExpirationTimeResponse(status = Some(status))) } @@ -1727,7 +1727,7 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { val subscriptionTypesEnabledPb = Option(adminClient.namespaces.getSubscriptionTypesEnabled(request.namespace)) match case None => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = "Subscription types enabled can't be null. Looks like a Pulsar error.") + val status = Status(code = Code.FAILED_PRECONDITION.value, message = "Subscription types enabled can't be null. Looks like a Pulsar error.") return Future.successful(GetSubscriptionTypesEnabledResponse(status = Some(status))) case Some(v) if v.size() == 0 => pb.GetSubscriptionTypesEnabledResponse.SubscriptionTypesEnabled.Inherited(new SubscriptionTypesEnabledInherited()) @@ -1737,12 +1737,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetSubscriptionTypesEnabledResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionTypesEnabled = subscriptionTypesEnabledPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionTypesEnabledResponse(status = Some(status))) } @@ -1762,10 +1762,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac val subscriptionTypesEnabled = request.types.map(pbToSubscriptionType).toSet.asJava adminClient.namespaces.setSubscriptionTypesEnabled(request.namespace, subscriptionTypesEnabled) - Future.successful(SetSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscriptionTypesEnabledResponse(status = Some(status))) } @@ -1775,10 +1775,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing subscription types enabled policy for namespace ${request.namespace}") adminClient.namespaces.removeSubscriptionTypesEnabled(request.namespace) - Future.successful(RemoveSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveSubscriptionTypesEnabledResponse(status = Some(status))) } @@ -1830,12 +1830,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac pb.GetOffloadPoliciesResponse.OffloadPolicies.Specified(offloadPoliciesToPb(v)) Future.successful(GetOffloadPoliciesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), offloadPolicies = offloadPoliciesPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetOffloadPoliciesResponse(status = Some(status))) } @@ -1887,15 +1887,15 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac request.offloadPolicies match case None => - val status = Status(code = Code.INVALID_ARGUMENT.index, "Offload policies should be specified") + val status = Status(code = Code.INVALID_ARGUMENT.value, "Offload policies should be specified") Future.successful(SetOffloadPoliciesResponse(status = Some(status))) case Some(v) => val offloadPolicies = offloadPoliciesFromPb(v) adminClient.namespaces.setOffloadPolicies(request.namespace, offloadPolicies) - Future.successful(SetOffloadPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetOffloadPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetOffloadPoliciesResponse(status = Some(status))) } @@ -1905,10 +1905,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing offload policies policy for namespace ${request.namespace}") adminClient.namespaces.removeOffloadPolicies(request.namespace) - Future.successful(RemoveOffloadPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveOffloadPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveOffloadPoliciesResponse(status = Some(status))) } override def getPublishRate(request: GetPublishRateRequest): Future[GetPublishRateResponse] = @@ -1925,12 +1925,12 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac )) Future.successful(GetPublishRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), publishRate = publishRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetPublishRateResponse(status = Some(status))) } override def setPublishRate(request: SetPublishRateRequest): Future[SetPublishRateResponse] = @@ -1940,10 +1940,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac logger.info(s"Setting publish rate policy for namespace ${request.namespace}. ${request.rateInMsg}, ${request.rateInByte}") val publishRate = PublishRate(request.rateInMsg, request.rateInByte) adminClient.namespaces.setPublishRate(request.namespace, publishRate) - Future.successful(SetPublishRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(SetPublishRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetPublishRateResponse(status = Some(status))) } override def removePublishRate(request: RemovePublishRateRequest): Future[RemovePublishRateResponse] = @@ -1952,10 +1952,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing publish rate policy for namespace ${request.namespace}") adminClient.namespaces.removePublishRate(request.namespace) - Future.successful(RemovePublishRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemovePublishRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemovePublishRateResponse(status = Some(status))) } @@ -1974,13 +1974,13 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac val resourceGroups = Option(adminClient.resourcegroups.getResourceGroups).map(_.asScala.toSeq).getOrElse(Seq.empty[String]) Future.successful(pb.GetResourceGroupResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), resourceGroup, resourceGroups, )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetResourceGroupResponse(status = Some(status))) } @@ -1990,10 +1990,10 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Setting resource group policy for namespace ${request.namespace}") adminClient.namespaces.setNamespaceResourceGroup(request.namespace, request.resourceGroup) - Future.successful(pb.SetResourceGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetResourceGroupResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetResourceGroupResponse(status = Some(status))) } @@ -2003,9 +2003,9 @@ class NamespacePoliciesServiceImpl extends NamespacePoliciesServiceGrpc.Namespac try { logger.info(s"Removing resource group policy for namespace ${request.namespace}") adminClient.namespaces.removeNamespaceResourceGroup(request.namespace) - Future.successful(RemoveResourceGroupResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(RemoveResourceGroupResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(RemoveResourceGroupResponse(status = Some(status))) } diff --git a/server/src/main/scala/producer/ProducerServiceImpl.scala b/server/src/main/scala/producer/ProducerServiceImpl.scala index ac1339ab2..b62096b45 100644 --- a/server/src/main/scala/producer/ProducerServiceImpl.scala +++ b/server/src/main/scala/producer/ProducerServiceImpl.scala @@ -1,6 +1,6 @@ package producer -import org.apache.pulsar.client.api.{Producer, ProducerAccessMode, Schema} +import org.apache.pulsar.client.api.{MessageId, Producer, ProducerAccessMode, Schema} import com.typesafe.scalalogging.Logger import com.google.rpc.status.Status import com.google.rpc.code.Code @@ -18,14 +18,32 @@ import io.circe.parser.parse as parseJson import pulsar_auth.RequestContext import java.nio.ByteBuffer -import scala.concurrent.Future +import java.util.concurrent.{CompletableFuture, ConcurrentHashMap} +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.FutureConverters.* import scala.util.boundary, boundary.break type ProducerName = String class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: val logger: Logger = Logger(getClass.getName) - var producers: Map[ProducerName, Producer[Array[Byte]]] = Map.empty + + /** Live broker producers, by the name the UI created them under. + * + * A ConcurrentHashMap rather than a `var Map`, because every entry is a resource: this service + * is a singleton bound on `ExecutionContext.global` (GrpcServer), so create/delete run + * concurrently, and `producers = producers + (name -> p)` is a read-modify-write. Two creates + * that read the same snapshot lost one entry - that producer stayed connected to the topic + * while its name vanished from the only map that could close it, so `deleteProducer` answered + * "no such producer" forever. `put`/`remove` here are atomic AND hand back whatever they + * displaced, which is what makes closing it possible at all. */ + private[producer] val producers: ConcurrentHashMap[ProducerName, Producer[Array[Byte]]] = ConcurrentHashMap() + + /** Close a producer this service is giving up. A failure here must not fail the RPC that caused + * it - the entry is already gone from the registry either way - but it must not be silent. */ + private def closeReleased(producerName: ProducerName, producer: Producer[Array[Byte]]): Unit = + try producer.close() + catch case err => logger.warn(s"Failed to close producer $producerName: ${err.getMessage}") override def createProducer(request: CreateProducerRequest): Future[CreateProducerResponse] = val producerName: ProducerName = request.producerName @@ -42,13 +60,16 @@ class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: .topic(request.topic) .create() - producers = producers + (producerName -> producer) + // Registering under a name that is already taken (a re-create after an edit or a page + // reload, or a concurrent create that got here second) used to overwrite the entry and + // leak the producer it displaced. Whatever this replaces is ours to close. + Option(producers.put(producerName, producer)).foreach(closeReleased(producerName, _)) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(CreateProducerResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateProducerResponse(status = Some(status))) } @@ -56,21 +77,23 @@ class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: val producerName: ProducerName = request.producerName logger.info(s"Deleting producer: $producerName") - producers.get(producerName) match + // Remove-then-close as one atomic claim: the producer this call closes is exactly the one it + // took out of the registry, so two concurrent deletes cannot both close the same handle and + // a create racing alongside cannot have its brand-new producer removed by the loser. + Option(producers.remove(producerName)) match case Some(p) => try { - producers = producers.removed(producerName) p.close() - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(DeleteProducerResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteProducerResponse(status = Some(status))) } case _ => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = s"No such producer: $producerName") + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = s"No such producer: $producerName") Future.successful(DeleteProducerResponse(status = Some(status))) override def send(request: SendRequest): Future[SendResponse] = boundary: @@ -78,10 +101,10 @@ class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: logger.info(s"Sending message. Producer: $producerName") val adminClient = RequestContext.pulsarAdmin.get() - val producer = producers.get(producerName) match + val producer = Option(producers.get(producerName)) match case Some(p) => p case _ => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = s"No such producer: $producerName") + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = s"No such producer: $producerName") break(Future.successful(SendResponse(status = Some(status)))) val messages: Seq[Either[Throwable, Message]] = request.format match @@ -92,7 +115,7 @@ class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: catch { case _: PulsarAdminException.NotFoundException => None case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) break(Future.successful(SendResponse(status = Some(status)))) } @@ -117,35 +140,85 @@ class ProducerServiceImpl extends ProducerServiceGrpc.ProducerService: Right(message) ) - messages.foreach(msg => - msg match - case Right(message) => - try { - var newMessage = producer.newMessage - .value(message.value) - .properties(message.properties.asJava) - message.eventTime match - case Some(t) => newMessage = newMessage.eventTime(t) - case None => // do nothing - message.key match - case Some(k) => newMessage = newMessage.key(k) - case None => // do nothing - newMessage.sendAsync - } catch { - case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) - break(Future.successful(SendResponse(status = Some(status)))) - } - case Left(err) => - val status: Status = Status(code = Code.INVALID_ARGUMENT.index, message = err.getMessage) - break(Future.successful(SendResponse(status = Some(status)))) - ) - - val status: Status = Status(code = Code.OK.index) - Future.successful(SendResponse(status = Some(status))) + // Validate the WHOLE batch before publishing any of it. Validation used to be INTERLEAVED + // with `sendAsync` - each item was checked immediately before its own send - so a valid item + // preceding an invalid one was already on the topic when the call answered INVALID_ARGUMENT. + // The caller sees a wholly failed batch, retries it, and duplicates whatever did land. + messages.collectFirst { case Left(err) => err } match + case Some(err) => + val status: Status = Status(code = Code.INVALID_ARGUMENT.value, message = err.getMessage) + break(Future.successful(SendResponse(status = Some(status)))) + case None => // the whole batch converted; publishing it is safe + + // Publication is NOT atomic and cannot be made so after the fact: the items are submitted one + // at a time, and a send already handed to the client cannot be recalled. A builder/`sendAsync` + // that throws on item N (producer closed, payload over the max message size) used to break + // straight out of `send` with FAILED_PRECONDITION, ABANDONING the futures of items 1..N-1 - + // so the RPC reported failure while part of its own batch was still travelling to the topic, + // and the caller's retry duplicated whatever landed. Stop submitting at the first failure, + // but keep every future that was submitted and let `awaitSends` settle all of them before the + // verdict exists. A partial publish therefore remains possible; what is guaranteed is that + // the batch is finished travelling by the time its verdict is delivered. + val sendFutures = scala.collection.mutable.ArrayBuffer.empty[CompletableFuture[MessageId]] + var submitFailure: Option[Throwable] = None + + val toPublish = messages.collect { case Right(message) => message }.iterator + while toPublish.hasNext && submitFailure.isEmpty do + val message = toPublish.next() + try { + var newMessage = producer.newMessage + .value(message.value) + .properties(message.properties.asJava) + message.eventTime match + case Some(t) => newMessage = newMessage.eventTime(t) + case None => // do nothing + message.key match + case Some(k) => newMessage = newMessage.key(k) + case None => // do nothing + sendFutures += newMessage.sendAsync + } catch { + case err => submitFailure = Some(err) + } + + awaitSends(sendFutures.toSeq, submitFailure) override def getStats(request: GetStatsRequest): Future[GetStatsResponse] = ??? +/** Build the send response from the in-flight `sendAsync` futures, plus the failure (if any) that + * stopped `send` from submitting the rest of the batch. + * + * A Pulsar send future completes only when the BROKER has acknowledged (or rejected) the message, + * so answering before then is a guess: `send` used to discard every future and return Code.OK + * immediately, which reported a successful publish for messages the broker went on to reject + * (schema incompatibility, producer fenced, exceeded quota, terminated topic, send timeout). + * + * Every submitted future must SETTLE before the verdict exists. `Future.sequence` is fail-fast, so + * one rejection completed the RPC as failed while its siblings were still in flight; those siblings + * then landed on the topic AFTER the caller had been told the batch failed, and the natural retry + * duplicated them. Each future is therefore lifted to a `Try` (which never fails) and only the + * collected results decide the answer. The semantics are still non-atomic - a partial publish is + * possible and a retry can duplicate what landed - but the batch is no longer in motion when its + * verdict is delivered. + * + * Composed rather than blocked on, so the gRPC thread is not parked while the broker decides; + * `parasitic` runs the continuation on whichever thread completes the last future. + */ +def awaitSends(sendFutures: Seq[CompletableFuture[MessageId]], submitFailure: Option[Throwable] = None): Future[SendResponse] = + given ExecutionContext = ExecutionContext.parasitic + + val settled: Seq[Future[scala.util.Try[MessageId]]] = + sendFutures.map(_.asScala.transform(scala.util.Success(_))) + + Future.sequence(settled).map { results => + // The submit failure wins when there is one: it is the reason the batch is incomplete, and + // it says more than a sibling's broker error would. + submitFailure.orElse(results.collectFirst { case scala.util.Failure(err) => err }) match + case None => SendResponse(status = Some(Status(code = Code.OK.value))) + case Some(err) => + val message = Option(err.getMessage).getOrElse(err.toString) + SendResponse(status = Some(Status(code = Code.FAILED_PRECONDITION.value, message = s"Failed to send message. $message"))) + } + case class Message( key: Option[String], value: Array[Byte], @@ -153,6 +226,10 @@ case class Message( properties: Map[String, String] ) +/** True only for a payload that is a single, well-formed JSON number. */ +def isJsonNumber(payload: String): Boolean = + parseJson(payload).exists(_.isNumber) + def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwable, Array[Byte]] = val result: Either[Throwable, Array[Byte]] = schemaInfo.getType match case SchemaType.AVRO => @@ -163,13 +240,23 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa protobufnative.converters.fromJson(schemaInfo.getSchema, jsonAsBytes) match case Right(v) => Right(v) case Left(err) => Left(err) - case SchemaType.JSON => Right(jsonAsBytes) + case SchemaType.JSON => + // The topic's schema says JSON, so the payload has to BE JSON. Returning the bytes + // unparsed published anything at all - the producer reported success and the consumer + // side then failed to deserialize what had already landed on the topic. Syntax only: + // the payload is not validated against the schema definition (that would need a JSON + // Schema validator), and the bytes are forwarded unchanged rather than re-serialized. + parseJson(String(jsonAsBytes, "UTF-8")) match + case Right(_) => Right(jsonAsBytes) + case Left(err) => Left(new Exception(s"Message should be formatted as JSON. ${err.getMessage}")) case SchemaType.STRING => parseJson(String(jsonAsBytes, "UTF-8")) match case Left(err) => Left(err) case Right(json) if json.isString => val str = json.asString.getOrElse("") - Right(str.getBytes) + // Explicit UTF-8: the read path (primitiveConv.bytesToString) decodes as UTF-8, + // so relying on the platform default here would diverge under -Dfile.encoding. + Right(str.getBytes(java.nio.charset.StandardCharsets.UTF_8)) case _ => Left(new Exception("Message should be formatted as JSON string.")) case SchemaType.NONE => Right(jsonAsBytes) case SchemaType.BOOLEAN => @@ -182,6 +269,12 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(Array(v)) case SchemaType.INT8 => val jsonString = String(jsonAsBytes, "UTF-8") + // Guava's parser accepts JAVA integer literal syntax, not JSON: a leading zero (`01`, + // `00`, `-01`) parsed and was encoded onto the topic even though JSON forbids it. Same + // gate the FLOAT/DOUBLE branches below already apply, for the same reason - this is the + // JSON message format. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse INT8 value from the given JSON: $jsonString")) + val n = primitives.Ints.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse INT8 value from the given JSON: $jsonString")) @@ -193,6 +286,9 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(Array(primitives.SignedBytes.checkedCast(n.toLong))) case SchemaType.INT16 => val jsonString = String(jsonAsBytes, "UTF-8") + // See the INT8 branch: Guava accepts leading zeros, JSON does not. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse INT16 value from the given the JSON: $jsonString")) + val n = primitives.Ints.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse INT16 value from the given the JSON: $jsonString")) @@ -203,6 +299,9 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(primitives.Shorts.toByteArray(n.toShort)) case SchemaType.INT32 => val jsonString = String(jsonAsBytes, "UTF-8") + // See the INT8 branch: Guava accepts leading zeros, JSON does not. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse INT32 value from the given the JSON: $jsonString")) + val n = primitives.Ints.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse INT32 value from the given the JSON: $jsonString")) @@ -213,6 +312,9 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(primitives.Ints.toByteArray(n)) case SchemaType.INT64 => val jsonString = String(jsonAsBytes, "UTF-8") + // See the INT8 branch: Guava accepts leading zeros, JSON does not. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse INT64 value from the given the JSON: $jsonString")) + val n = primitives.Longs.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse INT64 value from the given the JSON: $jsonString")) @@ -223,9 +325,19 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(primitives.Longs.toByteArray(n)) case SchemaType.FLOAT => val jsonString = String(jsonAsBytes, "UTF-8") + // Guava's parser accepts JAVA float literal syntax, not JSON: `+1`, `01`, `.5`, `1.`, + // hex float literals and a trailing f/d suffix all parsed and were encoded onto the + // topic. Gate on JSON number syntax first (the STRING branch below has always required + // real JSON), then let Guava do the numeric conversion. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse FLOAT value from the given JSON: $jsonString")) + val n = primitives.Floats.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse FLOAT value from the given JSON: $jsonString")) + // NaN compares false against BOTH bounds, so it slipped through the very guard that + // rejects Infinity and got encoded onto the topic (and NaN is not valid JSON either). + if n.isNaN then return Left(new Exception(s"FLOAT value must be a number. Given: $jsonString")) + val MinValue = Float.MinValue val MaxValue = Float.MaxValue if n > MaxValue || n < MinValue then return Left(new Exception(s"FLOAT value should be in range from $MinValue to $MaxValue. Given: $n")) @@ -233,9 +345,15 @@ def jsonToValue(schemaInfo: SchemaInfo, jsonAsBytes: Array[Byte]): Either[Throwa Right(ByteBuffer.allocate(4).putFloat(n).array) case SchemaType.DOUBLE => val jsonString = String(jsonAsBytes, "UTF-8") + // See the FLOAT branch: same Guava parser, same non-JSON literal forms. + if !isJsonNumber(jsonString) then return Left(new Exception(s"Unable to parse DOUBLE value from the given JSON: $jsonString")) + val n = primitives.Doubles.tryParse(jsonString) if n == null then return Left(new Exception(s"Unable to parse DOUBLE value from the given JSON: $jsonString")) + // See the FLOAT branch: NaN evades both bounds checks. + if n.isNaN then return Left(new Exception(s"DOUBLE value must be a number. Given: $jsonString")) + val MinValue = Double.MinValue val MaxValue = Double.MaxValue if n > MaxValue || n < MinValue then return Left(new Exception(s"DOUBLE value should be in range from $MinValue to $MaxValue. Given: $n")) diff --git a/server/src/main/scala/pulsar_auth/PulsarAuth.scala b/server/src/main/scala/pulsar_auth/PulsarAuth.scala index 2dcdc11f6..606ef085e 100644 --- a/server/src/main/scala/pulsar_auth/PulsarAuth.scala +++ b/server/src/main/scala/pulsar_auth/PulsarAuth.scala @@ -136,11 +136,29 @@ def parsePulsarAuthCookie(json: Option[String]): Either[Throwable, PulsarAuth] = val clientPulsarAuth = json match case None => Right(defaultPulsarAuth) case Some(encodedValue) => - val v = URLDecoder.decode(encodedValue, UTF_8) + // URLDecoder throws IllegalArgumentException on malformed percent-encoding ("%", "%ZZ", + // "a%2"), and it used to sit OUTSIDE this Either - so a hand-edited cookie produced a + // server error instead of the intended 400. + val v = + try URLDecoder.decode(encodedValue, UTF_8) + catch + case err: IllegalArgumentException => + logger.warn(s"Malformed percent-encoding in cookie: ${err.getMessage}") + return Left(new Exception("Unable to parse pulsar_auth cookie.")) decode[PulsarAuth](v) match case Left(err) => - logger.warn(s"Unable to parse cookie: ${err.getMessage}") + // The circe message must stay OUT of the log: for malformed JSON, jawn's + // message quotes the raw input at the failure offset (`expected : got + // '"eyJhb...'`), and a decode failure's message carries the cursor path - + // i.e. the credential names. The input here is the credential cookie + // (tokens, private keys), so log only the failure kind and position. + val safeDetail = err match + case ParsingFailure(_, cause: org.typelevel.jawn.ParseException) => + s"not valid JSON (line ${cause.line}, column ${cause.col})" + case _: ParsingFailure => "not valid JSON" + case _: DecodingFailure => "valid JSON, but not a valid PulsarAuth document" + logger.warn(s"Unable to parse cookie: $safeDetail") Left(new Exception(s"Unable to parse pulsar_auth cookie.")) case Right(pulsarAuth) => Right( pulsarAuth @@ -148,40 +166,70 @@ def parsePulsarAuthCookie(json: Option[String]): Either[Throwable, PulsarAuth] = clientPulsarAuth -def pulsarAuthToCookie(pulsarAuth: PulsarAuth): String = - val pulsarAuthWithoutEncodingMetadata = pulsarAuth.copy( - credentials = pulsarAuth.credentials.map((name, credentials) => - credentials match - case cr: OAuth2Credentials => ( - name, - cr.copy( - issuerUrl = URLEncoder.encode(cr.issuerUrl, UTF_8), - privateKey = URLEncoder.encode(cr.privateKey, UTF_8), - audience = cr.audience.map(audience => URLEncoder.encode(audience, UTF_8)), - scope = cr.scope.map(scope => URLEncoder.encode(scope, UTF_8)) - ) - ) - case _ => (name, credentials) - ) - ) - +/** The cookie-hardening inputs are parameters (defaulting to the process config, so the single + * production call site is unchanged) purely so they can be varied in tests - the package-level + * `config` val is loaded once per process and cannot be. */ +def pulsarAuthToCookie( + pulsarAuth: PulsarAuth, + publicBaseUrl: Option[String] = config.publicBaseUrl, + cookieSecure: Option[Boolean] = config.cookieSecure, + cookieSameSite: Option[String] = config.cookieSameSite +): String = val cookieName = "pulsar_auth" - val cookieValue = pulsarAuthWithoutEncodingMetadata.asJson.noSpaces - val cookiePath = config.publicBaseUrl.map { + // The ENTIRE serialized JSON is URL-encoded here, exactly once; parsePulsarAuthCookie + // URL-decodes the whole value exactly once. A lossless pair, with no raw remainder that + // could terminate the header value. + // + // The previous scheme encoded individual FIELDS (the OAuth2 fields and authParams) and wrote + // the surrounding JSON raw - but JSON does not escape `;`, and two strings travel here that + // are not charset-validated anywhere: `authPluginClassName` (free text from the + // /pulsar-auth/add BODY; only the name path segment is validated) and the credential NAMES + // (parsePulsarAuthCookie accepts any keys from a hand-crafted cookie, and + // setCookieAndSuccess echoes them back). A `;` in either truncated the stored cookie at the + // browser - the next request failed to parse, a silent credential corruption - and promoted + // the remainder to REAL cookie attributes (`Domain=evil.example`). setCookieAndSuccess also + // re-injects the operator-configured Default credentials into every response, so one bad + // configured value rewrote the Set-Cookie header for every user. + // + // Legacy cookies written by the per-field scheme still parse: the read path is unchanged, + // and a legacy value is raw JSON whose only `%xx`/`+` sequences sit inside the fields the + // old writer encoded - one whole-value decode yields exactly what the old reader saw. + val cookieValue = URLEncoder.encode(pulsarAuth.asJson.noSpaces, UTF_8) + + val cookiePath = publicBaseUrl.map { java.net.URI.create(_).getPath match case "" => "/" case path => path }.getOrElse("/") - val cookieSecureValue = config.cookieSecure match + val cookieSecureValue = cookieSecure match case Some(true) => "Secure; " case _ => "" - val cookieSameSiteValue = (config.cookieSecure, config.cookieSameSite) match - case (_, Some("lax")) => "SameSite=Lax; " - case (_, Some("strict")) => "SameSite=Strict; " - case (Some(true), Some("none")) => "SameSite=None; " - case _ => "" - - s"$cookieName=$cookieValue; Path=$cookiePath; HttpOnly; Max-Age=31536000; $cookieSameSiteValue$cookieSameSiteValue" + s"$cookieName=$cookieValue; Path=$cookiePath; HttpOnly; Max-Age=31536000; $cookieSecureValue${sameSiteAttribute(cookieSecure, cookieSameSite)}" + +/** Render the SameSite attribute, the cookie's built-in CSRF control. + * + * The value is matched case-insensitively and trimmed: it arrives from a YAML key or the + * DEKAF_COOKIE_SAME_SITE environment variable, where `Lax`, `STRICT` and a stray trailing space are + * all ordinary things for an operator to write. A strict lowercase match silently fell through to + * "", emitting NO SameSite attribute at all - so a capitalised value looked configured but left the + * cookie on the browser default. Anything still unrecognised after normalisation is logged loudly + * for the same reason: dropping the attribute must never be the quiet outcome of a typo. + */ +def sameSiteAttribute(cookieSecure: Option[Boolean], cookieSameSite: Option[String]): String = + cookieSameSite.map(_.trim.toLowerCase) match + case None | Some("") => "" + case Some("lax") => "SameSite=Lax; " + case Some("strict") => "SameSite=Strict; " + case Some("none") => + // Browsers reject SameSite=None unless the cookie is also Secure, so emitting it on a + // plain-HTTP deployment would drop the cookie entirely and break auth. + if cookieSecure.contains(true) then "SameSite=None; " + else + logger.warn("cookieSameSite=none requires cookieSecure=true; omitting SameSite (browsers reject None without Secure).") + "" + case Some(other) => + logger.warn(s"Unknown cookieSameSite value '$other' (expected lax, strict or none); omitting SameSite.") + "" diff --git a/server/src/main/scala/pulsar_auth/PulsarAuthRoutes.scala b/server/src/main/scala/pulsar_auth/PulsarAuthRoutes.scala index e451b5649..03c645c16 100644 --- a/server/src/main/scala/pulsar_auth/PulsarAuthRoutes.scala +++ b/server/src/main/scala/pulsar_auth/PulsarAuthRoutes.scala @@ -10,13 +10,30 @@ import _root_.pulsar_auth.{defaultPulsarAuth, jwtCredentialsDecoder, validCreden import io.circe.parser.decode as decodeJson object PulsarAuthRoutes: - val routes: EndpointGroup = () => { - addCredentials() - useCredentials() - deleteCredentials() - } + /** What every successful route writes into `Set-Cookie`. */ + private type SetCookie = (io.javalin.http.Context, PulsarAuth) => Unit - private def addCredentials(): Unit = + def routes: EndpointGroup = routesWith() + + /** The cookie-hardening inputs are parameters defaulting to the process config - same reason as + * `pulsarAuthToCookie`: the package-level `config` val is loaded once per process and cannot be + * varied, so nothing could otherwise observe over HTTP that these routes really emit the + * CONFIGURED Secure/SameSite attributes. `routes` keeps the production call site unchanged. */ + def routesWith( + publicBaseUrl: Option[String] = config.publicBaseUrl, + cookieSecure: Option[Boolean] = config.cookieSecure, + cookieSameSite: Option[String] = config.cookieSameSite + ): EndpointGroup = + val setCookie: SetCookie = + (ctx, pulsarAuth) => setCookieAndSuccess(ctx, pulsarAuth, publicBaseUrl, cookieSecure, cookieSameSite) + + () => { + addCredentials(setCookie) + useCredentials(setCookie) + deleteCredentials(setCookie) + } + + private def addCredentials(setCookie: SetCookie): Unit = post( s"/pulsar-auth/add/{credentialsName}", ctx => @@ -51,13 +68,13 @@ object PulsarAuthRoutes: current = Some(credentialsName), credentials = pulsarAuth.credentials + (credentialsName -> credentials) ) - setCookieAndSuccess(ctx, newPulsarAuth) + setCookie(ctx, newPulsarAuth) case _ => ctx.status(400) ctx.result("Credentials name contains illegal characters. Only alphanumerics, underscores(_) and dashes(-) are allowed.") ) - private def useCredentials(): Unit = + private def useCredentials(setCookie: SetCookie): Unit = post( s"/pulsar-auth/use/{credentialsName}", ctx => @@ -73,12 +90,18 @@ object PulsarAuthRoutes: if credentialsName.isBlank then ctx.status(400) ctx.result("Credentials name shouldn't be blank") + // Selecting a name that isn't in the map used to succeed, after which every + // client construction failed and the interceptor answered UNAUTHENTICATED + // for all calls - a self-inflicted brick from one request. + else if !pulsarAuth.credentials.contains(credentialsName) then + ctx.status(404) + ctx.result(s"No credentials named '$credentialsName'") else val newPulsarAuth = pulsarAuth.copy(current = Some(credentialsName)) - setCookieAndSuccess(ctx, newPulsarAuth) + setCookie(ctx, newPulsarAuth) ) - private def deleteCredentials(): Unit = + private def deleteCredentials(setCookie: SetCookie): Unit = post( "/pulsar-auth/delete/{credentialsName}", ctx => @@ -99,15 +122,34 @@ object PulsarAuthRoutes: case DefaultCredentialsName => ctx.status(400) ctx.result(s"Can't delete default credentials") + // Deleting a name that isn't in the map used to answer 200 and still + // rewrite `current` - a typo silently changed which credentials every + // later Pulsar call ran under. Refuse, and write no cookie at all. + case credentialsName: String if !pulsarAuth.credentials.contains(credentialsName) => + ctx.status(404) + ctx.result(s"No credentials named '$credentialsName'") case credentialsName: String => val newCredentials = pulsarAuth.credentials - credentialsName - val newPulsarAuth = - pulsarAuth.copy(credentials = newCredentials, current = newCredentials.keys.headOption.orElse(Some("Default"))) + // `current` used to be reassigned unconditionally to + // `newCredentials.keys.headOption`, so removing an UNRELATED + // credential repointed the session at whatever came first in map + // iteration order. The selection may only move when the deleted + // name is the selected one - and then only to Default, which + // setCookieAndSuccess guarantees still exists. + val newCurrent = + if pulsarAuth.current.contains(credentialsName) then Some(DefaultCredentialsName) + else pulsarAuth.current - setCookieAndSuccess(ctx, newPulsarAuth) + setCookie(ctx, pulsarAuth.copy(credentials = newCredentials, current = newCurrent)) ) - def setCookieAndSuccess(ctx: io.javalin.http.Context, pulsarAuth: PulsarAuth): Unit = + def setCookieAndSuccess( + ctx: io.javalin.http.Context, + pulsarAuth: PulsarAuth, + publicBaseUrl: Option[String] = config.publicBaseUrl, + cookieSecure: Option[Boolean] = config.cookieSecure, + cookieSameSite: Option[String] = config.cookieSameSite + ): Unit = def withNewDefaultAuth(pulsarAuth: PulsarAuth): PulsarAuth = // Dekaf admin can change default credentials, // so we need deliver new default credentials to users. @@ -117,7 +159,7 @@ object PulsarAuthRoutes: ) ) - val newCookieHeader = pulsar_auth.pulsarAuthToCookie(withNewDefaultAuth(pulsarAuth)) + val newCookieHeader = pulsar_auth.pulsarAuthToCookie(withNewDefaultAuth(pulsarAuth), publicBaseUrl, cookieSecure, cookieSameSite) ctx.header( "Set-Cookie", diff --git a/server/src/main/scala/pulsar_auth/PulsarAuthServiceImpl.scala b/server/src/main/scala/pulsar_auth/PulsarAuthServiceImpl.scala index 91a2473d3..984a97e44 100644 --- a/server/src/main/scala/pulsar_auth/PulsarAuthServiceImpl.scala +++ b/server/src/main/scala/pulsar_auth/PulsarAuthServiceImpl.scala @@ -21,7 +21,7 @@ class PulsarAuthServiceImpl extends pb.PulsarAuthServiceGrpc.PulsarAuthService: override def getMaskedCredentials(request: GetMaskedCredentialsRequest): Future[GetMaskedCredentialsResponse] = val pulsarAuth = RequestContext.pulsarAuth.get() - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful( GetMaskedCredentialsResponse( status = Some(status), @@ -39,7 +39,7 @@ class PulsarAuthServiceImpl extends pb.PulsarAuthServiceGrpc.PulsarAuthService: override def getCurrentCredentials(request: GetCurrentCredentialsRequest): Future[GetCurrentCredentialsResponse] = val pulsarAuth = RequestContext.pulsarAuth.get() - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful( GetCurrentCredentialsResponse( status = Some(status), diff --git a/server/src/main/scala/schema/SchemaServiceImpl.scala b/server/src/main/scala/schema/SchemaServiceImpl.scala index c761fb31e..9d58a09c0 100644 --- a/server/src/main/scala/schema/SchemaServiceImpl.scala +++ b/server/src/main/scala/schema/SchemaServiceImpl.scala @@ -47,17 +47,17 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: adminClient.schemas.createSchema(request.topic, schemaInfo) logger.info(s"Successfully created schema with name ${s.name} for topic ${request.topic}.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(CreateSchemaResponse(status = Some(status))) } catch { case err => logger.info(s"Failed to create schema with name ${s.name} for topic ${request.topic}. Reason: ${err.getMessage}.") - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(CreateSchemaResponse(status = Some(status))) } case _ => - val status = Status(code = Code.INVALID_ARGUMENT.index) + val status = Status(code = Code.INVALID_ARGUMENT.value) Future.successful(CreateSchemaResponse(status = Some(status))) override def deleteSchema(request: DeleteSchemaRequest): Future[DeleteSchemaResponse] = @@ -69,12 +69,12 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: adminClient.schemas.deleteSchema(request.topic, request.force) logger.info(s"Successfully deleted latest schema for topic ${request.topic}.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(DeleteSchemaResponse(status = Some(status))) } catch { case err => logger.info(s"Failed to delete latest schema for topic ${request.topic}. Reason: ${err.getMessage}.") - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(DeleteSchemaResponse(status = Some(status))) } @@ -84,7 +84,7 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: try { val schemaInfoWithVersion = adminClient.schemas.getSchemaInfoWithVersion(request.topic) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) logger.info(s"Successfully got latest schema info for topic ${request.topic}.") Future.successful( @@ -97,11 +97,11 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: } catch { case (_: PulsarAdminException.NotFoundException) => logger.info(s"No schema where found for topic ${request.topic}.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(GetLatestSchemaInfoResponse(status = Some(status), schemaInfo = None, schemaVersion = None)) case (err: PulsarAdminException) => logger.info(s"Failed to get latest schema info for topic ${request.topic}. Reason: ${err.getMessage}.") - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetLatestSchemaInfoResponse(status = Some(status))) } @@ -124,12 +124,12 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: .map(v => SchemaInfoWithVersion(schemaInfo = Some(schemaInfoToPb(v._1)), schemaVersion = v._2)) logger.info(s"Successfully listed schemas for topic ${request.topic}.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(ListSchemasResponse(status = Some(status), schemas = schemas)) } catch { case err => logger.info(s"Failed to list schemas for topic ${request.topic}. Reason: ${err.getMessage}.") - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(ListSchemasResponse(status = Some(status))) } @@ -163,7 +163,7 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: ) logger.info(s"Compiled ${files.size} protobuf native files.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(CompileProtobufNativeResponse(status = Some(status), files)) override def testCompatibility(request: TestCompatibilityRequest): Future[TestCompatibilityResponse] = @@ -174,14 +174,14 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: case Some(spb) => schemaInfoFromPb(spb) case None => logger.info(s"Successfully tested schema compatibility for topic ${request.topic}.") - val status = Status(code = Code.INVALID_ARGUMENT.index) + val status = Status(code = Code.INVALID_ARGUMENT.value) return Future.successful(TestCompatibilityResponse(status = Some(status))) val compatibilityTestResult = protobufnative.schemaCompatibility.test(pulsarAdmin = adminClient, topic = request.topic, schemaInfo = schemaInfo) logger.info(s"Successfully tested schema compatibility for topic ${request.topic}.") - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( TestCompatibilityResponse( status = Some(status), @@ -196,7 +196,7 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: request.schemaType match case SchemaTypePb.SCHEMA_TYPE_PROTOBUF_NATIVE => val descriptor = ProtobufNativeSchemaUtils.deserialize(request.rawSchema.toByteArray) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetHumanReadableSchemaResponse( status = Some(status), @@ -204,7 +204,7 @@ class SchemaServiceImpl extends SchemaServiceGrpc.SchemaService: ) ) case _ => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( GetHumanReadableSchemaResponse( status = Some(status), diff --git a/server/src/main/scala/tenant/TenantServiceImpl.scala b/server/src/main/scala/tenant/TenantServiceImpl.scala index 36932db76..627401365 100644 --- a/server/src/main/scala/tenant/TenantServiceImpl.scala +++ b/server/src/main/scala/tenant/TenantServiceImpl.scala @@ -30,11 +30,11 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: try { adminClient.tenants.createTenant(request.tenantName, config.build) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.CreateTenantResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.CreateTenantResponse(status = Some(status))) } @@ -52,11 +52,11 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: try { adminClient.tenants.updateTenant(request.tenantName, config.build) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.UpdateTenantResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.UpdateTenantResponse(status = Some(status))) } @@ -66,11 +66,11 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: try { adminClient.tenants.deleteTenant(request.tenantName, request.force) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.DeleteTenantResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.DeleteTenantResponse(status = Some(status))) } @@ -96,7 +96,7 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: request.tenants.zip(tenantsInfo).toMap } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.GetTenantsResponse(status = Some(status))) } @@ -110,12 +110,12 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: Map.empty } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.GetTenantsResponse(status = Some(status))) } - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetTenantsResponse( status = Some(status), tenants, @@ -130,9 +130,9 @@ class TenantServiceImpl extends pb.TenantServiceGrpc.TenantService: adminClient.tenants.getTenants.asScala } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.ListTenantsResponse(status = Some(status))) } - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ListTenantsResponse(status = Some(status), tenants = tenants.toSeq)) diff --git a/server/src/main/scala/topic/TopicServiceImpl.scala b/server/src/main/scala/topic/TopicServiceImpl.scala index 09de3f3ed..e3ee2019b 100644 --- a/server/src/main/scala/topic/TopicServiceImpl.scala +++ b/server/src/main/scala/topic/TopicServiceImpl.scala @@ -31,11 +31,11 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: try { adminClient.topics.createPartitionedTopic(request.topic, request.numPartitions, request.properties.asJava) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.CreatePartitionedTopicResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.CreatePartitionedTopicResponse(status = Some(status))) } @@ -45,11 +45,11 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: try { adminClient.topics.createNonPartitionedTopic(request.topic, request.properties.asJava) - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.CreateNonPartitionedTopicResponse(status = Some(status))) } catch { case err => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.CreateNonPartitionedTopicResponse(status = Some(status))) } @@ -71,11 +71,11 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: persistent ++ nonPersistent catch { case err: Throwable => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.ListTopicsResponse(status = Some(status))) } - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ListTopicsResponse(status = Some(status), topics = topics)) override def listPartitionedTopics(request: pb.ListPartitionedTopicsRequest): Future[pb.ListPartitionedTopicsResponse] = @@ -88,11 +88,11 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: adminClient.topics.getPartitionedTopicList(request.namespace, options) catch { case err: Throwable => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) return Future.successful(pb.ListPartitionedTopicsResponse(status = Some(status))) } - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ListPartitionedTopicsResponse(status = Some(status), topics = topics.asScala.toSeq)) override def getTopicsInternalStats(request: pb.GetTopicsInternalStatsRequest): Future[pb.GetTopicsInternalStatsResponse] = @@ -110,7 +110,7 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: case _ => None }.toMap - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetTopicsInternalStatsResponse(status = Some(status), stats = stats)) override def deleteTopic(request: pb.DeleteTopicRequest): Future[pb.DeleteTopicResponse] = @@ -126,12 +126,12 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: def lookupNonPartitionedTopic(): Try[Unit] = Try(adminClient.lookups().lookupTopic(request.topicName)) def handleSuccess(): Future[pb.DeleteTopicResponse] = { - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(pb.DeleteTopicResponse(status = Some(status))) } def handleFailure(err: Throwable): Future[pb.DeleteTopicResponse] = { - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.DeleteTopicResponse(status = Some(status))) } @@ -154,10 +154,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.unload(request.topicName)) match case Success(_) => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(pb.UnloadTopicResponse(status = Some(status))) case Failure(err) => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.UnloadTopicResponse(status = Some(status))) override def getTopicsStats(request: pb.GetTopicsStatsRequest): Future[pb.GetTopicsStatsResponse] = @@ -216,7 +216,7 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: // This RPC method always returns Code.OK because in case we request stats for a single topic, // we want to avoid additional API calls to detect is topic partitioned or not. - val status: Status = Status(code = Code.OK.index, message = errors.map(_.getMessage).mkString(". ")) + val status: Status = Status(code = Code.OK.value, message = errors.map(_.getMessage).mkString(". ")) Future.successful(pb.GetTopicsStatsResponse( status = Some(status), @@ -244,10 +244,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: match case Failure(err) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetTopicsPropertiesResponse(status = Some(status))) case Success(properties) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetTopicsPropertiesResponse( status = Some(status), topicsProperties = properties @@ -269,10 +269,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: adminClient.topics.updateProperties(request.topic, request.topicProperties.asJava) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetTopicPropertiesResponse(status = Some(status))) case Success(value) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.SetTopicPropertiesResponse( status = Some(status) )) @@ -282,11 +282,11 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(_root_.topic.getTopicPartitioning(adminClient, request.topicFqn)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetIsPartitionedTopicResponse(status = Some(status))) case Success(partitioning: TopicPartitioning) => val isPartitioned = partitioning.`type` == TopicPartitioningType.Partitioned - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetIsPartitionedTopicResponse( status = Some(status), @@ -300,10 +300,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.updatePartitionedTopic(request.topicFqn, request.numPartitions, request.updateLocalTopicOnly, request.force)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.UpdatePartitionedTopicResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.UpdatePartitionedTopicResponse(status = Some(status))) override def createMissedPartitions(request: CreateMissedPartitionsRequest): Future[CreateMissedPartitionsResponse] = @@ -311,10 +311,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.createMissedPartitions(request.topicFqn)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.CreateMissedPartitionsResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.CreateMissedPartitionsResponse(status = Some(status))) override def getCompactionStatus(request: GetCompactionStatusRequest): Future[GetCompactionStatusResponse] = @@ -322,10 +322,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.compactionStatus(request.topicFqn)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetCompactionStatusResponse(status = Some(status))) case Success(lrps) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.GetCompactionStatusResponse( status = Some(status), processStatus = Some(LongRunningProcessStatus.toPb(lrps)) @@ -336,10 +336,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.triggerCompaction(request.topicFqn)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.TriggerCompactionResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.TriggerCompactionResponse(status = Some(status))) override def deleteSubscription(request: DeleteSubscriptionRequest): Future[DeleteSubscriptionResponse] = @@ -347,10 +347,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics.deleteSubscription(request.topicFqn, request.subscriptionName, request.isForce)) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.DeleteSubscriptionResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.DeleteSubscriptionResponse(status = Some(status))) override def createSubscription(request: CreateSubscriptionRequest): Future[CreateSubscriptionResponse] = @@ -378,10 +378,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: ) match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.CreateSubscriptionResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.CreateSubscriptionResponse(status = Some(status))) override def expireMessages(request: ExpireMessagesRequest): Future[ExpireMessagesResponse] = @@ -424,10 +424,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: throw RuntimeException("Empty expire messages target (should be either expire of all subscriptions or on a specific one)") match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.ExpireMessagesResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.ExpireMessagesResponse(status = Some(status))) override def skipSubscriptionMessages(request: SkipSubscriptionMessagesRequest): Future[SkipSubscriptionMessagesResponse] = @@ -445,10 +445,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: throw RuntimeException("Empty skip messages target (should be either skip of all messages or exact number of messages)") match case Failure(err: Throwable) => - val status: Status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status: Status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SkipSubscriptionMessagesResponse(status = Some(status))) case Success(_) => - val status: Status = Status(code = Code.OK.index) + val status: Status = Status(code = Code.OK.value) Future.successful(pb.SkipSubscriptionMessagesResponse(status = Some(status))) override def resetCursor(request: ResetCursorRequest): Future[ResetCursorResponse] = @@ -468,10 +468,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: adminClient.topics().resetCursor(request.topicFqn, request.subscriptionName, timestamp) match case Failure(err: Throwable) => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(ResetCursorResponse(status = Some(status))) case Success(_) => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(ResetCursorResponse(status = Some(status))) override def getSubscriptionStats(request: GetSubscriptionStatsRequest): Future[GetSubscriptionStatsResponse] = @@ -501,10 +501,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: .getOrElse(throw new Exception(s"Subscription \"${request.subscriptionName}\" not found on topic \"${request.topicFqn}\"")) match case Failure(err: Throwable) => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionStatsResponse(status = Some(status))) case Success(subscriptionStats) => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(GetSubscriptionStatsResponse( status = Some(status), subscriptionStats = Some(subscriptionStatsToPb(subscriptionStats)) @@ -515,10 +515,10 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics().getSubscriptionProperties(request.topicFqn, request.subscriptionName)) match case Failure(err: Throwable) => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(GetSubscriptionPropertiesResponse(status = Some(status))) case Success(properties) => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(GetSubscriptionPropertiesResponse( status = Some(status), properties = Option(properties).map(_.asScala.toMap).getOrElse(Map.empty) @@ -530,8 +530,8 @@ class TopicServiceImpl extends pb.TopicServiceGrpc.TopicService: Try(adminClient.topics().updateSubscriptionProperties(request.topicFqn, request.subscriptionName, request.properties.asJava)) match case Failure(err: Throwable) => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(SetSubscriptionPropertiesResponse(status = Some(status))) case Success(_) => - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(SetSubscriptionPropertiesResponse(status = Some(status))) diff --git a/server/src/main/scala/topic_policies/TopicPoliciesServiceImpl.scala b/server/src/main/scala/topic_policies/TopicPoliciesServiceImpl.scala index fb93bd61e..c424766ca 100644 --- a/server/src/main/scala/topic_policies/TopicPoliciesServiceImpl.scala +++ b/server/src/main/scala/topic_policies/TopicPoliciesServiceImpl.scala @@ -51,14 +51,14 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetBacklogQuotasResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), destinationStorage = destinationStorageBacklogQuotaPb, messageAge = messageAgeBacklogQuotaPb, ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetBacklogQuotasResponse(status = Some(status))) } override def setBacklogQuotas(request: pb.SetBacklogQuotasRequest): Future[pb.SetBacklogQuotasResponse] = @@ -104,10 +104,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies adminClient.topicPolicies(request.isGlobal).setBacklogQuota(request.topic, backlogQuota, BacklogQuotaType.message_age) case None => - Future.successful(pb.SetBacklogQuotasResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetBacklogQuotasResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetBacklogQuotasResponse(status = Some(status))) } override def removeBacklogQuota(request: pb.RemoveBacklogQuotaRequest): Future[pb.RemoveBacklogQuotaResponse] = @@ -122,13 +122,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing backlog quota (message age) on topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeBacklogQuota(request.topic, BacklogQuotaType.message_age) case _ => - val status = Status(code = Code.INVALID_ARGUMENT.index, message = "Backlog quota type should be specified") + val status = Status(code = Code.INVALID_ARGUMENT.value, message = "Backlog quota type should be specified") return Future.successful(pb.RemoveBacklogQuotaResponse(status = Some(status))) - Future.successful(pb.RemoveBacklogQuotaResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveBacklogQuotaResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveBacklogQuotaResponse(status = Some(status))) } override def getDelayedDelivery(request: pb.GetDelayedDeliveryRequest): Future[pb.GetDelayedDeliveryResponse] = @@ -147,12 +147,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetDelayedDeliveryResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), delayedDelivery = delayedDeliveryPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetDelayedDeliveryResponse(status = Some(status))) } override def setDelayedDelivery(request: pb.SetDelayedDeliveryRequest): Future[pb.SetDelayedDeliveryResponse] = @@ -166,10 +166,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies .build() adminClient.topicPolicies(request.isGlobal).setDelayedDeliveryPolicy(request.topic, delayedDeliveryPolicies) - Future.successful(pb.SetDelayedDeliveryResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetDelayedDeliveryResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetDelayedDeliveryResponse(status = Some(status))) } override def removeDelayedDelivery(request: pb.RemoveDelayedDeliveryRequest): Future[pb.RemoveDelayedDeliveryResponse] = @@ -179,10 +179,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing delayed delivery policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeDelayedDeliveryPolicy(request.topic) - Future.successful(pb.RemoveDelayedDeliveryResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveDelayedDeliveryResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveDelayedDeliveryResponse(status = Some(status))) } override def getMessageTtl(request: pb.GetMessageTtlRequest): Future[pb.GetMessageTtlResponse] = @@ -200,12 +200,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetMessageTtlResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), messageTtl = messageTtlPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMessageTtlResponse(status = Some(status))) } override def setMessageTtl(request: pb.SetMessageTtlRequest): Future[pb.SetMessageTtlResponse] = @@ -215,10 +215,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting message TTL policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMessageTTL(request.topic, request.messageTtlSeconds) - Future.successful(pb.SetMessageTtlResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMessageTtlResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMessageTtlResponse(status = Some(status))) } override def removeMessageTtl(request: pb.RemoveMessageTtlRequest): Future[pb.RemoveMessageTtlResponse] = @@ -228,10 +228,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing message TTL policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMessageTTL(request.topic) - Future.successful(pb.RemoveMessageTtlResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMessageTtlResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMessageTtlResponse(status = Some(status))) } override def getRetention(request: pb.GetRetentionRequest): Future[pb.GetRetentionResponse] = @@ -250,12 +250,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetRetentionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), retention = retentionPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetRetentionResponse(status = Some(status))) } override def setRetention(request: pb.SetRetentionRequest): Future[pb.SetRetentionResponse] = @@ -266,10 +266,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies val retention = new RetentionPolicies(request.retentionTimeInMinutes, request.retentionSizeInMb) adminClient.topicPolicies(request.isGlobal).setRetention(request.topic, retention) - Future.successful(pb.SetRetentionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetRetentionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetRetentionResponse(status = Some(status))) } override def removeRetention(request: pb.RemoveRetentionRequest): Future[pb.RemoveRetentionResponse] = @@ -279,10 +279,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing retention for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeRetention(request.topic) - Future.successful(pb.RemoveRetentionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveRetentionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveRetentionResponse(status = Some(status))) } override def getMaxUnackedMessagesOnConsumer(request: pb.GetMaxUnackedMessagesOnConsumerRequest): Future[pb.GetMaxUnackedMessagesOnConsumerResponse] = @@ -300,12 +300,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetMaxUnackedMessagesOnConsumerResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxUnackedMessagesOnConsumer = maxUnackedMessagesOnConsumerPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxUnackedMessagesOnConsumerResponse(status = Some(status))) } override def setMaxUnackedMessagesOnConsumer(request: pb.SetMaxUnackedMessagesOnConsumerRequest): Future[pb.SetMaxUnackedMessagesOnConsumerResponse] = @@ -315,10 +315,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max unacked messages on consumer policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxUnackedMessagesOnConsumer(request.topic, request.maxUnackedMessagesOnConsumer) - Future.successful(pb.SetMaxUnackedMessagesOnConsumerResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxUnackedMessagesOnConsumerResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxUnackedMessagesOnConsumerResponse(status = Some(status))) } @@ -329,10 +329,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max unacked messages on consumer policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxUnackedMessagesOnConsumer(request.topic) - Future.successful(pb.RemoveMaxUnackedMessagesOnConsumerResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxUnackedMessagesOnConsumerResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxUnackedMessagesOnConsumerResponse(status = Some(status))) } override def getMaxUnackedMessagesOnSubscription(request: pb.GetMaxUnackedMessagesOnSubscriptionRequest): Future[pb.GetMaxUnackedMessagesOnSubscriptionResponse] = @@ -350,12 +350,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetMaxUnackedMessagesOnSubscriptionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxUnackedMessagesOnSubscription = maxUnackedMessagesOnSubscriptionPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxUnackedMessagesOnSubscriptionResponse(status = Some(status))) } override def setMaxUnackedMessagesOnSubscription(request: pb.SetMaxUnackedMessagesOnSubscriptionRequest): Future[pb.SetMaxUnackedMessagesOnSubscriptionResponse] = @@ -365,10 +365,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max unacked messages on subscription policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxUnackedMessagesOnSubscription(request.topic, request.maxUnackedMessagesOnSubscription) - Future.successful(pb.SetMaxUnackedMessagesOnSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxUnackedMessagesOnSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxUnackedMessagesOnSubscriptionResponse(status = Some(status))) } override def removeMaxUnackedMessagesOnSubscription(request: pb.RemoveMaxUnackedMessagesOnSubscriptionRequest): Future[pb.RemoveMaxUnackedMessagesOnSubscriptionResponse] = @@ -378,10 +378,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max unacked messages on subscription policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxUnackedMessagesOnSubscription(request.topic) - Future.successful(pb.RemoveMaxUnackedMessagesOnSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxUnackedMessagesOnSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxUnackedMessagesOnSubscriptionResponse(status = Some(status))) } override def getInactiveTopicPolicies(request: pb.GetInactiveTopicPoliciesRequest): Future[pb.GetInactiveTopicPoliciesResponse] = @@ -406,12 +406,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetInactiveTopicPoliciesResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), inactiveTopicPolicies = inactiveTopicPoliciesPb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetInactiveTopicPoliciesResponse(status = Some(status))) } override def setInactiveTopicPolicies(request: pb.SetInactiveTopicPoliciesRequest): Future[pb.SetInactiveTopicPoliciesResponse] = @@ -433,10 +433,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies throw new IllegalArgumentException("InactiveTopicPoliciesDeleteMode should be specified.") adminClient.topicPolicies(request.isGlobal).setInactiveTopicPolicies(request.topic, inactiveTopicPolicies) - Future.successful(pb.SetInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetInactiveTopicPoliciesResponse(status = Some(status))) } override def removeInactiveTopicPolicies(request: pb.RemoveInactiveTopicPoliciesRequest): Future[pb.RemoveInactiveTopicPoliciesResponse] = @@ -446,10 +446,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing inactive topic policies for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeInactiveTopicPolicies(request.topic) - Future.successful(pb.RemoveInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveInactiveTopicPoliciesResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveInactiveTopicPoliciesResponse(status = Some(status))) } override def getPersistence(request: pb.GetPersistenceRequest): Future[pb.GetPersistenceResponse] = @@ -470,12 +470,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetPersistenceResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), persistence = persistencePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetPersistenceResponse(status = Some(status))) } override def setPersistence(request: pb.SetPersistenceRequest): Future[pb.SetPersistenceResponse] = @@ -486,10 +486,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies val persistencePolicies = PersistencePolicies(request.bookkeeperEnsemble, request.bookkeeperWriteQuorum, request.bookkeeperAckQuorum, request.managedLedgerMaxMarkDeleteRate) adminClient.topicPolicies(request.isGlobal).setPersistence(request.topic, persistencePolicies) - Future.successful(pb.SetPersistenceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetPersistenceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetPersistenceResponse(status = Some(status))) } override def removePersistence(request: pb.RemovePersistenceRequest): Future[pb.RemovePersistenceResponse] = @@ -499,10 +499,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing persistence policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removePersistence(request.topic) - Future.successful(pb.RemovePersistenceResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemovePersistenceResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemovePersistenceResponse(status = Some(status))) } override def getDeduplication(request: pb.GetDeduplicationRequest): Future[pb.GetDeduplicationResponse] = @@ -516,12 +516,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies pb.GetDeduplicationResponse.Deduplication.Specified(new pb.DeduplicationSpecified(enabled = v)) Future.successful(pb.GetDeduplicationResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), deduplication )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetDeduplicationResponse(status = Some(status))) } override def setDeduplication(request: pb.SetDeduplicationRequest): Future[pb.SetDeduplicationResponse] = @@ -530,10 +530,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies try { logger.info(s"Setting deduplication policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setDeduplicationStatus(request.topic, request.enabled) - Future.successful(pb.SetDeduplicationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetDeduplicationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetDeduplicationResponse(status = Some(status))) } override def removeDeduplication(request: pb.RemoveDeduplicationRequest): Future[pb.RemoveDeduplicationResponse] = @@ -543,10 +543,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing deduplication policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeDeduplicationStatus(request.topic) - Future.successful(pb.RemoveDeduplicationResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveDeduplicationResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveDeduplicationResponse(status = Some(status))) } override def getDeduplicationSnapshotInterval(request: pb.GetDeduplicationSnapshotIntervalRequest): Future[pb.GetDeduplicationSnapshotIntervalResponse] = @@ -560,12 +560,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies pb.GetDeduplicationSnapshotIntervalResponse.Interval.Enabled(new pb.DeduplicationSnapshotIntervalEnabled(interval = v)) Future.successful(pb.GetDeduplicationSnapshotIntervalResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), interval )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetDeduplicationSnapshotIntervalResponse(status = Some(status))) } override def setDeduplicationSnapshotInterval(request: pb.SetDeduplicationSnapshotIntervalRequest): Future[pb.SetDeduplicationSnapshotIntervalResponse] = @@ -575,10 +575,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting deduplication snapshot interval policy for topic ${request.topic}. ${request.interval}") adminClient.topicPolicies(request.isGlobal).setDeduplicationSnapshotInterval(request.topic, request.interval) - Future.successful(pb.SetDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetDeduplicationSnapshotIntervalResponse(status = Some(status))) } override def removeDeduplicationSnapshotInterval(request: pb.RemoveDeduplicationSnapshotIntervalRequest): Future[pb.RemoveDeduplicationSnapshotIntervalResponse] = @@ -588,10 +588,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing deduplication snapshot interval policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeDeduplicationSnapshotInterval(request.topic) - Future.successful(pb.RemoveDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveDeduplicationSnapshotIntervalResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveDeduplicationSnapshotIntervalResponse(status = Some(status))) } override def getDispatchRate(request: pb.GetDispatchRateRequest): Future[pb.GetDispatchRateResponse] = @@ -612,12 +612,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), dispatchRate = dispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetDispatchRateResponse(status = Some(status))) } override def setDispatchRate(request: pb.SetDispatchRateRequest): Future[pb.SetDispatchRateResponse] = @@ -633,10 +633,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies .build adminClient.topicPolicies(request.isGlobal).setDispatchRate(request.topic, dispatchRate) - Future.successful(pb.SetDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetDispatchRateResponse(status = Some(status))) } override def removeDispatchRate(request: pb.RemoveDispatchRateRequest): Future[pb.RemoveDispatchRateResponse] = @@ -646,10 +646,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing dispatch rate policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeDispatchRate(request.topic) - Future.successful(pb.RemoveDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveDispatchRateResponse(status = Some(status))) } override def getReplicatorDispatchRate(request: pb.GetReplicatorDispatchRateRequest): Future[pb.GetReplicatorDispatchRateResponse] = @@ -670,12 +670,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetReplicatorDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), replicatorDispatchRate = replicatorDispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetReplicatorDispatchRateResponse(status = Some(status))) } override def setReplicatorDispatchRate(request: pb.SetReplicatorDispatchRateRequest): Future[pb.SetReplicatorDispatchRateResponse] = @@ -691,10 +691,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies .build adminClient.topicPolicies(request.isGlobal).setReplicatorDispatchRate(request.topic, dispatchRate) - Future.successful(pb.SetReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetReplicatorDispatchRateResponse(status = Some(status))) } override def removeReplicatorDispatchRate(request: pb.RemoveReplicatorDispatchRateRequest): Future[pb.RemoveReplicatorDispatchRateResponse] = @@ -704,10 +704,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing replicator dispatch rate for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeReplicatorDispatchRate(request.topic) - Future.successful(pb.RemoveReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveReplicatorDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveReplicatorDispatchRateResponse(status = Some(status))) } override def getSubscriptionDispatchRate(request: pb.GetSubscriptionDispatchRateRequest): Future[pb.GetSubscriptionDispatchRateResponse] = @@ -728,12 +728,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetSubscriptionDispatchRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionDispatchRate = subscriptionDispatchRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetSubscriptionDispatchRateResponse(status = Some(status))) } override def setSubscriptionDispatchRate(request: pb.SetSubscriptionDispatchRateRequest): Future[pb.SetSubscriptionDispatchRateResponse] = @@ -749,10 +749,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies .build adminClient.topicPolicies(request.isGlobal).setSubscriptionDispatchRate(request.topic, dispatchRate) - Future.successful(pb.SetSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetSubscriptionDispatchRateResponse(status = Some(status))) } override def removeSubscriptionDispatchRate(request: pb.RemoveSubscriptionDispatchRateRequest): Future[pb.RemoveSubscriptionDispatchRateResponse] = @@ -762,10 +762,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing subscription dispatch rate for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeSubscriptionDispatchRate(request.topic) - Future.successful(pb.RemoveSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveSubscriptionDispatchRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveSubscriptionDispatchRateResponse(status = Some(status))) } override def getCompactionThreshold(request: pb.GetCompactionThresholdRequest): Future[pb.GetCompactionThresholdResponse] = @@ -776,12 +776,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies case None => pb.GetCompactionThresholdResponse.Threshold.Disabled(new pb.CompactionThresholdDisabled()) case Some(v) => pb.GetCompactionThresholdResponse.Threshold.Enabled(new pb.CompactionThresholdEnabled(threshold = v)) Future.successful(pb.GetCompactionThresholdResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), threshold )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetCompactionThresholdResponse(status = Some(status))) } override def setCompactionThreshold(request: pb.SetCompactionThresholdRequest): Future[pb.SetCompactionThresholdResponse] = @@ -791,10 +791,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting compaction threshold policy for topic ${request.topic}. ${request.threshold}") adminClient.topicPolicies(request.isGlobal).setCompactionThreshold(request.topic, request.threshold) - Future.successful(pb.SetCompactionThresholdResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetCompactionThresholdResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetCompactionThresholdResponse(status = Some(status))) } override def removeCompactionThreshold(request: pb.RemoveCompactionThresholdRequest): Future[pb.RemoveCompactionThresholdResponse] = @@ -804,10 +804,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing compaction threshold policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeCompactionThreshold(request.topic) - Future.successful(pb.RemoveCompactionThresholdResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveCompactionThresholdResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveCompactionThresholdResponse(status = Some(status))) } override def getPublishRate(request: pb.GetPublishRateRequest): Future[pb.GetPublishRateResponse] = @@ -826,12 +826,12 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful(pb.GetPublishRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), publishRate = publishRatePb )) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetPublishRateResponse(status = Some(status))) } override def setPublishRate(request: pb.SetPublishRateRequest): Future[pb.SetPublishRateResponse] = @@ -842,10 +842,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies val publishRate = PublishRate( request.rateInMsg, request.rateInByte ) adminClient.topicPolicies(request.isGlobal).setPublishRate(request.topic, publishRate) - Future.successful(pb.SetPublishRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetPublishRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetPublishRateResponse(status = Some(status))) } override def removePublishRate(request: pb.RemovePublishRateRequest): Future[pb.RemovePublishRateResponse] = @@ -855,10 +855,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing publish rate policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removePublishRate(request.topic) - Future.successful(pb.RemovePublishRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemovePublishRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemovePublishRateResponse(status = Some(status))) } override def getMaxConsumersPerSubscription(request: pb.GetMaxConsumersPerSubscriptionRequest): Future[pb.GetMaxConsumersPerSubscriptionResponse] = @@ -877,13 +877,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetMaxConsumersPerSubscriptionResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxConsumersPerSubscription = maxConsumersPerSubscriptionPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxConsumersPerSubscriptionResponse(status = Some(status))) } override def setMaxConsumersPerSubscription(request: pb.SetMaxConsumersPerSubscriptionRequest): Future[pb.SetMaxConsumersPerSubscriptionResponse] = @@ -893,10 +893,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max consumers per subscription policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxConsumersPerSubscription(request.topic, request.maxConsumersPerSubscription) - Future.successful(pb.SetMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxConsumersPerSubscriptionResponse(status = Some(status))) } override def removeMaxConsumersPerSubscription(request: pb.RemoveMaxConsumersPerSubscriptionRequest): Future[pb.RemoveMaxConsumersPerSubscriptionResponse] = @@ -906,10 +906,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max consumers per subscription policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxConsumersPerSubscription(request.topic) - Future.successful(pb.RemoveMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxConsumersPerSubscriptionResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxConsumersPerSubscriptionResponse(status = Some(status))) } override def getMaxProducers(request: pb.GetMaxProducersRequest): Future[pb.GetMaxProducersResponse] = @@ -926,13 +926,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetMaxProducersResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxProducers = maxProducersPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxProducersResponse(status = Some(status))) } override def setMaxProducers(request: pb.SetMaxProducersRequest): Future[pb.SetMaxProducersResponse] = @@ -942,10 +942,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max producers per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxProducers(request.topic, request.maxProducers) - Future.successful(pb.SetMaxProducersResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxProducersResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxProducersResponse(status = Some(status))) } override def removeMaxProducers(request: pb.RemoveMaxProducersRequest): Future[pb.RemoveMaxProducersResponse] = @@ -955,10 +955,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max producers per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxProducers(request.topic) - Future.successful(pb.RemoveMaxProducersResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxProducersResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxProducersResponse(status = Some(status))) } override def getMaxSubscriptionsPerTopic(request: pb.GetMaxSubscriptionsPerTopicRequest): Future[pb.GetMaxSubscriptionsPerTopicResponse] = @@ -977,13 +977,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetMaxSubscriptionsPerTopicResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxSubscriptionsPerTopic = maxSubscriptionsPerTopicPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxSubscriptionsPerTopicResponse(status = Some(status))) } override def setMaxSubscriptionsPerTopic(request: pb.SetMaxSubscriptionsPerTopicRequest): Future[pb.SetMaxSubscriptionsPerTopicResponse] = @@ -993,10 +993,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max subscriptions per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxSubscriptionsPerTopic(request.topic, request.maxSubscriptionsPerTopic) - Future.successful(pb.SetMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxSubscriptionsPerTopicResponse(status = Some(status))) } override def removeMaxSubscriptionsPerTopic(request: pb.RemoveMaxSubscriptionsPerTopicRequest): Future[pb.RemoveMaxSubscriptionsPerTopicResponse] = @@ -1006,10 +1006,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max subscriptions per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxSubscriptionsPerTopic(request.topic) - Future.successful(pb.RemoveMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxSubscriptionsPerTopicResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxSubscriptionsPerTopicResponse(status = Some(status))) } override def getMaxConsumers(request: pb.GetMaxConsumersRequest): Future[pb.GetMaxConsumersResponse] = @@ -1028,13 +1028,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetMaxConsumersResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxConsumers = maxConsumersPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxConsumersResponse(status = Some(status))) } override def setMaxConsumers(request: pb.SetMaxConsumersRequest): Future[pb.SetMaxConsumersResponse] = @@ -1044,10 +1044,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max consumers per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).setMaxConsumers(request.topic, request.maxConsumers) - Future.successful(pb.SetMaxConsumersResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxConsumersResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxConsumersResponse(status = Some(status))) } override def removeMaxConsumers(request: pb.RemoveMaxConsumersRequest): Future[pb.RemoveMaxConsumersResponse] = @@ -1057,10 +1057,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max consumers per topic policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxConsumers(request.topic) - Future.successful(pb.RemoveMaxConsumersResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxConsumersResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxConsumersResponse(status = Some(status))) } override def getSubscriptionTypesEnabled(request: pb.GetSubscriptionTypesEnabledRequest): Future[pb.GetSubscriptionTypesEnabledResponse] = @@ -1086,13 +1086,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) Future.successful( pb.GetSubscriptionTypesEnabledResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscriptionTypesEnabled = subscriptionTypesEnabledPb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetSubscriptionTypesEnabledResponse(status = Some(status))) } override def setSubscriptionTypesEnabled(request: pb.SetSubscriptionTypesEnabledRequest): Future[pb.SetSubscriptionTypesEnabledResponse] = @@ -1112,10 +1112,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies val subscriptionTypesEnabled = request.types.map(pbToSubscriptionType).toSet.asJava adminClient.topicPolicies(request.isGlobal).setSubscriptionTypesEnabled(request.topic, subscriptionTypesEnabled) - Future.successful(pb.SetSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetSubscriptionTypesEnabledResponse(status = Some(status))) } override def removeSubscriptionTypesEnabled(request: pb.RemoveSubscriptionTypesEnabledRequest): Future[pb.RemoveSubscriptionTypesEnabledResponse] = @@ -1125,10 +1125,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing subscription types enabled policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeSubscriptionTypesEnabled(request.topic) - Future.successful(pb.RemoveSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveSubscriptionTypesEnabledResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveSubscriptionTypesEnabledResponse(status = Some(status))) } override def getSubscribeRate(request: pb.GetSubscribeRateRequest): Future[pb.GetSubscribeRateResponse] = @@ -1148,13 +1148,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetSubscribeRateResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), subscribeRate = subscribeRatePb ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetSubscribeRateResponse(status = Some(status))) } override def setSubscribeRate(request: pb.SetSubscribeRateRequest): Future[pb.SetSubscribeRateResponse] = @@ -1165,10 +1165,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies val subscribeRate = new SubscribeRate(request.subscribeThrottlingRatePerConsumer, request.ratePeriodInSeconds) adminClient.topicPolicies(request.isGlobal).setSubscribeRate(request.topic, subscribeRate) - Future.successful(pb.SetSubscribeRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetSubscribeRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetSubscribeRateResponse(status = Some(status))) } override def removeSubscribeRate(request: pb.RemoveSubscribeRateRequest): Future[pb.RemoveSubscribeRateResponse] = @@ -1177,10 +1177,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies try { logger.info(s"Removing subscribe rate policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeSubscribeRate(request.topic) - Future.successful(pb.RemoveSubscribeRateResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveSubscribeRateResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveSubscribeRateResponse(status = Some(status))) } override def getSchemaCompatibilityStrategy(request: pb.GetSchemaCompatibilityStrategyRequest): Future[pb.GetSchemaCompatibilityStrategyResponse] = @@ -1194,7 +1194,7 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies pb.GetSchemaCompatibilityStrategyResponse.Strategy.Specified(new pb.SchemaCompatibilityStrategySpecified( strategy = schemaCompatibilityStrategyToPb(v) )) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful( pb.GetSchemaCompatibilityStrategyResponse( status = Some(status), @@ -1203,7 +1203,7 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetSchemaCompatibilityStrategyResponse(status = Some(status))) } override def setSchemaCompatibilityStrategy(request: pb.SetSchemaCompatibilityStrategyRequest): Future[pb.SetSchemaCompatibilityStrategyResponse] = @@ -1212,11 +1212,11 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies try { adminClient.topicPolicies(request.isGlobal).setSchemaCompatibilityStrategy(request.topic, schemaCompatibilityStrategyFromPb(request.strategy)) - val status = Status(code = Code.OK.index) + val status = Status(code = Code.OK.value) Future.successful(pb.SetSchemaCompatibilityStrategyResponse(status = Some(status))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetSchemaCompatibilityStrategyResponse(status = Some(status))) } override def removeSchemaCompatibilityStrategy(request: pb.RemoveSchemaCompatibilityStrategyRequest): Future[pb.RemoveSchemaCompatibilityStrategyResponse] = @@ -1225,10 +1225,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies try { logger.info(s"Removing schema compatibility strategy policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeSchemaCompatibilityStrategy(request.topic) - Future.successful(pb.RemoveSchemaCompatibilityStrategyResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveSchemaCompatibilityStrategyResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveSchemaCompatibilityStrategyResponse(status = Some(status))) } override def getMaxMessageSize(request: pb.GetMaxMessageSizeRequest): Future[pb.GetMaxMessageSizeResponse] = @@ -1241,13 +1241,13 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies Future.successful( pb.GetMaxMessageSizeResponse( - status = Some(Status(code = Code.OK.index)), + status = Some(Status(code = Code.OK.value)), maxMessageSize ) ) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.GetMaxMessageSizeResponse(status = Some(status))) } override def setMaxMessageSize(request: pb.SetMaxMessageSizeRequest): Future[pb.SetMaxMessageSizeResponse] = @@ -1257,10 +1257,10 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Setting max message size policy for topic ${request.topic}. ${request.maxMessageSize}") adminClient.topicPolicies(request.isGlobal).setMaxMessageSize(request.topic, request.maxMessageSize) - Future.successful(pb.SetMaxMessageSizeResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.SetMaxMessageSizeResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.SetMaxMessageSizeResponse(status = Some(status))) } override def removeMaxMessageSize(request: pb.RemoveMaxMessageSizeRequest): Future[pb.RemoveMaxMessageSizeResponse] = @@ -1270,9 +1270,9 @@ class TopicPoliciesServiceImpl extends pb.TopicPoliciesServiceGrpc.TopicPolicies logger.info(s"Removing max message size policy for topic ${request.topic}") adminClient.topicPolicies(request.isGlobal).removeMaxMessageSize(request.topic) - Future.successful(pb.RemoveMaxMessageSizeResponse(status = Some(Status(code = Code.OK.index)))) + Future.successful(pb.RemoveMaxMessageSizeResponse(status = Some(Status(code = Code.OK.value)))) } catch { case err: Exception => - val status = Status(code = Code.FAILED_PRECONDITION.index, message = err.getMessage) + val status = Status(code = Code.FAILED_PRECONDITION.value, message = err.getMessage) Future.successful(pb.RemoveMaxMessageSizeResponse(status = Some(status))) } diff --git a/server/src/test/scala/config/mergeConfigsTest.scala b/server/src/test/scala/config/mergeConfigsTest.scala new file mode 100644 index 000000000..49d2b86b4 --- /dev/null +++ b/server/src/test/scala/config/mergeConfigsTest.scala @@ -0,0 +1,222 @@ +package config + +// NB: no `import zio.*` here - it shadows this package's `Config` with `zio.Config` +// (the same trap as `zio.System` vs `java.lang.System`). +import zio.test.* + +/** `mergeConfigs` hand-copies every field of `Config`, and Scala's named arguments + case-class + * defaults make an OMITTED field compile silently - the field just resolves to its default. That + * is exactly how `cookieSecure` and `cookieSameSite` came to be dropped (both documented in + * docs/configuration-reference.md, both dead in practice), despite the warning comment on + * `Config`. These tests are written so the SAME mistake on field #34 fails immediately. + */ +object mergeConfigsTest extends ZIOSpecDefault: + + /** Every field set to a value that differs from the case-class default. The identity property + * below only detects a dropped field if the fixture's value differs from that field's default, + * hence the deliberately odd values. */ + private val allSet = Config( + bindAddress = Some("10.1.2.3"), + port = Some(19999), + publicBaseUrl = Some("http://example.test/dekaf"), + basePath = Some("/dekaf"), + protocol = Some("https"), + tlsCertificateFilePath = Some("/tls/cert.pem"), + tlsKeyFilePath = Some("/tls/key.pem"), + cookieSecure = Some(true), + cookieSameSite = Some("strict"), + dataDir = Some("/var/lib/dekaf-test"), + pulsarName = Some("fixture-pulsar"), + pulsarColor = Some("rebeccapurple"), + pulsarWebUrl = Some("http://pulsar.test:8080"), + pulsarBrokerUrl = Some("pulsar://pulsar.test:6650"), + pulsarListenerName = Some("external"), + pulsarTlsKeyFilePath = Some("/tls/pulsar-key.pem"), + pulsarTlsCertificateFilePath = Some("/tls/pulsar-cert.pem"), + pulsarTlsTrustCertsFilePath = Some("/tls/pulsar-ca.pem"), + pulsarAllowTlsInsecureConnection = Some(true), + pulsarEnableTlsHostnameVerification = Some(true), + pulsarUseKeyStoreTls = Some(true), + pulsarSslProvider = Some("Conscrypt"), + pulsarTlsKeyStoreType = Some("PKCS12"), + pulsarTlsKeyStorePath = Some("/tls/keystore.p12"), + pulsarTlsKeyStorePassword = Some("keystore-pass"), + pulsarTlsTrustStoreType = Some("JKS"), + pulsarTlsTrustStorePath = Some("/tls/truststore.p12"), + pulsarTlsTrustStorePassword = Some("truststore-pass"), + pulsarTlsCiphers = Some(List("TLS_AES_256_GCM_SHA384")), + pulsarTlsProtocols = Some(List("TLSv1.3")), + defaultPulsarAuth = Some("""{"type":"empty"}"""), + internalHttpPort = Some(18001), + internalGrpcPort = Some(18002) + ) + + /** A second fixture whose value for EVERY field differs from `allSet`'s value for that same + * field. The per-field crosswire loop below builds a `high` config that is `allSet` with exactly + * one field taken from here, so each field's "changed" value is guaranteed to actually differ. + * Booleans can only flip, so their values necessarily repeat across the three TLS flags - which + * is precisely why pairwise-distinct fixtures alone cannot catch a crosswire between two equal + * boolean fields, and why the loop (not the identity/high-wins tests) is what closes that hole. */ + private val allSetAlt = Config( + bindAddress = Some("10.9.8.7"), + port = Some(20001), + publicBaseUrl = Some("http://alt.test/dekaf-alt"), + basePath = Some("/dekaf-alt"), + protocol = Some("http"), + tlsCertificateFilePath = Some("/tls/alt-cert.pem"), + tlsKeyFilePath = Some("/tls/alt-key.pem"), + cookieSecure = Some(false), + cookieSameSite = Some("lax"), + dataDir = Some("/var/lib/dekaf-alt"), + pulsarName = Some("alt-pulsar"), + pulsarColor = Some("goldenrod"), + pulsarWebUrl = Some("http://alt.test:18080"), + pulsarBrokerUrl = Some("pulsar://alt.test:16650"), + pulsarListenerName = Some("internal"), + pulsarTlsKeyFilePath = Some("/tls/alt-pulsar-key.pem"), + pulsarTlsCertificateFilePath = Some("/tls/alt-pulsar-cert.pem"), + pulsarTlsTrustCertsFilePath = Some("/tls/alt-pulsar-ca.pem"), + pulsarAllowTlsInsecureConnection = Some(false), + pulsarEnableTlsHostnameVerification = Some(false), + pulsarUseKeyStoreTls = Some(false), + pulsarSslProvider = Some("SunJSSE"), + pulsarTlsKeyStoreType = Some("JKS"), + pulsarTlsKeyStorePath = Some("/tls/alt-keystore.jks"), + pulsarTlsKeyStorePassword = Some("alt-keystore-pass"), + pulsarTlsTrustStoreType = Some("PKCS12"), + pulsarTlsTrustStorePath = Some("/tls/alt-truststore.p12"), + pulsarTlsTrustStorePassword = Some("alt-truststore-pass"), + pulsarTlsCiphers = Some(List("TLS_CHACHA20_POLY1305_SHA256")), + pulsarTlsProtocols = Some(List("TLSv1.2")), + defaultPulsarAuth = Some("""{"type":"token"}"""), + internalHttpPort = Some(28001), + internalGrpcPort = Some(28002) + ) + + private val mirror = summon[scala.deriving.Mirror.ProductOf[Config]] + + /** `base` with field #`index` replaced by `value`, reconstructed generically so the loop below + * does not have to hand-name 33 `.copy(...)` calls (the very hand-copying that this suite exists + * to police). Relies on case-class product order matching the constructor order. */ + private def withField(base: Config, index: Int, value: Any): Config = + val arr = base.productIterator.toArray + arr(index) = value + mirror.fromProduct(Tuple.fromArray(arr)) + + private def fieldsOf(c: Config): Map[String, Any] = + c.productElementNames.zip(c.productIterator).toMap + + private def differingFields(a: Config, b: Config): List[String] = + val fa = fieldsOf(a) + fieldsOf(b).collect { case (name, bv) if fa(name) != bv => name }.toList.sorted + + def spec = suite(this.getClass.toString)( + test("Config declares exactly the fields this suite knows about") { + // The None-check below and the identity property both go blind to a field whose default + // is NOT None: the fixture would carry `Some(default)` (so it is not "unset"), and a + // mergeConfigs that forgot to copy it would fall back to that same default (so nothing + // differs). Config is mostly non-None defaults - bindAddress, port, publicBaseUrl, + // basePath, protocol, dataDir, pulsarName, pulsarColor, pulsarWebUrl, pulsarBrokerUrl - + // so that hole is not hypothetical. + // + // Pinning the NAME SET closes it: adding a field to Config fails here first, which is + // the reminder that `// XXX - don't forget to make changes in mergeConfigs.scala` asks + // for. Deliberately a name set rather than a count, so the failure says which field. + val expected = Set( + "bindAddress", "port", "publicBaseUrl", "basePath", "protocol", + "tlsCertificateFilePath", "tlsKeyFilePath", "cookieSecure", "cookieSameSite", + "dataDir", "pulsarName", "pulsarColor", "pulsarWebUrl", "pulsarBrokerUrl", + "pulsarListenerName", "pulsarTlsKeyFilePath", "pulsarTlsCertificateFilePath", + "pulsarTlsTrustCertsFilePath", "pulsarAllowTlsInsecureConnection", + "pulsarEnableTlsHostnameVerification", "pulsarUseKeyStoreTls", "pulsarSslProvider", + "pulsarTlsKeyStoreType", "pulsarTlsKeyStorePath", "pulsarTlsKeyStorePassword", + "pulsarTlsTrustStoreType", "pulsarTlsTrustStorePath", "pulsarTlsTrustStorePassword", + "pulsarTlsCiphers", "pulsarTlsProtocols", "defaultPulsarAuth", + "internalHttpPort", "internalGrpcPort" + ) + val actual = fieldsOf(allSet).keySet + assertTrue(actual == expected) ?? + s"added: ${(actual -- expected).mkString(", ")}; removed: ${(expected -- actual).mkString(", ")}" + }, + test("the fixture sets every field of Config") { + // Guards the tests below: a field added to Config but not to `allSet` would be None + // here, and the identity property could no longer detect it being dropped. + val unset = fieldsOf(allSet).collect { case (name, None) => name }.toList.sorted + assertTrue(unset.isEmpty) ?? s"fields missing from the fixture: ${unset.mkString(", ")}" + }, + test("merging a config with itself preserves every field") { + // The key property: a field mergeConfigs forgets to copy silently falls back to its + // case-class default, which differs from the fixture value - so this catches drops. + val merged = mergeConfigs(allSet, allSet) + val lost = differingFields(allSet, merged) + assertTrue(lost.isEmpty) ?? s"mergeConfigs dropped: ${lost.mkString(", ")}" + }, + test("each field is merged from its OWN source - no crosswire between fields") { + // The identity and high-wins tests below go blind to a copy-paste crosswire whenever the + // two swapped fields hold the SAME value: e.g. + // pulsarEnableTlsHostnameVerification = highPriority.pulsarAllowTlsInsecureConnection... + // still yields Some(true) either way when both booleans are Some(true), so every other + // test stays green. This loop discriminates PER FIELD: for each field it makes `high` + // equal to `allSet` except for that one field (taken from `allSetAlt`), merges it over + // `allSet`, and asserts the merge changed EXACTLY that field. A crosswire trips it twice - + // the field that is read is now written into two outputs (an extra field changes), and the + // field that is no longer read never changes when it should. + val names = allSet.productElementNames.toVector + val altValues = allSetAlt.productIterator.toVector + val offenders = (0 until allSet.productArity).flatMap { i => + val high = withField(allSet, i, altValues(i)) + val merged = mergeConfigs(allSet, high) + val changed = differingFields(allSet, merged).toSet + val expected = Set(names(i)) + if changed == expected then None + else Some(s"${names(i)} -> changed={${changed.toList.sorted.mkString(",")}} expected={${names(i)}}") + } + assertTrue(offenders.isEmpty) ?? offenders.mkString("; ") + }, + test("the high-priority config wins for every field") { + val low = allSet + val high = allSet.copy( + bindAddress = Some("127.0.0.9"), + port = Some(12345), + cookieSecure = Some(false), + cookieSameSite = Some("lax"), + pulsarTlsCiphers = Some(List("TLS_CHACHA20_POLY1305_SHA256")) + ) + val merged = mergeConfigs(low, high) + assertTrue( + merged.bindAddress == Some("127.0.0.9"), + merged.port == Some(12345), + merged.cookieSecure == Some(false), + merged.cookieSameSite == Some("lax"), + merged.pulsarTlsCiphers == Some(List("TLS_CHACHA20_POLY1305_SHA256")) + ) + }, + test("an unset high-priority field falls back to the low-priority value") { + val high = Config( + bindAddress = None, + port = None, + cookieSecure = None, + cookieSameSite = None, + dataDir = None, + pulsarName = None + ) + val merged = mergeConfigs(allSet, high) + assertTrue( + merged.bindAddress == allSet.bindAddress, + merged.port == allSet.port, + merged.cookieSecure == allSet.cookieSecure, + merged.cookieSameSite == allSet.cookieSameSite, + merged.dataDir == allSet.dataDir, + merged.pulsarName == allSet.pulsarName + ) + }, + test("cookie hardening options survive a merge") { + // Regression: both were absent from mergeConfigs, so any deployment that set them got + // silent None and an unhardened cookie. + val merged = mergeConfigs(Config(cookieSecure = Some(true), cookieSameSite = Some("none")), Config()) + assertTrue( + merged.cookieSecure == Some(true), + merged.cookieSameSite == Some("none") + ) + } + ) diff --git a/server/src/test/scala/consumer/consumerRequestValidationTest.scala b/server/src/test/scala/consumer/consumerRequestValidationTest.scala new file mode 100644 index 000000000..922c24cb2 --- /dev/null +++ b/server/src/test/scala/consumer/consumerRequestValidationTest.scala @@ -0,0 +1,277 @@ +package consumer + +import _root_.consumer.session_config.{ConsumerSessionConfig, DeliveryOrderKey, MessageDeliveryOrder} +import _root_.consumer.session_runner.ConsumerSessionRunner +import com.google.rpc.code.Code +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import zio.test.* + +import java.util.concurrent.ConcurrentHashMap +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} +import scala.util.Try + +/** WHOSE FAULT IS IT, AND WHICH FIELD. + * + * Create used to answer FAILED_PRECONDITION for everything it refused: a request with no config at + * all, an enum this server has never heard of, a target that carries no consumption mode, a + * fraction of NaN, a broker that would not resolve a topic, and a server already at its session + * cap were one status apart from each other. FAILED_PRECONDITION says "the server or the world is + * in the wrong state, try again later"; a malformed request is INVALID_ARGUMENT and retrying it + * unchanged can never work. The two are also the difference between a client that can point at a + * field and one that can only shrug. + * + * And an enum value from a NEWER client is not a synonym for "the default". Absence is - proto3 + * zero is what every older client and every pre-field saved item sends - but `Unrecognized(n)` is + * a client asking for something this server does not implement, and silently running a different + * one is the quiet wrong answer this refuses. + */ +object consumerRequestValidationTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/cs-request-validation" + + private def targetPb( + readCompacted: Boolean = false, + isEnabled: Boolean = true, + topicFqns: Vector[String] = Vector(topicFqn) + ): consumerPb.ConsumerSessionTarget = + consumerPb.ConsumerSessionTarget( + isEnabled = isEnabled, + consumptionMode = Some( + consumerPb.ConsumerSessionTargetConsumptionMode(mode = + if readCompacted then + consumerPb.ConsumerSessionTargetConsumptionMode.Mode.ModeReadCompacted(consumerPb.ConsumerSessionTargetConsumptionMode.ReadCompacted()) + else consumerPb.ConsumerSessionTargetConsumptionMode.Mode.ModeRegular(consumerPb.ConsumerSessionTargetConsumptionMode.Regular()) + ) + ), + messageValueDeserializer = Some( + consumerPb.Deserializer(deserializer = + consumerPb.Deserializer.Deserializer.DeserializerUseLatestTopicSchema(consumerPb.Deserializer.UseLatestTopicSchema()) + ) + ), + topicSelector = Some( + consumerPb.TopicSelector(topicSelector = + consumerPb.TopicSelector.TopicSelector.MultiTopicSelector(consumerPb.MultiTopicSelector(topicFqns = topicFqns)) + ) + ) + ) + + private def startFromPb(startFrom: consumerPb.ConsumerSessionStartFrom.StartFrom): consumerPb.ConsumerSessionStartFrom = + consumerPb.ConsumerSessionStartFrom(startFrom = startFrom) + + /** A service whose builder never touches a broker, so every status below is this class's own + * classification and nothing else's. */ + private def service(build: (ConsumerSessionName, ConsumerSessionConfig) => ConsumerSessionRunner = (name, config) => + ConsumerSessionRunner( + sessionName = name, + sessionConfig = config, + sessionContextPool = _root_.consumer.session_runner.ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map.empty + ) + ): ConsumerServiceImpl = + ConsumerServiceImpl(new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner](), makeSession = build) + + private def create(svc: ConsumerServiceImpl, request: consumerPb.CreateConsumerRequest): com.google.rpc.status.Status = + Await.result(svc.createConsumer(request), Duration(60, SECONDS)).status.get + + private def createWith(config: consumerPb.ConsumerSessionConfig, name: String = "cs-request-validation"): com.google.rpc.status.Status = + create(service(), consumerPb.CreateConsumerRequest(consumerName = name, consumerSessionConfig = Some(config))) + + private def validConfig: consumerPb.ConsumerSessionConfig = + consumerPb.ConsumerSessionConfig(targets = Seq(targetPb())) + + private val unknownEnumSuite = suite("P2.4: an enum from a NEWER client is refused, not silently reinterpreted")( + test("an unrecognised delivery-order KEY is refused rather than quietly becoming publish time") { + // Publish time, broker publish time and event time order a merged session by three + // different clocks. A client that asked for a fourth got publish time and no word about + // it, so the session it saw was ordered by something it never chose. + val refused = Try(DeliveryOrderKey.fromPb(consumerPb.DeliveryOrderKey.Unrecognized(99))).failed.toOption + assertTrue( + refused.exists(_.getMessage.contains("99")), + refused.exists(_.getMessage.contains("delivery_order_key")) + ) ?? s"refused=${refused.map(_.getMessage)}" + }, + test("absence still means publish time - the documented default, and what every older client sends") { + assertTrue( + DeliveryOrderKey.fromPb(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_UNSPECIFIED) == DeliveryOrderKey.PublishTime, + DeliveryOrderKey.fromPb(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME) == DeliveryOrderKey.PublishTime, + DeliveryOrderKey.fromPb(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME) == DeliveryOrderKey.BrokerPublishTime, + DeliveryOrderKey.fromPb(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME) == DeliveryOrderKey.EventTime + ) + }, + test("an unrecognised delivery ORDER stays the settled product default - that one is deliberate") { + // Not a regression of the above: the owner decided this enum resolves an unknown value + // to the product default rather than accidentally selecting Fastest. Owner decision + // (2026-08-11, direct instruction): the default is Guaranteed - the third move of this + // default; the plan file's decision log is the record. + assertTrue(MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.Unrecognized(99)) == MessageDeliveryOrder.Guaranteed) + }, + test("the create RPC refuses the unknown key with INVALID_ARGUMENT and names the field") { + val status = createWith(validConfig.withDeliveryOrderKey(consumerPb.DeliveryOrderKey.Unrecognized(99))) + assertTrue( + status.code == Code.INVALID_ARGUMENT.value, + status.message.contains("delivery_order_key"), + status.message.contains("99") + ) ?? s"status=$status" + } + ) + + private val malformedConfigSuite = suite("P2.5: a malformed config says WHICH field, and is bad INPUT")( + test("a request with no config at all is INVALID_ARGUMENT, not a precondition failure") { + val status = create(service(), consumerPb.CreateConsumerRequest(consumerName = "cs-no-config")) + assertTrue( + status.code == Code.INVALID_ARGUMENT.value, + status.message.contains("consumer_session_config") + ) ?? s"status=$status" + }, + test("an empty TARGET names the target index and the field inside it") { + // The shallow saved-session check lets `{}` through as a target; on the server it used + // to surface as a bare 'Invalid ConsumerSessionTargetConsumptionMode mode.' under + // FAILED_PRECONDITION, with nothing saying which of the session's targets was at fault. + val status = createWith(consumerPb.ConsumerSessionConfig(targets = Seq(targetPb(), consumerPb.ConsumerSessionTarget()))) + assertTrue( + status.code == Code.INVALID_ARGUMENT.value, + status.message.contains("targets[1]"), + status.message.contains("consumption_mode") + ) ?? s"status=$status" + }, + test("a target with no topic selector names targets[n] and topic_selector") { + val broken = targetPb().clearTopicSelector + val status = createWith(consumerPb.ConsumerSessionConfig(targets = Seq(broken))) + assertTrue( + status.code == Code.INVALID_ARGUMENT.value, + status.message.contains("targets[0]"), + status.message.contains("topic_selector") + ) ?? s"status=$status" + }, + test("a start_from carrying no mode at all names start_from") { + val status = createWith(validConfig.withStartFrom(consumerPb.ConsumerSessionStartFrom())) + assertTrue( + status.code == Code.INVALID_ARGUMENT.value, + status.message.contains("start_from") + ) ?? s"status=$status" + } + ) + + private val statusCodeSuite = suite("P2.13: malformed INPUT is INVALID_ARGUMENT; the world's state is FAILED_PRECONDITION")( + test("a negative 'skip first n' is bad input") { + val status = createWith( + validConfig.withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromNthMessageAfterEarliest(consumerPb.NthMessageAfterEarliest(n = -1)) + ) + ) + ) + assertTrue(status.code == Code.INVALID_ARGUMENT.value, status.message.contains("Skip first n")) ?? s"status=$status" + }, + test("a 'latest n' past the accepted ceiling is bad input") { + val status = createWith( + validConfig.withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromNthMessageBeforeLatest( + consumerPb.NthMessageBeforeLatest(n = _root_.consumer.session_runner.latestNMaxAccepted + 1) + ) + ) + ) + ) + assertTrue(status.code == Code.INVALID_ARGUMENT.value, status.message.contains("Latest n messages")) ?? s"status=$status" + }, + test("a fraction that is not a fraction is bad input - both approximate modes") { + val nan = createWith( + validConfig.withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition( + consumerPb.ApproximateEntryPosition(fraction = Double.NaN) + ) + ) + ) + ) + val outOfRange = createWith( + validConfig.withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromApproximatePublishTimePosition( + consumerPb.ApproximatePublishTimePosition(fraction = 1.5) + ) + ) + ) + ) + assertTrue( + nan.code == Code.INVALID_ARGUMENT.value, + nan.message.contains("% of data"), + outOfRange.code == Code.INVALID_ARGUMENT.value, + outOfRange.message.contains("% of time") + ) ?? s"nan=$nan outOfRange=$outOfRange" + }, + test("more enabled targets than one session may run is bad input") { + val targets = (0 to ConsumerSessionRunner.maxEnabledTargetsPerSession).map(_ => targetPb()) + val status = createWith(consumerPb.ConsumerSessionConfig(targets = targets)) + assertTrue(status.code == Code.INVALID_ARGUMENT.value, status.message.contains("enabled targets")) ?? s"status=$status" + }, + test("a start position that cannot describe a compacted target is bad input - it is a CONFIG contradiction") { + val status = createWith( + consumerPb + .ConsumerSessionConfig(targets = Seq(targetPb(readCompacted = true))) + .withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition( + consumerPb.ApproximateEntryPosition(fraction = 0.5) + ) + ) + ) + ) + assertTrue(status.code == Code.INVALID_ARGUMENT.value, status.message.contains("read compacted")) ?? s"status=$status" + }, + test("a DISABLED target's contradiction is not held against the request") { + // The read-compacted refusal is indexed over ENABLED targets only, exactly as the + // session builder counts them - a disabled target is not part of this session. + val status = createWith( + consumerPb + .ConsumerSessionConfig(targets = Seq(targetPb(), targetPb(readCompacted = true, isEnabled = false))) + .withStartFrom( + startFromPb( + consumerPb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition( + consumerPb.ApproximateEntryPosition(fraction = 0.5) + ) + ) + ) + ) + assertTrue(status.code == Code.OK.value) ?? s"status=$status" + }, + test("a broker that will not resolve the topology stays FAILED_PRECONDITION - retrying CAN work") { + val svc = service(build = (_, _) => throw new RuntimeException("Failed to resolve topic persistent://x/y/z. broker unreachable")) + val status = create(svc, consumerPb.CreateConsumerRequest(consumerName = "cs-topology", consumerSessionConfig = Some(validConfig))) + assertTrue( + status.code == Code.FAILED_PRECONDITION.value, + status.message.contains("broker unreachable") + ) ?? s"status=$status" + }, + test("the server's own session cap stays FAILED_PRECONDITION - it is CURRENT STATE, not the request") { + val svc = ConsumerServiceImpl( + new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner](), + makeSession = (name, config) => + ConsumerSessionRunner( + sessionName = name, + sessionConfig = config, + sessionContextPool = _root_.consumer.session_runner.ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map.empty + ), + maxActiveSessions = 1 + ) + val first = create(svc, consumerPb.CreateConsumerRequest(consumerName = "cs-cap-1", consumerSessionConfig = Some(validConfig))) + val second = create(svc, consumerPb.CreateConsumerRequest(consumerName = "cs-cap-2", consumerSessionConfig = Some(validConfig))) + assertTrue( + first.code == Code.OK.value, + second.code == Code.FAILED_PRECONDITION.value, + second.message.contains("limit") + ) ?? s"first=$first second=$second" + }, + test("a well-formed request is still accepted - the validation refuses nothing it should not") { + assertTrue(createWith(validConfig).code == Code.OK.value) + } + ) + + def spec = suite(this.getClass.toString)(unknownEnumSuite, malformedConfigSuite, statusCodeSuite) diff --git a/server/src/test/scala/consumer/consumerServiceDeleteTest.scala b/server/src/test/scala/consumer/consumerServiceDeleteTest.scala new file mode 100644 index 000000000..c7518c13e --- /dev/null +++ b/server/src/test/scala/consumer/consumerServiceDeleteTest.scala @@ -0,0 +1,151 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} + +/** Deleting a consumer session: what the client is told, and what is left behind. + * + * Regression context: `deleteConsumer` removed the session from the map ONLY when `stop` returned + * without throwing - and `stop` never threw, because it swallowed every unsubscribe failure and + * reported nothing. So a broker that refused to delete a subscription produced a cheerful OK with + * the subscription still on the broker. Making `stop` honest then exposed the other half: if the + * removal stayed conditional on success, one undeletable subscription would make the session name + * permanently undeletable too. + */ +object consumerServiceDeleteTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/delete-me" + + private final class RecordingConsumer(unsubscribeFails: Boolean): + val unsubscribed = AtomicBoolean(false) + val closed = AtomicBoolean(false) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "unsubscribe" => + unsubscribed.set(true) + if unsubscribeFails then throw new RuntimeException("broker refused to delete the subscription") + null + case "close" => closed.set(true); null + case "pause" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => "proxy-consumer" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def session(consumer: Consumer[Array[Byte]]): ConsumerSessionRunner = + val pool = ConsumerSessionContextPool() + val target = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(topicFqn), + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = Map(topicFqn -> consumer), + pauseArbiters = (Map(topicFqn -> consumer)).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + ConsumerSessionRunner( + sessionName = "cs-delete", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + private def delete(service: ConsumerServiceImpl, name: String): consumerPb.DeleteConsumerResponse = + Await.result(service.deleteConsumer(consumerPb.DeleteConsumerRequest(consumerName = name)), Duration(30, SECONDS)) + + def spec = suite(this.getClass.toString)( + test("a clean delete releases the consumer and answers OK") { + val recording = RecordingConsumer(unsubscribeFails = false) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-delete", session(recording.consumer)) + val service = ConsumerServiceImpl(sessions) + + val response = delete(service, "cs-delete") + + assertTrue( + response.getStatus.code == com.google.rpc.code.Code.OK.value, + recording.unsubscribed.get, + recording.closed.get, + sessions.isEmpty + ) + }, + test("a delete whose unsubscribe fails REPORTS it instead of answering OK") { + // The regression: the failure was printed to stdout and the client was told OK, so a + // subscription left on the broker looked like a successful delete. + val recording = RecordingConsumer(unsubscribeFails = true) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-delete", session(recording.consumer)) + val service = ConsumerServiceImpl(sessions) + + val response = delete(service, "cs-delete") + + assertTrue( + response.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + response.getStatus.message.contains(topicFqn) + ) ?? s"status=${response.getStatus}" + }, + test("a delete whose unsubscribe fails still removes the handle and closes the consumer") { + // Otherwise one undeletable subscription strands the session forever: the entry stays, + // nothing can reach it, and every retry fails the same way. + val recording = RecordingConsumer(unsubscribeFails = true) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-delete", session(recording.consumer)) + val service = ConsumerServiceImpl(sessions) + + delete(service, "cs-delete") + + assertTrue(sessions.isEmpty, recording.closed.get) ?? + s"sessionsLeft=${sessions.size} closed=${recording.closed.get}" + }, + test("deleting a session that does not exist is a clean FAILED_PRECONDITION") { + val service = ConsumerServiceImpl(new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]()) + val response = delete(service, "nope") + assertTrue(response.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value) + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/consumerServiceDeliveryOrderTest.scala b/server/src/test/scala/consumer/consumerServiceDeliveryOrderTest.scala new file mode 100644 index 000000000..4b7f12955 --- /dev/null +++ b/server/src/test/scala/consumer/consumerServiceDeliveryOrderTest.scala @@ -0,0 +1,184 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} + +/** `SetDeliveryOrder`: the RPC that makes a disclosed Guaranteed stall ACTIONABLE. + * + * A dedicated RPC rather than a field on Resume, because it MUTATES a running session - and it + * changes the live session only, never the saved configuration. What is pinned here is the + * SURFACE: which requests are honoured, which are refused and with which status code, and that an + * unknown or already-closed session is answered cleanly. + * + * That a successful call really reaches the ordering layer and releases what the barrier was + * holding is pinned end to end - through this same RPC - by + * `consumer.session_runner.liveDeliveryOrderSwitchTest`, which lives next to the layer so it can + * inject the merge's clock. + * + * The session map is a constructor parameter so a real session sits behind the RPC without a + * broker. + */ +object consumerServiceDeliveryOrderTest extends ZIOSpecDefault: + + private val sessionName = "cs-delivery-order" + private def p(i: Int): String = s"persistent://public/default/set-order-partition-$i" + + private def call(service: ConsumerServiceImpl, name: String, order: consumerPb.MessageDeliveryOrder): consumerPb.SetDeliveryOrderResponse = + Await.result( + service.setDeliveryOrder(consumerPb.SetDeliveryOrderRequest(consumerName = name, messageDeliveryOrder = order)), + Duration(30, SECONDS) + ) + + private def codeOf(response: consumerPb.SetDeliveryOrderResponse): Int = response.getStatus.code + private def messageOf(response: consumerPb.SetDeliveryOrderResponse): String = response.getStatus.message + + private def targetRunner( + listener: ConsumerListener, + consumers: Map[String, Consumer[Array[Byte]]] = Map.empty + ): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p(0), p(1)))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p(0), p(1)), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(order: MessageDeliveryOrder, listener: ConsumerListener, consumers: Map[String, Consumer[Array[Byte]]] = Map.empty) + : ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty), + messageDeliveryOrder = order + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> targetRunner(listener, consumers)) + ) + + private def plainListener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def serviceWith(runner: ConsumerSessionRunner): ConsumerServiceImpl = + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, runner) + ConsumerServiceImpl(sessions) + + def spec = suite(this.getClass.toString)( + test("GUARANTEED -> BEST EFFORT is accepted, and the session's LIVE order really changes") { + val runner = session(MessageDeliveryOrder.Guaranteed, plainListener()) + val service = serviceWith(runner) + val orderBefore = runner.deliveryOrder + + val response = call(service, sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) + // Idempotent: a second click, or a second client, must not become an error. + val again = call(service, sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) + + assertTrue( + orderBefore == MessageDeliveryOrder.Guaranteed, + codeOf(response) == com.google.rpc.code.Code.OK.value, + runner.deliveryOrder == MessageDeliveryOrder.BestEffort, + codeOf(again) == com.google.rpc.code.Code.OK.value, + runner.deliveryOrder == MessageDeliveryOrder.BestEffort, + // The SAVED configuration is untouched - this is a live control, and writing the + // new order into the session definition is the client's decision, not this RPC's. + runner.sessionConfig.messageDeliveryOrder == MessageDeliveryOrder.Guaranteed + ) ?? s"response=${response.getStatus} again=${again.getStatus} live=${runner.deliveryOrder}" + }, + test("PROMOTION IS REFUSED with FAILED_PRECONDITION and a reason the user can act on") { + val runner = session(MessageDeliveryOrder.BestEffort, plainListener()) + val service = serviceWith(runner) + + val response = call(service, sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED) + + assertTrue( + codeOf(response) == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + messageOf(response).contains("Start the session again"), + runner.deliveryOrder == MessageDeliveryOrder.BestEffort // unchanged + ) ?? s"status=${response.getStatus} live=${runner.deliveryOrder}" + }, + test("SWITCHING TO FASTEST IS REFUSED, and says where Fastest does belong") { + val runner = session(MessageDeliveryOrder.Guaranteed, plainListener()) + val service = serviceWith(runner) + + val response = call(service, sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED) + + assertTrue( + codeOf(response) == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + messageOf(response).contains("Play"), + runner.deliveryOrder == MessageDeliveryOrder.Guaranteed + ) ?? s"status=${response.getStatus} live=${runner.deliveryOrder}" + }, + test("A REQUEST THAT NAMES NO ORDER IS INVALID_ARGUMENT, never the product default") { + // `MessageDeliveryOrder.fromPb` resolves absence to Best effort, which is right for a + // session configuration and would be a silent SWITCH here - this session runs + // Guaranteed and never asked to leave it. The wire value has to be checked before the + // conversion, and this is what pins it. + val runner = session(MessageDeliveryOrder.Guaranteed, plainListener()) + val service = serviceWith(runner) + + val unspecified = call(service, sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED) + val fromTheFuture = call(service, sessionName, consumerPb.MessageDeliveryOrder.Unrecognized(99)) + + assertTrue( + codeOf(unspecified) == com.google.rpc.code.Code.INVALID_ARGUMENT.value, + codeOf(fromTheFuture) == com.google.rpc.code.Code.INVALID_ARGUMENT.value, + runner.deliveryOrder == MessageDeliveryOrder.Guaranteed + ) ?? s"unspecified=${unspecified.getStatus} unrecognized=${fromTheFuture.getStatus} live=${runner.deliveryOrder}" + }, + test("an unknown session, and a session whose stream has ended, are both clean refusals") { + val emptyService = ConsumerServiceImpl(new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]()) + val unknown = call(emptyService, "nope", consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) + + val stopped = session(MessageDeliveryOrder.Guaranteed, plainListener()) + stopped.stop() + val closed = call(serviceWith(stopped), sessionName, consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) + + assertTrue( + codeOf(unknown) == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + messageOf(unknown).contains("No such consumer session"), + codeOf(closed) == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + messageOf(closed).contains("closed") + ) ?? s"unknown=${unknown.getStatus} closed=${closed.getStatus}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/consumerServiceLifecycleTest.scala b/server/src/test/scala/consumer/consumerServiceLifecycleTest.scala new file mode 100644 index 000000000..5d0ce6331 --- /dev/null +++ b/server/src/test/scala/consumer/consumerServiceLifecycleTest.scala @@ -0,0 +1,335 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong} +import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} +import scala.jdk.CollectionConverters.* + +/** CREATE AND DELETE UNDER ONE SESSION NAME ARE ONE OPERATION, NOT TWO. + * + * Every target of a session subscribes as `${sessionName}-${targetIndex}`, NON-DURABLE and + * EXCLUSIVE. Two runners under one name are therefore not merely wasteful, they are mutually + * exclusive on the broker: whichever subscribes second is refused outright. + * + * Two orderings made that reachable from ordinary use, and the browser re-creates a session on + * every configuration change, so both were the common path rather than a corner: + * + * - CREATE built and SUBSCRIBED the replacement in full and only then stopped the session it was + * replacing, so on the same topic the still-live predecessor rejected it before the atomic map + * swap was ever reached; + * - DELETE read the runner, stopped it, and then removed the NAME unconditionally - so a create + * that installed a new session meanwhile had its session silently unhooked, left running with + * nothing holding a handle to it. + * + * The broker sits behind proxy consumers and an injected session builder, so all of it runs + * offline. + */ +object consumerServiceLifecycleTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/lifecycle" + private val sessionName = "cs-lifecycle" + + /** A consumer that records what was done to it and can be held inside `unsubscribe`, which is + * where `stop` spends its time in production. `unsubscribeEntered` lets a test wait until a + * stop is COMMITTED to its broker work; `unsubscribeFails` makes the stop report a failure. */ + private final class RecordingConsumer(label: String, unsubscribeGate: Option[CountDownLatch] = None, unsubscribeFails: Boolean = false): + val unsubscribed = AtomicBoolean(false) + val closed = AtomicBoolean(false) + val unsubscribeEntered = CountDownLatch(1) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "unsubscribe" => + unsubscribeEntered.countDown() + unsubscribeGate.foreach(_.await(60, TimeUnit.SECONDS)) + unsubscribed.set(true) + if unsubscribeFails then throw new RuntimeException("broker refused to delete the subscription") + null + case "close" => closed.set(true); null + case "pause" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(label.hashCode) + case "toString" => s"proxy-consumer($label)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def session(consumer: Consumer[Array[Byte]]): ConsumerSessionRunner = + val pool = ConsumerSessionContextPool() + val target = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(topicFqn), + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = Map(topicFqn -> consumer), + pauseArbiters = (Map(topicFqn -> consumer)).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + private def createRequest(name: String = sessionName): consumerPb.CreateConsumerRequest = + consumerPb.CreateConsumerRequest(consumerName = name, consumerSessionConfig = Some(consumerPb.ConsumerSessionConfig())) + + private def create(service: ConsumerServiceImpl, name: String = sessionName): consumerPb.CreateConsumerResponse = + Await.result(service.createConsumer(createRequest(name)), Duration(60, SECONDS)) + + private def delete(service: ConsumerServiceImpl, name: String = sessionName): consumerPb.DeleteConsumerResponse = + Await.result(service.deleteConsumer(consumerPb.DeleteConsumerRequest(consumerName = name)), Duration(60, SECONDS)) + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + /** The fresh observer a racing resume brings - what the client's play stream sees. */ + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + val received = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + val completed = AtomicBoolean(false) + override def onNext(value: consumerPb.ResumeResponse): Unit = + received.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = completed.set(true) + + def spec = suite(this.getClass.toString)( + test("CREATING OVER AN EXISTING SESSION STOPS IT BEFORE THE REPLACEMENT IS BUILT") { + // THE ordering defect. Both runners want the same exclusive, non-durable subscription, + // so building the replacement first meant the predecessor - still live - refused it. + val events = ConcurrentLinkedQueue[String]() + val predecessor = RecordingConsumer("old") + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, session(predecessor.consumer)) + + val service = ConsumerServiceImpl( + sessions, + (_, _) => + events.add(if predecessor.unsubscribed.get then "built-after-stop" else "built-while-predecessor-live") + session(RecordingConsumer("new").consumer) + ) + + val response = create(service) + + assertTrue( + response.getStatus.code == com.google.rpc.code.Code.OK.value, + predecessor.unsubscribed.get, + predecessor.closed.get, + events.asScala.toVector == Vector("built-after-stop") + ) ?? s"status=${response.getStatus} events=${events.asScala.toVector}" + }, + test("TWO CONCURRENT CREATES UNDER ONE NAME NEVER BUILD AT THE SAME TIME") { + // Nothing serialized them, so two browser tabs (or a retry) could have two runners + // subscribing to one exclusive subscription at once. + val inside = AtomicInteger(0) + val overlaps = AtomicInteger(0) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val service = ConsumerServiceImpl( + sessions, + (_, _) => + if inside.incrementAndGet() > 1 then overlaps.incrementAndGet() + // WIDENS the window on purpose: "they did not overlap" must be a claim about + // the serialization, not about how fast the two threads happened to run. + Thread.sleep(300) + inside.decrementAndGet() + session(RecordingConsumer("concurrent").consumer) + ) + + val first = worker("create-1")(create(service)) + val second = worker("create-2")(create(service)) + first.start() + second.start() + first.join(60_000) + second.join(60_000) + + assertTrue(overlaps.get == 0, sessions.size == 1) ?? + s"${overlaps.get} concurrent builds under one session name" + }, + test("A DELETE THAT RACES A CREATE REMOVES ONLY THE SESSION IT ACTUALLY STOPPED") { + // Delete read runner A, stopped it, then removed the NAME. A create that installed B + // meanwhile lost it: B stayed running with nothing holding a handle to it, and the + // browser kept a session the server no longer knew about. + val gate = CountDownLatch(1) + val doomed = RecordingConsumer("A", unsubscribeGate = Some(gate)) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, session(doomed.consumer)) + val service = ConsumerServiceImpl(sessions) + + val deleting = worker("delete-A")(delete(service)) + deleting.start() + // Wait until delete is committed to stopping A, then install B exactly as a create + // would. A BARE `put`, deliberately: the per-name lifecycle lock would keep a real + // create out of this window, and the point here is that the compare-and-remove holds on + // its own rather than only because of the lock. + Thread.sleep(300) + val replacement = session(RecordingConsumer("B").consumer) + sessions.put(sessionName, replacement) + + gate.countDown() + deleting.join(60_000) + + assertTrue(sessions.get(sessionName) eq replacement) ?? + s"the delete removed a session it never stopped; left ${Option(sessions.get(sessionName)).map(_ => "something else").getOrElse("nothing")}" + }, + test("an ordinary create under a fresh name stores exactly one session") { + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val service = ConsumerServiceImpl(sessions, (_, _) => session(RecordingConsumer("fresh").consumer)) + + val response = create(service) + + assertTrue(response.getStatus.code == com.google.rpc.code.Code.OK.value, sessions.size == 1) + }, + test("a create that FAILS to build reports it and leaves no session behind") { + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val service = ConsumerServiceImpl(sessions, (_, _) => throw new IllegalArgumentException("no enabled targets")) + + val response = create(service) + + assertTrue( + response.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + response.getStatus.message.contains("no enabled targets"), + sessions.isEmpty + ) ?? s"status=${response.getStatus} sessions=${sessions.size}" + }, + test("TWO CREATES UNDER DIFFERENT NAMES DO NOT SERIALIZE - the lifecycle lock is per name") { + // The lifecycle lock is held across the predecessor's stop and the replacement's WHOLE + // build - subscribing every consumer, seeking it, and for a Latest-N start-from a + // backward walk of one admin lookup per entry - which is unbounded broker work, not + // "one broker round trip". The old 64-stripe scheme made two DIFFERENT names share a + // lock whenever their hashes collided mod 64; these two names collide there by + // construction, so this test fails against any name-independent striping. + val nameA = "cs-lifecycle-stripe-a" + val nameB = (1 to 100_000).view + .map(i => s"cs-lifecycle-stripe-b$i") + .find(candidate => math.floorMod(candidate.hashCode, 64) == math.floorMod(nameA.hashCode, 64)) + .get + val gate = CountDownLatch(1) + val enteredSlowBuild = CountDownLatch(1) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val service = ConsumerServiceImpl( + sessions, + (name, _) => + if name == nameA then + enteredSlowBuild.countDown() + gate.await(60, TimeUnit.SECONDS) + session(RecordingConsumer(name).consumer) + ) + + val slow = worker("create-slow-name")(create(service, nameA)) + slow.start() + enteredSlowBuild.await(60, TimeUnit.SECONDS) + + val fast = worker("create-fast-name")(create(service, nameB)) + fast.start() + fast.join(5_000) + val fastFinishedWhileSlowHeldItsLock = !fast.isAlive + + gate.countDown() + slow.join(60_000) + fast.join(60_000) + + assertTrue(fastFinishedWhileSlowHeldItsLock, sessions.size == 2) ?? + s"create($nameB) sat behind create($nameA)'s broker work despite the different name" + }, + test("A RESUME THAT RACES A DELETE WAITS FOR IT AND IS TOLD THE SESSION IS GONE") { + // Unserialized, resume read the runner mid-delete and wired the fresh observer into a + // runner whose stop was already in flight; the stop then completed the stream with no + // status frame, and the client's play stream hung silent forever with nothing to show + // and nothing to say. + val gate = CountDownLatch(1) + val doomed = RecordingConsumer("doomed", unsubscribeGate = Some(gate)) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, session(doomed.consumer)) + val service = ConsumerServiceImpl(sessions) + val observer = RecordingObserver() + + val deleting = worker("delete-racing")(delete(service)) + deleting.start() + doomed.unsubscribeEntered.await(60, TimeUnit.SECONDS) + + val resuming = worker("resume-racing")( + service.resume(consumerPb.ResumeRequest(consumerName = sessionName, includeConsumerStats = true), observer) + ) + resuming.start() + resuming.join(1_500) + val resumeWaitedForTheDelete = resuming.isAlive + + gate.countDown() + deleting.join(60_000) + resuming.join(60_000) + + val statuses = observer.received.asScala.toVector.flatMap(_.status).map(_.code) + assertTrue( + resumeWaitedForTheDelete, + statuses == Vector(com.google.rpc.code.Code.FAILED_PRECONDITION.value), + observer.completed.get + ) ?? s"waitedForDelete=$resumeWaitedForTheDelete statuses=$statuses completed=${observer.completed.get}" + }, + test("REPLACING A SESSION LOGS A PREDECESSOR THAT COULD NOT BE RELEASED, as its comment promises") { + // `storeConsumerSession`'s scaladoc says the stop failure "is logged" - but + // `Try(replaced.stop())` discarded it, so a predecessor that failed to release + // disappeared without a trace in exactly the situation an operator needs the trace. + val appender = new ch.qos.logback.core.read.ListAppender[ch.qos.logback.classic.spi.ILoggingEvent]() + appender.start() + val logbackLogger = org.slf4j.LoggerFactory + .getLogger("consumer.session_runner.storeConsumerSession") + .asInstanceOf[ch.qos.logback.classic.Logger] + logbackLogger.addAppender(appender) + try + val stubborn = RecordingConsumer("stubborn", unsubscribeFails = true) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, session(stubborn.consumer)) + + storeConsumerSession(sessions, sessionName, session(RecordingConsumer("replacement").consumer)) + + val warned = appender.list.asScala.toVector.map(_.getFormattedMessage) + assertTrue(warned.exists(m => m.contains(sessionName) && m.contains("could not be fully released"))) ?? + s"the replaced runner's stop failure was logged nowhere; logged=$warned" + finally logbackLogger.detachAppender(appender) + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/consumerServiceResumeTest.scala b/server/src/test/scala/consumer/consumerServiceResumeTest.scala new file mode 100644 index 000000000..0fce8f159 --- /dev/null +++ b/server/src/test/scala/consumer/consumerServiceResumeTest.scala @@ -0,0 +1,218 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import zio.test.* + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters.* + +/** What `ConsumerServiceImpl.resume` does with the flags on the request it was handed. + * + * Regression context: `ResumeRequest.include_consumer_stats` has existed in the proto since the + * progress API was added, and the browser sets it - but the service read only `is_debug` off the + * request and passed that alone to the session. Every client therefore received consumer stats, + * including the MESSAGE-LESS progress frames a skip in flight pushes, whether or not it had said + * it could handle them. + * + * The session map is a constructor parameter so this test can put a real session behind the RPC + * without a broker: everything the resume path touches is the runner and the observer. + */ +object consumerServiceResumeTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/resume-flags" + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val responses = java.util.concurrent.ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + val completed = java.util.concurrent.atomic.AtomicBoolean(false) + val nextAfterCompleted = java.util.concurrent.atomic.AtomicInteger(0) + override def onNext(value: consumerPb.ResumeResponse): Unit = + if completed.get then nextAfterCompleted.incrementAndGet() + responses.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = completed.set(true) + def received: Vector[consumerPb.ResumeResponse] = responses.asScala.toVector + def statsFrames: Vector[consumerPb.ConsumerStats] = received.flatMap(_.consumerStats) + + /** A consumer whose `resume()` throws - the late-target failure that used to leave earlier + * targets' listeners pushing into an observer the catch had already completed. */ + private def resumeThrowingConsumer(): org.apache.pulsar.client.api.Consumer[Array[Byte]] = + val handler = new java.lang.reflect.InvocationHandler: + override def invoke(proxy: Object, method: java.lang.reflect.Method, args: Array[Object]): Object = + method.getName match + case "resume" => throw new IllegalStateException("the broker refused to resume this consumer") + case "getTopic" => topicFqn + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => "proxy-consumer(resume-throws)" + case _ => null + java.lang.reflect.Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[org.apache.pulsar.client.api.Consumer[Array[Byte]]]), handler) + .asInstanceOf[org.apache.pulsar.client.api.Consumer[Array[Byte]]] + + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetRunner( + consumerListener: ConsumerListener, + consumers: Map[String, org.apache.pulsar.client.api.Consumer[Array[Byte]]] = Map.empty + ): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(topicFqn), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = (consumers).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(consumerListener: ConsumerListener): ConsumerSessionRunner = + sessionWith(Map(0 -> targetRunner(consumerListener))) + + private def sessionWith(targets: Map[Int, ConsumerSessionTargetRunner]): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-resume-flags", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = targets + ) + + /** A service holding one session that is part-way through a skip of 3. */ + private def serviceWithSkippingSession(): (ConsumerServiceImpl, ConsumerListener) = + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-resume-flags", session(l)) + (ConsumerServiceImpl(sessions), l) + + def spec = suite(this.getClass.toString)( + test("include_consumer_stats = false really does suppress the stats") { + val (service, l) = serviceWithSkippingSession() + val observer = RecordingObserver() + + service.resume( + consumerPb.ResumeRequest(consumerName = "cs-resume-flags", includeConsumerStats = false, isDebug = false), + observer + ) + (1 to 3).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue(observer.statsFrames.isEmpty, observer.received.isEmpty) ?? + s"the request asked for no consumer stats and got ${observer.received.size} frames: ${observer.statsFrames}" + }, + test("include_consumer_stats = true delivers them") { + val (service, l) = serviceWithSkippingSession() + val observer = RecordingObserver() + + service.resume( + consumerPb.ResumeRequest(consumerName = "cs-resume-flags", includeConsumerStats = true, isDebug = false), + observer + ) + (1 to 3).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue( + observer.statsFrames.flatMap(_.startFromProgress).map(_.messagesToSkip) == Vector(3L, 3L) + ) ?? s"got ${observer.statsFrames}" + }, + test("resuming a session that does not exist is still a clean FAILED_PRECONDITION") { + val service = ConsumerServiceImpl(new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]()) + val observer = RecordingObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "nope", includeConsumerStats = true), observer) + + assertTrue( + observer.received.size == 1, + observer.received.head.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value + ) + }, + test("A RESUME THAT FAILS MID-WIRING ENDS THE STREAM THROUGH THE TERMINAL GATE, exactly once") { + // Two targets: the first resumes fine and its listener threads may already be pushing; + // the second throws. The catch used to write a status frame and onCompleted STRAIGHT to + // the observer - outside the send lock, without setting the terminal flag - so the + // earlier target's pushes kept landing in a stream that had already ended. + val healthy = listener() + val broken = listener() + val runner = sessionWith(Map( + 0 -> targetRunner(healthy), + 1 -> targetRunner(broken, consumers = Map(topicFqn -> resumeThrowingConsumer())) + )) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-resume-flags", runner) + val service = ConsumerServiceImpl(sessions) + val observer = RecordingObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-resume-flags", includeConsumerStats = true), observer) + val framesAfterCatch = observer.received.size + + // A listener thread still in flight pushes now. The terminal flag must silence it - + // AND TELL IT SO: silence alone let the healthy target's `deliverNow` read a normal + // return as "the client has it" and acknowledge a message no browser ever saw. The + // refusal is what routes it back to the broker for redelivery instead. + val lateSendRefused = scala.util.Try(runner.sendResponse(observer, Seq(consumerPb.Message()), Vector.empty)).isFailure + + assertTrue( + framesAfterCatch == 1, + observer.received.head.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + observer.completed.get, + lateSendRefused, + observer.received.size == 1, + observer.nextAfterCompleted.get == 0 + ) ?? (s"framesAfterCatch=$framesAfterCatch total=${observer.received.size} lateSendRefused=$lateSendRefused " + + s"nextAfterCompleted=${observer.nextAfterCompleted.get} completed=${observer.completed.get}") + }, + test("RESUMING A SESSION WHOSE STREAM HAS ENDED answers non-OK instead of wiring a dead runner") { + // The delete/resume race, after the delete has won: the runner is stopped (terminal) + // but still reachable. Wiring the fresh observer into it hung the play stream silently + // - the terminal flag swallows every send, so the client waited on a stream that could + // never speak. It must be answered and completed instead; its remedy is to recreate. + val runner = session(listener()) + runner.stop() + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-resume-flags", runner) + val service = ConsumerServiceImpl(sessions) + val observer = RecordingObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-resume-flags", includeConsumerStats = true), observer) + + assertTrue( + observer.received.size == 1, + observer.received.head.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + observer.completed.get + ) ?? s"a dead runner answered with frames=${observer.received.size} completed=${observer.completed.get}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/consumerServiceTopicPositionsTest.scala b/server/src/test/scala/consumer/consumerServiceTopicPositionsTest.scala new file mode 100644 index 000000000..ddf3ff629 --- /dev/null +++ b/server/src/test/scala/consumer/consumerServiceTopicPositionsTest.scala @@ -0,0 +1,391 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import _root_.pulsar_auth.RequestContext +import com.google.rpc.code.Code +import com.tools.teal.pulsar.ui.api.v1.consumer.{GetTopicPositionsRequest, GetTopicPositionsResponse} +import consumer.session_runner.{ + ConsumerListener, + ConsumerPauseArbiter, + ConsumerSessionContextPool, + ConsumerSessionRunner, + ConsumerSessionTargetMessageHandler, + ConsumerSessionTargetRunner, + ConsumerSessionTargetStats, + TopicConsumedBounds, + TopicCursor, + mergeConsumedBounds, + topicPositionLookupParallelism +} +import org.apache.pulsar.client.admin.{PulsarAdmin, Topics} +import org.apache.pulsar.client.api.{Consumer, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong, AtomicReference} +import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit} +import scala.concurrent.duration.{Duration, SECONDS} +import scala.concurrent.{Await, Future} + +/** The Topic Positions RPC and the cursor bookkeeping behind it. + * + * The arithmetic lives in `topicPositionsTest`; what is pinned here is everything that arithmetic + * cannot see - the answer for a session that does not exist, and how the read position is + * accumulated across listener threads and across targets that share a topic. + * + * NO BROKER. The session lookup happens before the admin client is resolved, so the not-found path + * runs without a request context; the cursor paths are the listener's own state. + */ +object consumerServiceTopicPositionsTest extends ZIOSpecDefault: + + /** A broker admin whose every lookup is counted, held at a gate, and watched for how many run + * at once - the three things the Topic Positions poll has to be honest about. */ + private final class CountingAdmin(gate: CountDownLatch): + val examineCalls = AtomicInteger(0) + val statsCalls = AtomicInteger(0) + val inFlight = AtomicInteger(0) + val peakInFlight = AtomicInteger(0) + val firstLookupStarted = CountDownLatch(1) + + private def held[A](answer: => A): A = + val current = inFlight.incrementAndGet() + peakInFlight.updateAndGet(previous => math.max(previous, current)) + firstLookupStarted.countDown() + gate.await(120, TimeUnit.SECONDS) + try answer + finally inFlight.decrementAndGet() + + private def examined(topicFqn: String): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(1_000L) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap("{}".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, 1L, -1)) + msg + + private val topicsApi: Topics = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "examineMessage" => + examineCalls.incrementAndGet() + held(examined(args(0).asInstanceOf[String])) + case "getInternalStats" => + statsCalls.incrementAndGet() + held { + val stats = new org.apache.pulsar.common.policies.data.PersistentTopicInternalStats() + stats.numberOfEntries = 10L + stats.currentLedgerEntries = 10L + stats + } + case "toString" => "proxy-topics" + case _ => null + Proxy.newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Topics]), handler).asInstanceOf[Topics] + + val admin: PulsarAdmin = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "topics" => topicsApi + case "toString" => "proxy-admin" + case _ => null + Proxy.newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[PulsarAdmin]), handler).asInstanceOf[PulsarAdmin] + + private def proxyConsumer(topicFqn: String): Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + /** A session over `topicCount` physical topics - the wide shape a partitioned selector produces + * and the one the poll's cost is measured in. */ + private def wideSession(sessionName: String, topicCount: Int): ConsumerSessionRunner = + val consumers = (1 to topicCount).map(i => s"persistent://public/default/$sessionName-partition-$i" -> proxyConsumer(s"$sessionName-$i")).toMap + val pool = ConsumerSessionContextPool() + val target = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumers.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumers.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + /** The admin client arrives through the gRPC request context, which is a thread-local - so a + * poll has to resolve it on the calling thread whatever it does with it afterwards. */ + private def withAdmin[A](admin: PulsarAdmin)(body: => A): A = + io.grpc.Context.current().withValue(RequestContext.pulsarAdmin, admin).call(() => body) + + private def worker(name: String)(body: => Unit): Thread = + val thread = new Thread((() => body): Runnable, name) + thread.setDaemon(true) + thread + + private def msgId(ledger: Long, entry: Long) = MessageIdImpl(ledger, entry, -1) + private def bounds(firstEntry: Long, lastEntry: Long, firstTime: Long = 1000, lastTime: Long = 2000) = + TopicConsumedBounds(TopicCursor(msgId(1, firstEntry), firstTime), TopicCursor(msgId(1, lastEntry), lastTime)) + + def spec = suite("consumerService topic positions")( + suite("a session that is not running")( + test("answers FAILED_PRECONDITION rather than failing the call") { + // The tab polls as soon as it is opened, which is routinely BEFORE the play button. + // That has to be a status the client can render as "not started yet", not a fault - + // and it must not need a broker to say so. + val service = ConsumerServiceImpl(new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]()) + val response = service.getTopicPositions(GetTopicPositionsRequest(consumerName = "never-created")) + + for res <- zio.ZIO.fromFuture(_ => response) + yield assertTrue(res.status.exists(_.code == Code.FAILED_PRECONDITION.value)) && + assertTrue(res.positions.isEmpty) && + assertTrue(res.status.exists(_.message.contains("never-created"))) + }, + test("P2.6: a session whose stream has ENDED answers the same 'session is gone' status, and scans nothing") { + // The tab polls a started session once a second and only stops when it is told the + // session is gone. A runner whose stream has been ended - stopped, replaced, or + // failed out of a resume - is gone by every meaning the client has for the word, yet + // it stayed in the map answering OK: the poll went on scanning the broker for a + // session nobody could ever see again, once a second, for as long as the tab lived. + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val runner = wideSession("cs-positions-ended", topicCount = 3) + sessions.put("cs-positions-ended", runner) + // What ending a stream looks like when a resume fails: terminal, still installed, + // consumers still held. + runner.failAndComplete(com.google.rpc.status.Status(code = Code.FAILED_PRECONDITION.value, message = "the resume failed")) + + val gate = CountDownLatch(1) + gate.countDown() + val admin = CountingAdmin(gate) + val service = ConsumerServiceImpl(sessions) + val response = withAdmin(admin.admin)(service.getTopicPositions(GetTopicPositionsRequest(consumerName = "cs-positions-ended"))) + + for res <- zio.ZIO.fromFuture(_ => response) + yield assertTrue( + res.status.exists(_.code == Code.FAILED_PRECONDITION.value), + res.positions.isEmpty, + res.status.exists(_.message.contains("cs-positions-ended")), + admin.examineCalls.get == 0, // nothing was asked of the broker for a dead session + admin.statsCalls.get == 0 + ) ?? s"status=${res.status} examines=${admin.examineCalls.get} stats=${admin.statsCalls.get}" + } + ), + suite("what a wide poll costs")( + test("A POLL IS OFF THE CALLER'S THREAD, BOUNDED, AND COALESCED - a second poll joins the scan in flight") { + // The tab polls this once a SECOND and a session may hold up to 2,000 physical + // topics, each costing two examineMessage calls plus getInternalStats. The + // implementation mapped them synchronously and serially on the gRPC thread before + // returning an already-completed Future, so one browser could hold a service thread + // for the whole of a 6,000-call sweep, a client deadline cancelled nothing, and + // every tick of the poll started ANOTHER full sweep on top of the one still running. + // + // Pinned by latches: nothing is asserted about how long anything takes, only that + // both calls have RETURNED while the scan is provably still inside the broker, and + // that the whole episode cost exactly one scan's worth of admin calls. + val topics = 100 + val gate = CountDownLatch(1) + val admin = CountingAdmin(gate) + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-tp-wide", wideSession("cs-tp-wide", topics)) + val service = ConsumerServiceImpl(sessions) + val request = GetTopicPositionsRequest(consumerName = "cs-tp-wide") + + val firstAnswer = AtomicReference[Future[GetTopicPositionsResponse]]() + val secondAnswer = AtomicReference[Future[GetTopicPositionsResponse]]() + val firstReturned = CountDownLatch(1) + val secondReturned = CountDownLatch(1) + + val firstPoll = worker("cs-tp-poll-1") { + firstAnswer.set(withAdmin(admin.admin)(service.getTopicPositions(request))) + firstReturned.countDown() + } + firstPoll.start() + val scanReallyStarted = admin.firstLookupStarted.await(120, TimeUnit.SECONDS) + + // The next tick of the same one-second poll, with the first scan still in the broker. + val secondPoll = worker("cs-tp-poll-2") { + secondAnswer.set(withAdmin(admin.admin)(service.getTopicPositions(request))) + secondReturned.countDown() + } + secondPoll.start() + val bothReturnedWhileScanning = + firstReturned.await(15, TimeUnit.SECONDS) && secondReturned.await(15, TimeUnit.SECONDS) + + gate.countDown() + firstPoll.join(120_000) + secondPoll.join(120_000) + val first = Await.result(firstAnswer.get, Duration(120, SECONDS)) + val second = Await.result(secondAnswer.get, Duration(120, SECONDS)) + + assertTrue( + scanReallyStarted, + bothReturnedWhileScanning, // neither call held its gRPC thread through the sweep + admin.examineCalls.get == 2 * topics, // ONE scan: first and last entry per topic + admin.statsCalls.get == topics, + admin.peakInFlight.get > 1, // genuinely off-thread, not just deferred + admin.peakInFlight.get <= topicPositionLookupParallelism, // and bounded while it is + first.status.exists(_.code == Code.OK.value), + second.status.exists(_.code == Code.OK.value), + first.positions.size == topics, + second.positions == first.positions // the joined poll gets the scan's answer + ) ?? (s"bothReturnedWhileScanning=$bothReturnedWhileScanning examine=${admin.examineCalls.get} " + + s"(one scan = ${2 * topics}) stats=${admin.statsCalls.get} peak=${admin.peakInFlight.get} " + + s"bound=$topicPositionLookupParallelism rows=${first.positions.size}/${second.positions.size}") + } + ), + suite("the listener's read position")( + test("records both the first and last consumed positions") { + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.recordCursor("persistent://t/n/a", msgId(1, 5), 1000) + listener.recordCursor("persistent://t/n/a", msgId(1, 9), 2000) + + val recorded = listener.consumedBounds("persistent://t/n/a") + assertTrue(recorded.first == TopicCursor(msgId(1, 5), 1000)) && + assertTrue(recorded.last == TopicCursor(msgId(1, 9), 2000)) + }, + test("an older later observation widens first but does NOT walk last backwards") { + // `negativeAcknowledge` and the merge's cap both hand messages back, so an older one + // legitimately arrives after a newer one has been counted. Letting the cursor follow + // it would make the view flicker between two positions, neither of them "how far has + // this read". + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.recordCursor("persistent://t/n/a", msgId(1, 9), 2000) + listener.recordCursor("persistent://t/n/a", msgId(1, 4), 1500) + + val recorded = listener.consumedBounds("persistent://t/n/a") + assertTrue(recorded.first == TopicCursor(msgId(1, 4), 1500)) && + assertTrue(recorded.last == TopicCursor(msgId(1, 9), 2000)) + }, + test("equal message ids keep the first observation and refresh the last one") { + // Non-persistent Pulsar messages may all expose 0:0. The id cannot distinguish + // them, but the range must not freeze both timestamps on the first observation. + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val sameId = msgId(0, 0) + listener.recordCursor("non-persistent://t/n/a", sameId, 1000) + listener.recordCursor("non-persistent://t/n/a", sameId, 2000) + + val recorded = listener.consumedBounds("non-persistent://t/n/a") + assertTrue(recorded.first.publishTime == 1000) && + assertTrue(recorded.last.publishTime == 2000) + }, + test("keeps one position PER TOPIC - a partitioned session must not share one") { + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.recordCursor("persistent://t/n/a-partition-0", msgId(1, 5), 1000) + listener.recordCursor("persistent://t/n/a-partition-1", msgId(2, 3), 1100) + + assertTrue(listener.consumedBounds.size == 2) && + assertTrue(listener.consumedBounds("persistent://t/n/a-partition-0").first.messageId == msgId(1, 5)) && + assertTrue(listener.consumedBounds("persistent://t/n/a-partition-1").last.messageId == msgId(2, 3)) + }, + test("survives concurrent recording from several listener threads") { + // One Pulsar listener thread per physical topic writes this while a gRPC thread + // reads it. The high-water mark must be the true maximum however the two interleave. + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val entries = (1 to 500).toVector + + for _ <- zio.ZIO.foreachParDiscard(entries)(entry => + zio.ZIO.succeed(listener.recordCursor("persistent://t/n/a", msgId(1, entry.toLong), entry.toLong)) + ) + yield + val recorded = listener.consumedBounds("persistent://t/n/a") + assertTrue(recorded.first.messageId == msgId(1, 1)) && + assertTrue(recorded.last.messageId == msgId(1, 500)) + } + ), + suite("reconciling targets that share a topic")( + test("takes the union of two targets reading one topic") { + // Two targets differing only in their filters is an ordinary configuration; each + // keeps its own listener and range, but the table has one row. + val middle = Map("persistent://t/n/a" -> bounds(3, 20)) + val wider = Map("persistent://t/n/a" -> bounds(1, 40, firstTime = 500, lastTime = 3000)) + + assertTrue(mergeConsumedBounds(Vector(middle, wider))("persistent://t/n/a") == wider("persistent://t/n/a")) && + // Order of the listeners must not change the answer. + assertTrue(mergeConsumedBounds(Vector(wider, middle))("persistent://t/n/a") == wider("persistent://t/n/a")) + }, + test("orders by MESSAGE ID, not publish time, so a producer clock cannot reorder it") { + // The further-along message carries the EARLIER publish time here, which a producer + // whose clock stepped back produces. Log order is the log's own and cannot invert. + val newerInLog = Map("persistent://t/n/a" -> bounds(40, 40, firstTime = 1000, lastTime = 1000)) + val olderInLog = Map("persistent://t/n/a" -> bounds(3, 3, firstTime = 9999, lastTime = 9999)) + + val merged = mergeConsumedBounds(Vector(olderInLog, newerInLog))("persistent://t/n/a") + assertTrue(merged.first.messageId == msgId(1, 3)) && + assertTrue(merged.last.messageId == msgId(1, 40)) + }, + test("equal-id target ranges use deterministic timestamp endpoints") { + val early = TopicConsumedBounds(TopicCursor(msgId(0, 0), 1000), TopicCursor(msgId(0, 0), 1500)) + val late = TopicConsumedBounds(TopicCursor(msgId(0, 0), 1200), TopicCursor(msgId(0, 0), 2000)) + + val merged = mergeConsumedBounds(Vector(Map("non-persistent://t/n/a" -> late), Map("non-persistent://t/n/a" -> early)))( + "non-persistent://t/n/a" + ) + assertTrue(merged.first.publishTime == 1000) && assertTrue(merged.last.publishTime == 2000) + }, + test("crossing a LEDGER boundary counts as further along") { + val earlierLedger = Map( + "persistent://t/n/a" -> TopicConsumedBounds(TopicCursor(msgId(1, 900), 1000), TopicCursor(msgId(1, 900), 1000)) + ) + val laterLedger = Map( + "persistent://t/n/a" -> TopicConsumedBounds(TopicCursor(msgId(2, 0), 2000), TopicCursor(msgId(2, 0), 2000)) + ) + + val merged = mergeConsumedBounds(Vector(earlierLedger, laterLedger))("persistent://t/n/a") + assertTrue(merged.first.messageId == msgId(1, 900)) && assertTrue(merged.last.messageId == msgId(2, 0)) + }, + test("keeps distinct topics apart rather than collapsing them") { + val one = Map("persistent://t/n/a" -> bounds(3, 3)) + val two = Map("persistent://t/n/b" -> bounds(7, 7)) + + assertTrue(mergeConsumedBounds(Vector(one, two)).size == 2) + }, + test("a session that has read nothing reports no bounds at all") { + assertTrue(mergeConsumedBounds(Vector(Map.empty)).isEmpty) && + assertTrue(mergeConsumedBounds(Vector.empty).isEmpty) + } + ) + ) diff --git a/server/src/test/scala/consumer/consumerSessionAbandonmentTest.scala b/server/src/test/scala/consumer/consumerSessionAbandonmentTest.scala new file mode 100644 index 000000000..935315d68 --- /dev/null +++ b/server/src/test/scala/consumer/consumerSessionAbandonmentTest.scala @@ -0,0 +1,439 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import zio.test.* + +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters.* + +/** THE CLIENT THAT VANISHES: what happens to a session when nobody says goodbye. + * + * The browser sends its Delete RPC from `beforeunload`, where delivery is best-effort - so a + * closed tab, a dropped connection or a crashed laptop leaves the session's consumers connected, + * consuming and acknowledging into a dead stream, and (once their namespace is deleted) + * reconnect-looping for the life of the process. A real run demonstrated exactly that: a session + * still fighting for a deleted namespace half an hour after its tab was gone. + * + * Three mechanisms end that, and this suite pins each without a broker: + * - the transport CANCELLATION handler pauses intake the moment the call dies; + * - a repeated Resume COMPLETES the predecessor stream instead of silently starving it; + * - the idle JANITOR stops and removes sessions that have had no live play stream for the TTL + * - and, when every consumer is also DISCONNECTED, on a much shorter leash. + */ +object consumerSessionAbandonmentTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/abandonment" + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val responses = java.util.concurrent.ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + val completed = java.util.concurrent.atomic.AtomicBoolean(false) + val completions = java.util.concurrent.atomic.AtomicInteger(0) + val nextAfterCompleted = java.util.concurrent.atomic.AtomicInteger(0) + override def onNext(value: consumerPb.ResumeResponse): Unit = + if completed.get then nextAfterCompleted.incrementAndGet() + responses.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = + completed.set(true) + completions.incrementAndGet() + () + def received: Vector[consumerPb.ResumeResponse] = responses.asScala.toVector + + /** The shape gRPC actually hands a server-streaming call: a ServerCallStreamObserver whose + * cancellation handler the service must install. `fireCancel` is the transport reporting the + * call dead - it runs the handler synchronously, like grpc-java does on its executor. + * + * @param cancelOnHandlerRegistration + * the call is ALREADY dead when the handler is installed, so the transport runs it right + * there. That pins the one window the service cannot close by ordering alone: the cancel + * callback has to be registered before the per-name lifecycle lock is taken (gRPC only + * allows handler registration early), and the observer is wired into the runner only + * afterwards - so this cancellation runs against a runner it was never wired into. + */ + private final class CancellableServerObserver(cancelOnHandlerRegistration: Boolean = false) + extends io.grpc.stub.ServerCallStreamObserver[consumerPb.ResumeResponse]: + val delegate = RecordingObserver() + @volatile private var cancelHandler: Option[Runnable] = None + @volatile private var cancelledFlag = false + def fireCancel(): Unit = + cancelledFlag = true + cancelHandler.foreach(_.run()) + def hasCancelHandler: Boolean = cancelHandler.isDefined + override def isCancelled: Boolean = cancelledFlag + override def setOnCancelHandler(handler: Runnable): Unit = + cancelHandler = Some(handler) + if cancelOnHandlerRegistration then fireCancel() + override def setOnCloseHandler(handler: Runnable): Unit = () + override def setCompression(compression: String): Unit = () + override def isReady: Boolean = true + override def setOnReadyHandler(handler: Runnable): Unit = () + override def disableAutoInboundFlowControl(): Unit = () + override def request(count: Int): Unit = () + override def setMessageCompression(enable: Boolean): Unit = () + override def onNext(value: consumerPb.ResumeResponse): Unit = delegate.onNext(value) + override def onError(t: Throwable): Unit = delegate.onError(t) + override def onCompleted(): Unit = delegate.onCompleted() + + /** A consumer that records every client call by name; async methods answer a completed future + * so a stop() walking the full release path never trips over the proxy. `connected` is the + * broker's side of the story - what `isConnected` answers the janitor - and the tests flip + * it to play a broker outage or a deleted topic without a broker. Answered directly, not + * logged: it is a poll, not a lifecycle command. */ + private final class RecordingConsumer: + private val callLog = java.util.concurrent.ConcurrentLinkedQueue[String]() + def calls: Vector[String] = callLog.asScala.toVector + @volatile var connected: Boolean = true + val consumer: org.apache.pulsar.client.api.Consumer[Array[Byte]] = + val handler = new java.lang.reflect.InvocationHandler: + override def invoke(proxy: Object, method: java.lang.reflect.Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "isConnected" => java.lang.Boolean.valueOf(connected) + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => "recording-consumer" + case name => + callLog.add(name) + if method.getReturnType == classOf[java.util.concurrent.CompletableFuture[?]] + then java.util.concurrent.CompletableFuture.completedFuture(null) + else null + java.lang.reflect.Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[org.apache.pulsar.client.api.Consumer[Array[Byte]]]), handler) + .asInstanceOf[org.apache.pulsar.client.api.Consumer[Array[Byte]]] + + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetRunner( + consumerListener: ConsumerListener, + consumers: Map[String, org.apache.pulsar.client.api.Consumer[Array[Byte]]] = Map.empty + ): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(topicFqn), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(sessionName: String, target: ConsumerSessionTargetRunner): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + private val ttlNanos = TimeUnit.MINUTES.toNanos(10) + private def wellPastTtl: Long = System.nanoTime() + ttlNanos + TimeUnit.MINUTES.toNanos(1) + + private def serviceWith(entries: (String, ConsumerSessionRunner)*): (ConsumerServiceImpl, ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]) = + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + entries.foreach((name, runner) => sessions.put(name, runner)) + (ConsumerServiceImpl(consumerSessions = sessions, idleSessionTtlNanos = ttlNanos), sessions) + + /** Far enough below [[ttlNanos]] (10 minutes) that several grace windows fit inside the TTL: + * every "reaped at the grace" assertion below is therefore about the SHORT rule, never an + * accidental TTL expiry. */ + private val graceNanos = TimeUnit.MINUTES.toNanos(2) + + private def serviceWithGrace(entries: (String, ConsumerSessionRunner)*): (ConsumerServiceImpl, ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]) = + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + entries.foreach((name, runner) => sessions.put(name, runner)) + (ConsumerServiceImpl(consumerSessions = sessions, idleSessionTtlNanos = ttlNanos, disconnectedSessionGraceNanos = graceNanos), sessions) + + def spec = suite(this.getClass.toString)( + test("THE VANISHED CLIENT: cancellation pauses intake at once, and the janitor reaps the session after the TTL") { + val recording = RecordingConsumer() + val runner = session("cs-vanished", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWith("cs-vanished" -> runner) + val observer = CancellableServerObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-vanished", includeConsumerStats = true), observer) + val handlerInstalled = observer.hasCancelHandler + val reapableWhileLive = runner.reapableSinceNanos + val pausesBeforeCancel = recording.calls.count(_ == "pause") + + observer.fireCancel() // the tab closed; the Delete RPC never arrived + + val pausedByCancellation = recording.calls.count(_ == "pause") > pausesBeforeCancel + val idleClockStarted = runner.reapableSinceNanos.isDefined + val reaped = service.reapIdleSessions(wellPastTtl) + + assertTrue( + handlerInstalled, + reapableWhileLive.isEmpty, + pausedByCancellation, + idleClockStarted, + reaped == Vector("cs-vanished"), + !sessions.containsKey("cs-vanished"), + runner.isStreamCompleted + ) ?? s"calls=${recording.calls} reaped=$reaped" + }, + test("A CANCELLATION THAT WINS THE WIRING WINDOW WIRES NOTHING, RESUMES NOTHING, AND LEAVES THE SESSION REAPABLE") { + // The narrow window the previous test cannot reach: it cancels only after a synchronous + // Resume has RETURNED, so the observer is wired and the compare-and-clear inside the + // runner does its job. Here the call dies between callback registration and wiring - + // `onPlayStreamCancelled` finds no matching observer, answers false, and does nothing. + // + // What must NOT follow is the resume wiring an already-dead stream: that opens intake + // and resumes the consumers (every message then fails into a stream nobody holds and is + // nacked forever) and clears the idle clock, so the janitor can never see the session as + // abandoned and its subscription pins broker resources for the life of the process. + val recording = RecordingConsumer() + // Gate CLOSED, like a session that has never been played - `listener()` opens it. + val gateClosed = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val runner = session("cs-cancel-race", targetRunner(gateClosed, consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWith("cs-cancel-race" -> runner) + val observer = CancellableServerObserver(cancelOnHandlerRegistration = true) + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-cancel-race", includeConsumerStats = true), observer) + + val intakeOpened = gateClosed.isAcceptingNewMessages + val consumersResumed = recording.calls.contains("resume") + val idleClockRunning = runner.reapableSinceNanos.isDefined + val reaped = service.reapIdleSessions(wellPastTtl) + + assertTrue( + !intakeOpened, + !consumersResumed, + idleClockRunning, + reaped == Vector("cs-cancel-race"), + !sessions.containsKey("cs-cancel-race") + ) ?? (s"intakeOpened=$intakeOpened consumersResumed=$consumersResumed " + + s"idleClockRunning=$idleClockRunning reaped=$reaped calls=${recording.calls}") + }, + test("A SECOND PLAY COMPLETES THE FIRST STREAM instead of starving it, and pushes reach only the successor") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(2) + val runner = session("cs-two-plays", targetRunner(l)) + val (service, _) = serviceWith("cs-two-plays" -> runner) + val first = RecordingObserver() + val second = RecordingObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-two-plays", includeConsumerStats = true), first) + service.resume(consumerPb.ResumeRequest(consumerName = "cs-two-plays", includeConsumerStats = true), second) + (1 to 2).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue( + first.completed.get, + first.completions.get == 1, + first.received.isEmpty, + first.nextAfterCompleted.get == 0, + !second.completed.get, + second.received.nonEmpty + ) ?? s"first=${first.received.size} frames, second=${second.received.size} frames" + }, + test("a LATE cancellation of the replaced stream is stale news - the successor's play is untouched") { + val recording = RecordingConsumer() + val runner = session("cs-late-cancel", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, _) = serviceWith("cs-late-cancel" -> runner) + val first = CancellableServerObserver() + val second = CancellableServerObserver() + + service.resume(consumerPb.ResumeRequest(consumerName = "cs-late-cancel", includeConsumerStats = true), first) + service.resume(consumerPb.ResumeRequest(consumerName = "cs-late-cancel", includeConsumerStats = true), second) + first.fireCancel() // the old tab's stream dies AFTER the new play was wired + + assertTrue( + first.delegate.completed.get, // said goodbye by the second resume, not left hanging + !recording.calls.contains("pause"), + runner.reapableSinceNanos.isEmpty + ) ?? s"calls=${recording.calls}" + }, + test("a send built for a REPLACED observer refuses loudly instead of vanishing into an ack") { + // The in-flight window of a second Play: an old handler closure still holds the old + // observer while the new one is being wired. Writing to it could silently disappear + // (grpc-java drops writes to a cancelled call once a cancel handler is installed), + // and the caller would then ACKNOWLEDGE a message no client received. The send must + // throw instead, so the caller's failure path hands the message back for redelivery. + val runner = session("cs-stale-send", targetRunner(listener())) + val (service, _) = serviceWith("cs-stale-send" -> runner) + val first = RecordingObserver() + val second = RecordingObserver() + service.resume(consumerPb.ResumeRequest(consumerName = "cs-stale-send", includeConsumerStats = true), first) + service.resume(consumerPb.ResumeRequest(consumerName = "cs-stale-send", includeConsumerStats = true), second) + + val staleRefused = scala.util.Try(runner.sendResponse(first, Seq(consumerPb.Message()), Vector.empty)).isFailure + runner.sendResponse(second, Seq(consumerPb.Message()), Vector.empty) + + assertTrue( + staleRefused, + first.nextAfterCompleted.get == 0, // nothing leaked into the finished stream + second.received.nonEmpty + ) ?? s"staleRefused=$staleRefused second=${second.received.size}" + }, + test("the janitor reaps the never-resumed and the terminal, and leaves the live stream alone") { + // Three fates in one map: a session created and forgotten (its idle clock started at + // construction), one whose stream ended terminally, and one somebody is WATCHING. + val abandoned = session("cs-never-resumed", targetRunner(listener())) + val failed = session("cs-terminal", targetRunner(listener())) + val live = session("cs-live", targetRunner(listener())) + val (service, sessions) = serviceWith("cs-never-resumed" -> abandoned, "cs-terminal" -> failed, "cs-live" -> live) + + failed.failAndComplete(com.google.rpc.status.Status(code = com.google.rpc.code.Code.FAILED_PRECONDITION.value, message = "boom")) + service.resume(consumerPb.ResumeRequest(consumerName = "cs-live", includeConsumerStats = true), RecordingObserver()) + + val reaped = service.reapIdleSessions(wellPastTtl) + + assertTrue( + reaped.toSet == Set("cs-never-resumed", "cs-terminal"), + sessions.keySet.asScala.toSet == Set("cs-live"), + !live.isStreamCompleted + ) ?? s"reaped=$reaped remaining=${sessions.keySet.asScala}" + }, + test("a session younger than the TTL is not touched, however idle it is right now") { + val runner = session("cs-fresh", targetRunner(listener())) + val (service, sessions) = serviceWith("cs-fresh" -> runner) + + val reaped = service.reapIdleSessions(System.nanoTime()) // now: idle for ~0 of the TTL + + assertTrue(reaped.isEmpty, sessions.containsKey("cs-fresh"), !runner.isStreamCompleted) + }, + test("DISCONNECTED AND UNWATCHED: reaped after the short grace, not after the hour") { + // The defect the short leash exists for: the tab is gone (no live stream) and the + // broker connection is gone too - in the worst case because the topics themselves + // were force-deleted, where the consumers reconnect-loop and can NEVER recover. + // Waiting out the full TTL is an hour of retry spam for a session nobody can see. + val recording = RecordingConsumer() + recording.connected = false + val runner = session("cs-dead-weight", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWithGrace("cs-dead-weight" -> runner) + + val t0 = System.nanoTime() + val firstSighting = service.reapIdleSessions(t0) + val afterGrace = service.reapIdleSessions(t0 + graceNanos) + val afterTtl = service.reapIdleSessions(wellPastTtl) + + assertTrue( + firstSighting.isEmpty, // one sighting is a blip until it is seen to HOLD + afterGrace == Vector("cs-dead-weight"), // reaped in minutes, not at the TTL + afterTtl.isEmpty, // long gone by the time the TTL would have said so + !sessions.containsKey("cs-dead-weight"), + runner.isStreamCompleted + ) ?? s"firstSighting=$firstSighting afterGrace=$afterGrace afterTtl=$afterTtl" + }, + test("PAUSED BUT CONNECTED: the user in a meeting keeps the whole hour, not the short grace") { + // No live play stream - the ordinary UI pause cancels its stream - but the consumers + // are still CONNECTED: nothing says this client is gone rather than merely quiet. + // The full TTL is that user's contract, and the short leash must not shorten it. + val recording = RecordingConsumer() // connected, like any healthy paused session + val runner = session("cs-meeting", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWithGrace("cs-meeting" -> runner) + + val t0 = System.nanoTime() + val withinTtl = Vector(t0, t0 + graceNanos, t0 + 4 * graceNanos).flatMap(service.reapIdleSessions) + val stillThere = sessions.containsKey("cs-meeting") + val afterTtl = service.reapIdleSessions(wellPastTtl) + + assertTrue( + withinTtl.isEmpty, // sweeps at, and well past, the grace: never reaped early + stillThere, + afterTtl == Vector("cs-meeting") // the 1-hour TTL still bounds the forever case + ) ?? s"withinTtl=$withinTtl afterTtl=$afterTtl" + }, + test("A BROKER BLIP WHILE THE USER WATCHES: the live stream protects the session, however long the disconnect") { + // Every consumer down - a broker restart, say - but somebody holds the play stream. + // The no-live-stream condition fails, so disconnection alone must never reap: when + // the broker returns the consumers reconnect, and the watching user never noticed. + val recording = RecordingConsumer() + recording.connected = false + val runner = session("cs-watched-blip", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWithGrace("cs-watched-blip" -> runner) + service.resume(consumerPb.ResumeRequest(consumerName = "cs-watched-blip", includeConsumerStats = true), RecordingObserver()) + + val t0 = System.nanoTime() + val sweeps = Vector(t0, t0 + graceNanos, wellPastTtl).flatMap(service.reapIdleSessions) + + assertTrue( + sweeps.isEmpty, + sessions.containsKey("cs-watched-blip"), + !runner.isStreamCompleted + ) ?? s"sweeps=$sweeps" + }, + test("ONE CONSUMER STILL CONNECTED: a partially disconnected session is not for the short leash") { + // Half the partitions lost their broker, half are fine - a session doing real work + // through a partial outage. EVERY consumer must be gone before the early rule may + // conclude there is nothing left to lose. + val dropped = RecordingConsumer() + dropped.connected = false + val healthy = RecordingConsumer() + val runner = session( + "cs-half-connected", + targetRunner(listener(), consumers = Map(s"$topicFqn-p0" -> dropped.consumer, s"$topicFqn-p1" -> healthy.consumer)) + ) + val (service, sessions) = serviceWithGrace("cs-half-connected" -> runner) + + val t0 = System.nanoTime() + val withinTtl = Vector(t0, t0 + graceNanos, t0 + 4 * graceNanos).flatMap(service.reapIdleSessions) + val afterTtl = service.reapIdleSessions(wellPastTtl) + + assertTrue( + withinTtl.isEmpty, // never reaped early while one consumer still holds on + afterTtl == Vector("cs-half-connected") // the TTL remains its only bound + ) ?? s"withinTtl=$withinTtl afterTtl=$afterTtl" + }, + test("FLAPPING CONNECTIVITY: a reconnect restarts the window - blips never accumulate into a reap") { + // The sustained-window requirement, pinned: disconnected at one sweep, back at the + // next, gone again at the third. Each recovery clears the clock, so only a + // disconnection that HOLDS across the whole window reaps. + val recording = RecordingConsumer() + recording.connected = false + val runner = session("cs-flapping", targetRunner(listener(), consumers = Map(topicFqn -> recording.consumer))) + val (service, sessions) = serviceWithGrace("cs-flapping" -> runner) + + val t0 = System.nanoTime() + val down = service.reapIdleSessions(t0) + recording.connected = true // the broker came back between sweeps + val recovered = service.reapIdleSessions(t0 + graceNanos) + recording.connected = false // and went away again + val downAgain = service.reapIdleSessions(t0 + 2 * graceNanos) + val sustained = service.reapIdleSessions(t0 + 3 * graceNanos) + + assertTrue( + down.isEmpty, + recovered.isEmpty, + downAgain.isEmpty, // the sighting before the recovery is FORGOTTEN - a fresh window starts here + sustained == Vector("cs-flapping"), // held for the window: now it is dead weight + !sessions.containsKey("cs-flapping") + ) ?? s"down=$down recovered=$recovered downAgain=$downAgain sustained=$sustained" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/consumerSessionAdmissionTest.scala b/server/src/test/scala/consumer/consumerSessionAdmissionTest.scala new file mode 100644 index 000000000..39da6ab30 --- /dev/null +++ b/server/src/test/scala/consumer/consumerSessionAdmissionTest.scala @@ -0,0 +1,339 @@ +package consumer + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_runner.* +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import zio.test.* + +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} +import scala.jdk.CollectionConverters.* + +/** ADMISSION: the caps that keep one browser from turning the server into a load test. + * + * The per-target topic cap alone was a fence with two open gates: the UI appends targets without + * limit, so N targets multiplied the bound away, and nothing bounded how many sessions exist at + * all - every abandoned tab used to add one forever. Three bounds close that, and each refusal + * has to say WHY and what to do instead, because the person hitting it is mid-investigation: + * + * - enabled targets per session (checked on the config alone, before any broker work); + * - physical streams per session, summed across targets (checked once the selectors resolved); + * - active sessions per server (checked at create, where a REPLACEMENT must stay admissible - + * the browser re-creates on every configuration change, so refusing it at the cap would + * strand the ordinary flow). + */ +object consumerSessionAdmissionTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/admission" + + private def targetConfig: ConsumerSessionTarget = + ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def sessionConfig(targets: Vector[ConsumerSessionTarget]): ConsumerSessionConfig = + ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = targets, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + /** A runner that never touches a broker - what the injected builder hands the service. */ + private def dummyRunner(name: String): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = name, + sessionConfig = sessionConfig(Vector.empty), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map.empty + ) + + private def createRequest(name: String): consumerPb.CreateConsumerRequest = + consumerPb.CreateConsumerRequest(consumerName = name, consumerSessionConfig = Some(consumerPb.ConsumerSessionConfig())) + + private def create(service: ConsumerServiceImpl, name: String): consumerPb.CreateConsumerResponse = + Await.result(service.createConsumer(createRequest(name)), Duration(60, SECONDS)) + + private def worker(name: String)(body: => Unit): Thread = + val thread = new Thread((() => body): Runnable, name) + thread.setDaemon(true) + thread + + /** Which top-level fields the encoded request actually carries. Setting an enum to zero in + * memory proves nothing about what an older client puts on the wire; this reads the tags back + * off the bytes, so "the field is absent" is a statement about the encoding. */ + private def wireFieldNumbers(bytes: Array[Byte]): Set[Int] = + val in = com.google.protobuf.CodedInputStream.newInstance(bytes) + val numbers = Set.newBuilder[Int] + var tag = in.readTag() + while tag != 0 do + numbers += (tag >>> 3) + in.skipField(tag) + tag = in.readTag() + numbers.result() + + def spec = suite(this.getClass.toString)( + test("the enabled-target bound sits exactly at the cap, and the refusal names both numbers") { + val atCap = ConsumerSessionRunner.enabledTargetCountRejectionReason("cs-adm", ConsumerSessionRunner.maxEnabledTargetsPerSession) + val pastCap = ConsumerSessionRunner.enabledTargetCountRejectionReason("cs-adm", ConsumerSessionRunner.maxEnabledTargetsPerSession + 1) + assertTrue( + atCap.isEmpty, + pastCap.exists(_.contains(s"${ConsumerSessionRunner.maxEnabledTargetsPerSession + 1} enabled targets")), + pastCap.exists(_.contains(ConsumerSessionRunner.maxEnabledTargetsPerSession.toString)) + ) ?? s"pastCap=$pastCap" + }, + test("the stream-total bound counts EVERY target's streams - two targets on one topic cost two") { + val atCap = ConsumerSessionRunner.sessionStreamTotalRejectionReason("cs-adm", Vector(1_000, 1_000)) + val pastCap = ConsumerSessionRunner.sessionStreamTotalRejectionReason("cs-adm", Vector(1_000, 1_000, 1)) + assertTrue( + atCap.isEmpty, + pastCap.exists(_.contains("2001 physical topic streams")), + pastCap.exists(_.contains("1000 + 1000 + 1")) + ) ?? s"pastCap=$pastCap" + }, + test("Best effort orders nothing on one stream; Guaranteed builds its replay layer at any width") { + // FLIPPED PIN (owner decision 2026-08-09, the exact-replay redesign): Guaranteed used + // to be a no-op on a single stream - one log is already in order. The replay + // boundary, the auto-pause and the caught-up signal live in the ordering layer, so + // Guaranteed now builds one at ANY stream count; Best effort keeps the single-stream + // free pass, and Fastest never builds one. + assertTrue( + ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.Guaranteed, 1), + !ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.BestEffort, 1), + ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.Guaranteed, 2), + ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.BestEffort, 2), + !ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.AsReceived, 2), + !ConsumerSessionRunner.needsDeliveryOrderLayer(MessageDeliveryOrder.AsReceived, 1) + ) + }, + test("an absent delivery order resolves to Guaranteed, the product default, while explicit choices stay explicit") { + // An ABSENT enum is proto3 zero - every older client and every config saved before the + // field existed. Owner decision (2026-08-11, direct instruction): the default is + // Guaranteed - the third move of this default (the plan file's decision log is the + // record), superseding the 2026-08-09 Best effort default. A session that never named + // an order replays recorded history exactly and auto-pauses when caught up. Explicit + // Best effort and explicit Fastest are choices and are never rewritten. + assertTrue( + MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED) == MessageDeliveryOrder.Guaranteed, + MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED) == MessageDeliveryOrder.AsReceived, + MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) == MessageDeliveryOrder.BestEffort, + MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED) == MessageDeliveryOrder.Guaranteed, + MessageDeliveryOrder.fromPb(consumerPb.MessageDeliveryOrder.Unrecognized(99)) == MessageDeliveryOrder.Guaranteed, + ConsumerSessionConfig.fromPb(consumerPb.ConsumerSessionConfig()).messageDeliveryOrder == MessageDeliveryOrder.Guaranteed, + sessionConfig(Vector.empty).messageDeliveryOrder == MessageDeliveryOrder.Guaranteed + ) + }, + test("a REQUEST whose delivery-order enum is zero on the wire resolves to Guaranteed, the default") { + // The request-side twin of the pre-field managed item: bytes an older client actually + // sends. Proto3 omits a zero enum entirely, so the field is absent on the wire - and + // the decoded request must still run the product default. + val requestBytes = consumerPb.ConsumerSessionConfig() + .withMessageDeliveryOrder(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED) + .toByteArray + val decoded = consumerPb.ConsumerSessionConfig.parseFrom(requestBytes) + + assertTrue( + // Field 8 in ConsumerSessionConfig; a protobuf tag is (fieldNumber << 3) | wireType. + !wireFieldNumbers(requestBytes).contains(8), + decoded.messageDeliveryOrder == consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED, + ConsumerSessionConfig.fromPb(decoded).messageDeliveryOrder == MessageDeliveryOrder.Guaranteed + ) ?? s"wireFields=${wireFieldNumbers(requestBytes)} decoded=${ConsumerSessionConfig.fromPb(decoded).messageDeliveryOrder}" + }, + test("a config past the target cap is refused BEFORE any broker work - the clients are never touched") { + // Null clients are the proof: reaching either would blow up with an NPE instead of the + // admission message. Disabled targets do not count against the cap. + val overloaded = sessionConfig( + Vector.fill(ConsumerSessionRunner.maxEnabledTargetsPerSession + 1)(targetConfig) + ++ Vector(targetConfig.copy(isEnabled = false)) + ) + val outcome = scala.util.Try( + ConsumerSessionRunner.make(pulsarClient = null, adminClient = null, sessionName = "cs-adm-wide", sessionConfig = overloaded) + ) + assertTrue( + outcome.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + outcome.failed.toOption.exists(_.getMessage.contains(s"${ConsumerSessionRunner.maxEnabledTargetsPerSession + 1} enabled targets")) + ) ?? s"outcome=$outcome" + }, + test("the session cap refuses a NEW name, keeps the map clean, and still admits a replacement") { + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val service = ConsumerServiceImpl( + consumerSessions = sessions, + makeSession = (name, _) => dummyRunner(name), + maxActiveSessions = 1 + ) + + val first = Await.result(service.createConsumer(createRequest("cs-adm-a")), Duration(10, SECONDS)) + val refused = Await.result(service.createConsumer(createRequest("cs-adm-b")), Duration(10, SECONDS)) + val replaced = Await.result(service.createConsumer(createRequest("cs-adm-a")), Duration(10, SECONDS)) + + assertTrue( + first.getStatus.code == com.google.rpc.code.Code.OK.value, + refused.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + refused.getStatus.message.contains("limit"), + !sessions.containsKey("cs-adm-b"), + replaced.getStatus.code == com.google.rpc.code.Code.OK.value, + sessions.size == 1 + ) ?? s"first=${first.getStatus} refused=${refused.getStatus} replaced=${replaced.getStatus}" + }, + test("MANY DISTINCT-NAME CREATES STARTED AT cap - 1 ADMIT EXACTLY ONE: the permit covers the BUILD") { + // THE admission defect. The count was read under a lock scoped to ONE session name and + // the expensive build then ran BEFORE the insertion, so every concurrent create of a + // DIFFERENT name saw the same pre-build size and every one of them passed. At an empty + // map an arbitrary number could pass a 100-session check, each of them free to + // subscribe up to 2,000 streams - the documented cap protected nothing at all. + // + // Pinned by latches, never by timing: every builder parks on `release`, so all the + // builds this cap admits are simultaneously in flight when the peak is read, and every + // racer has settled (refused, or parked in its build) before the peak is read at all. + val cap = 4 + val racers = 8 + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + (1 until cap).foreach(i => sessions.put(s"cs-adm-seed-$i", dummyRunner(s"cs-adm-seed-$i"))) + + val settled = CountDownLatch(racers) + val release = CountDownLatch(1) + val buildsInFlight = AtomicInteger(0) + val peakAdmitted = AtomicInteger(0) + + val service = ConsumerServiceImpl( + consumerSessions = sessions, + makeSession = (name, _) => + val inFlight = buildsInFlight.incrementAndGet() + // INSTALLED PLUS RESERVED, which is what the cap has to bound: a build in + // flight already owns the consumers, threads and heap the limit is about. + peakAdmitted.updateAndGet(previous => math.max(previous, sessions.size + inFlight)) + settled.countDown() + release.await(60, TimeUnit.SECONDS) + buildsInFlight.decrementAndGet() + dummyRunner(name), + maxActiveSessions = cap + ) + + val responses = ConcurrentLinkedQueue[com.google.rpc.status.Status]() + val threads = (1 to racers).map(i => + worker(s"cs-adm-race-$i") { + val response = create(service, s"cs-adm-race-$i") + responses.add(response.getStatus) + // A REFUSED create never reaches the builder, so it settles here instead. + if response.getStatus.code != com.google.rpc.code.Code.OK.value then settled.countDown() + } + ) + threads.foreach(_.start()) + val everyRacerSettled = settled.await(60, TimeUnit.SECONDS) + val peakWhileBuilding = peakAdmitted.get + release.countDown() + threads.foreach(_.join(60_000)) + + val statuses = responses.asScala.toVector + assertTrue( + everyRacerSettled, + peakWhileBuilding <= cap, + statuses.count(_.code == com.google.rpc.code.Code.OK.value) == 1, + statuses.count(_.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value) == racers - 1, + statuses.filter(_.code != com.google.rpc.code.Code.OK.value).forall(_.message.contains("limit")), + sessions.size == cap, + service.admittedSessionCount == cap + ) ?? (s"peakWhileBuilding=$peakWhileBuilding cap=$cap installed=${sessions.size} " + + s"admitted=${service.admittedSessionCount} statuses=${statuses.map(_.code)}") + }, + test("A REPLACEMENT AT THE CAP HOLDS ITS PREDECESSOR'S PERMIT for the whole build") { + // The other half of the same permit: the browser re-creates a session on every + // configuration change, so a replacement must stay admissible at the cap - and it must + // not do that by leaving the freed slot lying around for a DIFFERENT name to take + // while the replacement is still building. + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + sessions.put("cs-adm-replace", dummyRunner("cs-adm-replace")) + + val building = CountDownLatch(1) + val release = CountDownLatch(1) + val service = ConsumerServiceImpl( + consumerSessions = sessions, + makeSession = (name, _) => + if name == "cs-adm-replace" then + building.countDown() + release.await(60, TimeUnit.SECONDS) + dummyRunner(name), + maxActiveSessions = 1 + ) + + val replacing = worker("cs-adm-replacing")(create(service, "cs-adm-replace")) + replacing.start() + val replacementIsBuilding = building.await(60, TimeUnit.SECONDS) + + // The predecessor is gone from the map by now, so the ONLY thing standing between this + // create and a second session is the permit the replacement is holding across its build. + val intruder = create(service, "cs-adm-intruder") + val admittedDuringTheBuild = service.admittedSessionCount + + release.countDown() + replacing.join(60_000) + + assertTrue( + replacementIsBuilding, + intruder.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + intruder.getStatus.message.contains("limit"), + !sessions.containsKey("cs-adm-intruder"), + admittedDuringTheBuild == 1, + sessions.size == 1, + sessions.containsKey("cs-adm-replace"), + service.admittedSessionCount == 1 + ) ?? (s"intruder=${intruder.getStatus} admittedDuringTheBuild=$admittedDuringTheBuild " + + s"installed=${sessions.keySet.asScala.toVector.sorted} admitted=${service.admittedSessionCount}") + }, + test("a build that FAILS gives its permit back, and so does a delete") { + // A permit that leaked on the failure path would be worse than no permit at all: the + // server would refuse sessions it is not running, and only a restart would clear it. + val sessions = new ConcurrentHashMap[ConsumerSessionName, ConsumerSessionRunner]() + val fail = java.util.concurrent.atomic.AtomicBoolean(true) + val service = ConsumerServiceImpl( + consumerSessions = sessions, + makeSession = (name, _) => + if fail.get then throw new RuntimeException("the broker refused every consumer") + else dummyRunner(name), + maxActiveSessions = 1 + ) + + val failed = create(service, "cs-adm-doomed") + val admittedAfterTheFailure = service.admittedSessionCount + fail.set(false) + val afterFailure = create(service, "cs-adm-after-failure") + val deleted = Await.result( + service.deleteConsumer(consumerPb.DeleteConsumerRequest(consumerName = "cs-adm-after-failure")), + Duration(60, SECONDS) + ) + val afterDelete = create(service, "cs-adm-after-delete") + + assertTrue( + failed.getStatus.code == com.google.rpc.code.Code.FAILED_PRECONDITION.value, + admittedAfterTheFailure == 0, + afterFailure.getStatus.code == com.google.rpc.code.Code.OK.value, + deleted.getStatus.code == com.google.rpc.code.Code.OK.value, + afterDelete.getStatus.code == com.google.rpc.code.Code.OK.value, + service.admittedSessionCount == 1 + ) ?? (s"failed=${failed.getStatus} admittedAfterTheFailure=$admittedAfterTheFailure " + + s"afterFailure=${afterFailure.getStatus} afterDelete=${afterDelete.getStatus}") + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/convertersTest.scala b/server/src/test/scala/consumer/convertersTest.scala index 3b31a20cd..d03449515 100644 --- a/server/src/test/scala/consumer/convertersTest.scala +++ b/server/src/test/scala/consumer/convertersTest.scala @@ -41,6 +41,9 @@ val useLatestTopicSchemaDeserializer = Deserializer(deserializer = UseLatestTopi val treatBytesAsJsonDeserializer = Deserializer(deserializer = TreatBytesAsJson()) object convertersTest extends ZIOSpecDefault: + /* Renders a byte array as hex so a failing table case is identifiable in the report. */ + private def hex(bytes: Array[Byte]): String = bytes.map(b => f"0x$b%02x").mkString("[", " ", "]") + def spec = suite(s"${this.getClass.toString} - messageValueToJson()")( test("AVRO schema") { val avroSchemaDefinition = """ @@ -72,33 +75,33 @@ object convertersTest extends ZIOSpecDefault: .build val jsonToEncode = """{"name":"Alyssa","favorite_number":256}""" - val avroPayload = avro.converters.fromJson( + + avro.converters.fromJson( avroSchemaDefinition.getBytes, jsonToEncode.getBytes ) match - case Right(value) => value - case Left(error) => throw error - - val avroSchema = AvroSchema.of(schemaDefinition) - - val topicName = "topic-a" - val schemaVersion = 1L; - val messageMetadata = new MessageMetadata().setSchemaVersion(scala.math.BigInt(schemaVersion).toByteArray) - val message = MessageImpl.create[Array[Byte]]( - messageMetadata, - java.nio.ByteBuffer.wrap(avroPayload), - avroSchema, - topicName - ) - - val schemasByVersion: SchemasByVersion = Map(1L -> schemaInfo) - val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - - val decodedJson = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err - - assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) + case Left(error) => assertNever(s"failed to encode the AVRO test payload: $error") + case Right(avroPayload) => + val avroSchema = AvroSchema.of(schemaDefinition) + + val topicName = "topic-a" + val schemaVersion = 1L; + val messageMetadata = new MessageMetadata().setSchemaVersion(scala.math.BigInt(schemaVersion).toByteArray) + val message = MessageImpl.create[Array[Byte]]( + messageMetadata, + java.nio.ByteBuffer.wrap(avroPayload), + avroSchema, + topicName + ) + + val schemasByVersion: SchemasByVersion = Map(1L -> schemaInfo) + val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) + + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"messageValueToJson failed for the AVRO message: $err") + case Right(decodedJson) => + assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) ?? + s"decoded $decodedJson, expected $jsonToEncode" }, test("JSON schema") { val avroSchemaDefinition = @@ -148,11 +151,75 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val decodedJson = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"messageValueToJson failed for the JSON-schema message: $err") + case Right(decodedJson) => + assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) ?? + s"decoded $decodedJson, expected $jsonToEncode" + }, + test("a message that names NO schema version is read with the topic's LATEST schema") { + // THE CASE THIS DESERIALIZER IS NAMED FOR. A producer using Schema.BYTES (or an older + // client) stamps no schema version, so nothing but the topic's registry says what the + // bytes are. The old code answered `bytesToJsonString` here, which renders a JSON + // DOCUMENT as an escaped JSON STRING - `{\"id\":1,...}` instead of the object - and + // `SchemasByVersion.getLatest`, written for exactly this, was never called from + // anywhere. Confirmed end to end by e2e CS-SCH-1 (schema registered through the admin + // API, payload produced with Schema.BYTES). + val schemaDefinition = + """ + |{ + | "type": "record", + | "name": "TestOrder", + | "fields": [ + | { "name": "id", "type": "int" }, + | { "name": "item", "type": "string" } + | ] + |} + """.stripMargin + + val schemaInfo = SchemaInfo.builder + .`type`(SchemaType.JSON) + .schema(schemaDefinition.getBytes) + .build - assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) + val jsonToEncode = """{"id":1,"item":"boots"}""" + val topicName = "topic-a" + + // No setSchemaVersion: this is what a schema-less producer actually writes. + val messageMetadata = new MessageMetadata() + val message = MessageImpl.create[Array[Byte]]( + messageMetadata, + java.nio.ByteBuffer.wrap(jsonToEncode.getBytes), + Schema.BYTES, + topicName + ) + + val schemasByTopic: SchemasByTopic = Map(topicName -> Map(1L -> schemaInfo)) + + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"messageValueToJson failed for the version-less message: $err") + case Right(decodedJson) => + assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) ?? + s"decoded $decodedJson, expected the OBJECT $jsonToEncode (an escaped string means the raw-bytes fallback ran)" + }, + test("a version-less message on a topic with NO registered schema still reads as raw bytes") { + // The other side of the same branch: inferring a schema is only possible when the + // registry holds one. With nothing to infer from, the raw-string answer is correct + // and must not change. + val payload = """{"id":1}""" + val message = MessageImpl.create[Array[Byte]]( + new MessageMetadata(), + java.nio.ByteBuffer.wrap(payload.getBytes), + Schema.BYTES, + "topic-without-schema" + ) + + converters.messageValueToJson(Map.empty, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"messageValueToJson failed for the schema-less topic: $err") + case Right(decodedJson) => + // A JSON string literal - the document as text, quoted and escaped. + assertTrue(parseJson(decodedJson).toOption.exists(_.isString)) ?? + s"decoded $decodedJson, expected the raw payload as a JSON string" }, test("PROTOBUF_NATIVE schema") { val protoFileName = "user.proto" @@ -169,48 +236,50 @@ object convertersTest extends ZIOSpecDefault: """.stripMargin val compiledFiles = protobufnative.compiler.compileFiles(List(FileEntry(protoFileName, protoFileContent))) - val protoSchemaDefinition = compiledFiles.files.get(protoFileName) match + val compiledUserSchema: Either[String, Array[Byte]] = compiledFiles.files.get(protoFileName) match case Some(Right(file)) => - file.schemas.get("User") match - case Some(schema) => schema.rawSchema - case _ => throw new Exception(s"Failed to compile PROTOBUF_NATIVE message") - case _ => throw new Exception(s"Failed to compile PROTOBUF_NATIVE message") - - val schemaInfo = SchemaInfo.builder - .`type`(SchemaType.PROTOBUF_NATIVE) - .schema(protoSchemaDefinition) - .build - - val protoSchema = Schema.getSchema(schemaInfo).asInstanceOf[Schema[Array[Byte]]] - - val jsonToEncode = """{"name":"Alyssa","favorite_number":256}""" - val protoPayload = protobufnative.converters.fromJson(protoSchemaDefinition, jsonToEncode.getBytes) match - case Right(value) => value - case Left(error) => throw error - - val topicName = "topic-a" - val schemaVersion = 1L; - val messageMetadata = new MessageMetadata().setSchemaVersion(scala.math.BigInt(schemaVersion).toByteArray) - val message = MessageImpl.create[Array[Byte]]( - messageMetadata, - java.nio.ByteBuffer.wrap(protoPayload), - protoSchema, - topicName - ) - - val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) - val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - - val decodedJson = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err - - assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) + file.schemas.get("User").map(_.rawSchema).toRight(s"""compiled $protoFileName has no "User" message""") + case Some(Left(error)) => Left(s"failed to compile $protoFileName: $error") + case None => Left(s"the PROTOBUF_NATIVE compiler returned no result for $protoFileName") + + compiledUserSchema match + case Left(reason) => assertNever(reason) + case Right(protoSchemaDefinition) => + val schemaInfo = SchemaInfo.builder + .`type`(SchemaType.PROTOBUF_NATIVE) + .schema(protoSchemaDefinition) + .build + + val protoSchema = Schema.getSchema(schemaInfo).asInstanceOf[Schema[Array[Byte]]] + + val jsonToEncode = """{"name":"Alyssa","favorite_number":256}""" + + protobufnative.converters.fromJson(protoSchemaDefinition, jsonToEncode.getBytes) match + case Left(error) => assertNever(s"failed to encode the PROTOBUF_NATIVE test payload: $error") + case Right(protoPayload) => + val topicName = "topic-a" + val schemaVersion = 1L; + val messageMetadata = new MessageMetadata().setSchemaVersion(scala.math.BigInt(schemaVersion).toByteArray) + val message = MessageImpl.create[Array[Byte]]( + messageMetadata, + java.nio.ByteBuffer.wrap(protoPayload), + protoSchema, + topicName + ) + + val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) + val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) + + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"messageValueToJson failed for the PROTOBUF_NATIVE message: $err") + case Right(decodedJson) => + assertTrue(parseJson(decodedJson) == parseJson(jsonToEncode)) ?? + s"decoded $decodedJson, expected $jsonToEncode" }, test("BOOLEAN to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.BOOLEAN) .build @@ -228,23 +297,23 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: BOOLEAN payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "false"), TestCase(Array(1), "true") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("INT8 to json") { case class TestCase(messagePayload: Array[Byte], expected: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.INT8) .build @@ -262,11 +331,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: INT8 payload=${hex(testCase.messagePayload)}, expected ${testCase.expected}" - parseJson(json) == parseJson(testCase.expected) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expected)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0"), @@ -276,12 +345,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(-18), "-18") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("INT16 to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.INT16) .build @@ -299,11 +368,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: INT16 payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0"), @@ -315,12 +384,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0x7f, 0xff).map(_.toByte), Short.MaxValue.toString) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("INT32 to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.INT32) .build @@ -338,11 +407,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: INT32 payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0"), @@ -354,12 +423,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0x7f, 0xff, 0xff, 0xff).map(_.toByte), Int.MaxValue.toString) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("INT64 to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.INT64) .build @@ -377,11 +446,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: INT64 payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0"), @@ -393,12 +462,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff).map(_.toByte), s"""${Long.MaxValue.toString}""") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("FLOAT to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.FLOAT) .build @@ -416,11 +485,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: FLOAT payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0.0"), @@ -432,12 +501,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0xc6, 0xea, 0x60, 0x0f).map(_.toByte), "-30000.03") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("DOUBLE to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.DOUBLE) .build @@ -455,11 +524,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: DOUBLE payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(0), "0.0"), @@ -471,12 +540,12 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0xc0, 0xdd, 0x4c, 0x01, 0xeb, 0x85, 0x1e, 0xb8).map(_.toByte), "-30000.03") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, test("STRING to json") { case class TestCase(messagePayload: Array[Byte], expectedJson: String) - def runTestCase(testCase: TestCase): Boolean = + def runTestCase(testCase: TestCase, idx: Int): TestResult = val schemaInfo = SchemaInfo.builder .`type`(SchemaType.STRING) .build @@ -494,11 +563,11 @@ object convertersTest extends ZIOSpecDefault: val schemasByVersion: SchemasByVersion = Map(schemaVersion -> schemaInfo) val schemasByTopic: SchemasByTopic = Map(topicName -> schemasByVersion) - val json = converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match - case Right(value) => value - case Left(err) => throw err + val label = s"case #$idx: STRING payload=${hex(testCase.messagePayload)}, expected ${testCase.expectedJson}" - parseJson(json) == parseJson(testCase.expectedJson) + converters.messageValueToJson(schemasByTopic, message, useLatestTopicSchemaDeserializer) match + case Left(err) => assertNever(s"$label -- messageValueToJson failed: $err") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.expectedJson)) ?? s"$label, actual $json" val testCases = List[TestCase]( TestCase(Array(), "\"\""), @@ -512,6 +581,69 @@ object convertersTest extends ZIOSpecDefault: TestCase(Array(0x71, 0x75, 0x22, 0x6f, 0x74, 0x65, 0x22, 0x73).map(_.toByte), """"qu\"ote\"s"""") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) }, + test("TreatBytesAsJson deserializer - valid JSON payload") { + case class TestCase(payload: String) + + def runTestCase(testCase: TestCase, idx: Int): TestResult = + val topicName = "topic-a" + val message = MessageImpl.create[Array[Byte]]( + new MessageMetadata(), + ByteBuffer.wrap(testCase.payload.getBytes("UTF-8")), + BytesSchema.of.asInstanceOf[Schema[Array[Byte]]], + topicName + ) + + // The deserializer ignores the registered schemas by design - it treats the raw bytes as JSON. + val schemasByTopic: SchemasByTopic = Map.empty + + val label = s"case #$idx: payload=${testCase.payload}" + + converters.messageValueToJson(schemasByTopic, message, treatBytesAsJsonDeserializer) match + case Left(err) => assertNever(s"$label -- expected Right, got Left($err)") + case Right(json) => assertTrue(parseJson(json) == parseJson(testCase.payload)) ?? s"$label, actual $json" + + val testCases = List[TestCase]( + TestCase("""{"name":"Alyssa","favorite_number":256}"""), + TestCase("""{"a":2,"b":{"c":3}}"""), + TestCase("""[1,2,"a"]"""), + TestCase("[]"), + TestCase("null"), + TestCase("true"), + TestCase("-3.0"), + TestCase("\"Gruß\"") + ) + + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) + }, + test("TreatBytesAsJson deserializer - invalid JSON payload") { + case class TestCase(payload: String) + + def runTestCase(testCase: TestCase, idx: Int): TestResult = + val topicName = "topic-a" + val message = MessageImpl.create[Array[Byte]]( + new MessageMetadata(), + ByteBuffer.wrap(testCase.payload.getBytes("UTF-8")), + BytesSchema.of.asInstanceOf[Schema[Array[Byte]]], + topicName + ) + + val schemasByTopic: SchemasByTopic = Map.empty + + val label = s"case #$idx: payload=${testCase.payload}" + val actual = converters.messageValueToJson(schemasByTopic, message, treatBytesAsJsonDeserializer) + + assertTrue(actual.isLeft) ?? s"$label -- expected Left, actual $actual" + + val testCases = List[TestCase]( + TestCase(""), + TestCase("2z"), + TestCase("undefined"), + TestCase("""{a:2,"b":{"c":3}}"""), + TestCase("""{"a":}""") + ) + + testCases.zipWithIndex.map { (testCase, idx) => runTestCase(testCase, idx) }.reduce(_ && _) + } ) diff --git a/server/src/test/scala/consumer/message_filter/JsMessageFilterTest.scala b/server/src/test/scala/consumer/message_filter/JsMessageFilterTest.scala index cfd515752..b032ce898 100644 --- a/server/src/test/scala/consumer/message_filter/JsMessageFilterTest.scala +++ b/server/src/test/scala/consumer/message_filter/JsMessageFilterTest.scala @@ -21,8 +21,10 @@ object JsMessageFilterTest extends ZIOSpecDefault: isShouldFail: Boolean = false ) + // One pool (one GraalVM Engine) for the suite - see BasicMessageFilterTest. + private val sessionContextPool = ConsumerSessionContextPool() + def runTestSpec(spec: TestSpec): Boolean = - val sessionContextPool = ConsumerSessionContextPool() val jsMessageFilter = JsMessageFilter(jsCode = spec.jsCode) val filter = MessageFilter( isEnabled = true, @@ -36,9 +38,14 @@ object JsMessageFilterTest extends ZIOSpecDefault: val sessionContext = sessionContextPool.getNextContext sessionContext.setCurrentMessage(spec.messageAsJsonOmittingValue, Right(spec.messageValueAsJson.trim)) - val result = sessionContext.testMessageFilter(filter = filter).isOk + val result = sessionContext.testMessageFilter(filter = filter) - if spec.isShouldFail then !result else result + // See BasicMessageFilterTest: a thrown JS error must not satisfy an `isShouldFail` case. + if result.error.nonEmpty then + java.lang.System.err.println(s"[js-filter-test] unexpected evaluation error: ${result.error.get}") + false + else if spec.isShouldFail then !result.isOk + else result.isOk def spec = suite(s"${this.getClass.toString}")( test(JsMessageFilter.getClass.toString) { @@ -127,4 +134,4 @@ object JsMessageFilterTest extends ZIOSpecDefault: |""".stripMargin ))) } - ) + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/message_filter/basic_message_filter/BasicMessageFilterTest.scala b/server/src/test/scala/consumer/message_filter/basic_message_filter/BasicMessageFilterTest.scala index ce368032d..9b2e90892 100644 --- a/server/src/test/scala/consumer/message_filter/basic_message_filter/BasicMessageFilterTest.scala +++ b/server/src/test/scala/consumer/message_filter/basic_message_filter/BasicMessageFilterTest.scala @@ -23,8 +23,12 @@ object BasicMessageFilterTest extends ZIOSpecDefault: isShouldFail: Boolean = false ) + // ONE pool (and therefore one GraalVM Engine) for the whole suite: building an Engine per test + // cost ~132 of them. TestResult.error is populated regardless of the debug flag. Safe because + // ZIO Test runs the tests of a suite sequentially. + private val sessionContextPool = ConsumerSessionContextPool(isDebug = false) + def runTestSpec(spec: TestSpec): Boolean = - val sessionContextPool = ConsumerSessionContextPool(isDebug = true) val basicMessageFilter = BasicMessageFilter(op = spec.op) val filter = MessageFilter( isEnabled = true, @@ -38,9 +42,16 @@ object BasicMessageFilterTest extends ZIOSpecDefault: val sessionContext = sessionContextPool.getNextContext sessionContext.setCurrentMessage(spec.messageJsonOmittingValue.toJson, Right(spec.messageValueAsJson.trim)) - val result = sessionContext.testMessageFilter(filter = filter).isOk + val result = sessionContext.testMessageFilter(filter = filter) - if spec.isShouldFail then !result else result + // A filter that THROWS also yields isOk=false (BasicMessageFilter catches Throwable), so + // reading isOk alone made every `isShouldFail` case pass on a crash just as it does on a + // correct rejection. An evaluation error is never an expected outcome here: fail loudly. + if result.error.nonEmpty then + java.lang.System.err.println(s"[filter-test] unexpected evaluation error: ${result.error.get}") + false + else if spec.isShouldFail then !result.isOk + else result.isOk def spec = suite(s"${this.getClass.toString}")( /* @@ -2680,4 +2691,4 @@ object BasicMessageFilterTest extends ZIOSpecDefault: ) ))) } - ) + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/approximateEntryPositionTest.scala b/server/src/test/scala/consumer/session_runner/approximateEntryPositionTest.scala new file mode 100644 index 000000000..1561f9cb2 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/approximateEntryPositionTest.scala @@ -0,0 +1,378 @@ +package consumer.session_runner + +import zio.test.* + +import java.util.concurrent.atomic.AtomicInteger +import scala.util.Try + +/** Entry-fraction rounding, endpoint, validation, deduplication and lookup-budget coverage. An + * entry may contain a batch, so these tests deliberately make no message-percent claim. */ +object approximateEntryPositionTest extends ZIOSpecDefault: + + import ApproximateEntrySeek.* + + private def rejected(fraction: Double, numberOfEntries: Long = 100): Option[String] = + Try(resolveApproximateEntryPosition(fraction, numberOfEntries)).failed.toOption.map(_.getMessage) + + private val endpointsSuite = suite("endpoints")( + test("0.0 is the earliest retained message, not entry 1 by ordinal") { + // Both would deliver the same first message, but only MessageId.earliest keeps working + // when the entry count the ordinal was computed from is already stale. + assertTrue(resolveApproximateEntryPosition(0.0, 100) == Earliest) + }, + test("1.0 is the latest position, exactly as the 'Latest message' mode") { + // VERIFIED: a seek to MessageId.latest delivered nothing from a 100-message topic - the + // session shows only what is published from now on. 1.0 must mean that, not "the last + // entry". + assertTrue(resolveApproximateEntryPosition(1.0, 100) == Latest) + }, + test("the endpoints are exact whatever the log size, including a log of one entry") { + assertTrue( + resolveApproximateEntryPosition(0.0, 1) == Earliest, + resolveApproximateEntryPosition(1.0, 1) == Latest, + resolveApproximateEntryPosition(0.0, 1_000_000_000L) == Earliest, + resolveApproximateEntryPosition(1.0, 1_000_000_000L) == Latest + ) + }, + test("negative zero is still the earliest end") { + assertTrue(resolveApproximateEntryPosition(-0.0, 100) == Earliest) + } + ) + + private val roundingSuite = suite("rounding")( + test("100 entries: the fractions resolve to the entries the live broker delivered") { + // The oracle table. On the probe topic entry k held message m-k, and each of these + // seeks delivered exactly that message first. + assertTrue( + resolveApproximateEntryPosition(0.01, 100) == Entry(2), // m-2 + resolveApproximateEntryPosition(0.1, 100) == Entry(11), // m-11 + resolveApproximateEntryPosition(0.25, 100) == Entry(26), // m-26 + resolveApproximateEntryPosition(0.5, 100) == Entry(51), // m-51 + resolveApproximateEntryPosition(0.75, 100) == Entry(76), // m-76 + resolveApproximateEntryPosition(0.9, 100) == Entry(91), // m-91 + resolveApproximateEntryPosition(0.99, 100) == Entry(100) // m-100 + ) + }, + test("the rule is: leave floor(fraction * entries) entries behind") { + // Stated as a property rather than as the formula: what is skipped never overshoots the + // fraction asked for, and is never a whole entry short of it. + val entries = 997L + val violations = (1 to 999).map(_ / 1000.0).flatMap { fraction => + resolveApproximateEntryPosition(fraction, entries) match + case Entry(ordinal) => + val skipped = (ordinal - 1).toDouble + val exact = fraction * entries + Option.when(skipped > exact || exact - skipped >= 1.0)(s"$fraction -> entry $ordinal") + case other => Some(s"$fraction -> $other") + } + assertTrue(violations.isEmpty) ?? s"fractions that did not land within one entry of their proportion: $violations" + }, + test("a larger fraction never lands earlier in the log") { + val entries = 250L + val ordinals = (0 to 1000).map(_ / 1000.0).map { fraction => + resolveApproximateEntryPosition(fraction, entries) match + case Earliest => 0L + case Entry(ordinal) => ordinal + case Latest => Long.MaxValue + } + assertTrue(ordinals == ordinals.sorted) ?? "the resolved position must be monotonic in the fraction" + }, + test("the ordinal is 1-based and never runs past the end of the log") { + // examineMessage CLAMPS past the end instead of failing, so an ordinal of entries + 1 + // would silently land on the last entry and look like it worked. + assertTrue( + resolveApproximateEntryPosition(0.9999999, 10) == Entry(10), + resolveApproximateEntryPosition(0.5, 1) == Entry(1), + resolveApproximateEntryPosition(0.0000001, 10) == Entry(1) + ) + }, + test("a huge log resolves without overflowing") { + assertTrue(resolveApproximateEntryPosition(0.5, 4_000_000_000L) == Entry(2_000_000_001L)) + } + ) + + private val emptyLogSuite = suite("empty log")( + test("a topic with no retained entries starts from the beginning") { + // VERIFIED: examineMessage on an empty topic FAILS ("Could not examine messages due to + // the total message is zero"), so an ordinal must never be asked for here. Seeking to + // earliest is what "EarliestMessage" does on an empty topic - the session shows what + // gets published from now on. + assertTrue( + resolveApproximateEntryPosition(0.5, 0) == Earliest, + resolveApproximateEntryPosition(0.0, 0) == Earliest + ) + }, + test("1.0 on an empty topic still means latest") { + assertTrue(resolveApproximateEntryPosition(1.0, 0) == Latest) + }, + test("a negative entry count cannot produce an ordinal") { + assertTrue(resolveApproximateEntryPosition(0.5, -1) == Earliest) + } + ) + + private val rejectionSuite = suite("rejected fractions")( + test("NaN is rejected - every comparison against it is false, so it would slip through a range check") { + val message = rejected(Double.NaN) + assertTrue(message.exists(_.contains("NaN"))) ?? s"NaN must be rejected with a clear message, got: $message" + }, + test("a fraction below 0 or above 1 is rejected and the message names the value") { + val below = rejected(-0.5) + val above = rejected(1.5) + assertTrue( + below.exists(m => m.contains("Approximate position (% of data)") && m.contains("-0.5") && m.contains("0.0") && m.contains("1.0")), + above.exists(m => m.contains("Approximate position (% of data)") && m.contains("1.5") && m.contains("0.0") && m.contains("1.0")) + ) ?? s"out-of-range fractions must be rejected clearly, got: $below / $above" + }, + test("infinities are rejected") { + assertTrue( + rejected(Double.PositiveInfinity).isDefined, + rejected(Double.NegativeInfinity).isDefined + ) + }, + test("a fraction just inside the range is accepted") { + assertTrue( + rejected(0.0).isEmpty, + rejected(1.0).isEmpty, + rejected(0.9999999999).isEmpty + ) + } + ) + + private val lookupSuite = suite("bounded physical-topic resolution")( + test("0% and 100% resolve without one broker-derived lookup") { + val calls = AtomicInteger(0) + def forbiddenCount(topicFqn: String): Long = + calls.incrementAndGet() + throw AssertionError(s"endpoint asked for an entry count on $topicFqn") + def forbiddenEntry(topicFqn: String)(entryOrdinal: Long): Option[String] = + calls.incrementAndGet() + throw AssertionError(s"endpoint examined entry $entryOrdinal on $topicFqn") + def forbiddenEarliest(topicFqn: String): Option[String] = + calls.incrementAndGet() + throw AssertionError(s"endpoint re-read the retained start of $topicFqn") + + val topics = Vector("a", "a", "b") + val atStart = + resolveApproximateEntrySeeks(0.0, topics, "earliest", "latest", forbiddenCount, forbiddenEntry, forbiddenEarliest, _ < _) + val atEnd = + resolveApproximateEntrySeeks(1.0, topics, "earliest", "latest", forbiddenCount, forbiddenEntry, forbiddenEarliest, _ < _) + assertTrue( + calls.get == 0, + atStart == Map("a" -> "earliest", "b" -> "earliest"), + atEnd == Map("a" -> "latest", "b" -> "latest") + ) + }, + test("duplicate target consumers share exactly one resolved entry position per FQN") { + val countCalls = AtomicInteger(0) + val examineCalls = AtomicInteger(0) + val topics = Vector("a", "a", "b", "a", "b") + + val seeks = resolveApproximateEntrySeeks( + fraction = 0.5, + topicFqns = topics, + earliest = "earliest", + latest = "latest", + retainedEntryCountOf = _ => + countCalls.incrementAndGet() + 10L, + entryFromEarliestOf = topic => ordinal => + examineCalls.incrementAndGet() + Some(s"$topic-$ordinal"), + earliestRetainedEntryOf = topic => Some(s"$topic-1"), + entryIsOlder = (a, b) => a < b, + parallelism = 2 + ) + + assertTrue( + seeks == Map("a" -> "a-6", "b" -> "b-6"), + // Initial count + retention re-check, once for each DISTINCT topic. + countCalls.get == 4, + examineCalls.get == 2 + ) + }, + test("P2.3: retention that TRIMS AND APPENDS equally still invalidates the chosen anchor") { + // The re-check was count-only: read the count, examine the ordinal, read the count + // again, accept while it is not below the ordinal. A topic that trimmed 5 entries off + // the front and appended 5 to the tail in that window has exactly the same count and a + // completely different entry at that ordinal - and the id the examine handed back has + // been DELETED. The seek then lands wherever the broker puts a trimmed id, and the + // session silently starts somewhere the user did not ask for. + val entryOrdinals = AtomicInteger(0) + val seeks = resolveApproximateEntrySeeks( + fraction = 0.5, + topicFqns = Vector("trimming"), + earliest = "earliest", + latest = "latest", + // Unchanged across the whole resolution: 5 trimmed, 5 appended. + retainedEntryCountOf = _ => 10L, + entryFromEarliestOf = _ => + ordinal => + entryOrdinals.incrementAndGet() + Some(s"entry-$ordinal"), + // The front of the log is now PAST the anchor the examine returned. + earliestRetainedEntryOf = _ => Some("entry-9"), + entryIsOlder = (a, b) => a < b + ) + + assertTrue( + entryOrdinals.get == 1, // vacuity guard: the anchor really was examined + seeks == Map("trimming" -> "earliest") // ...and refused, not handed back as the position + ) ?? s"seeks=$seeks examines=${entryOrdinals.get}" + }, + test("P2.3: an anchor that is still retained is accepted, re-check and all") { + val seeks = resolveApproximateEntrySeeks( + fraction = 0.5, + topicFqns = Vector("stable"), + earliest = "earliest", + latest = "latest", + retainedEntryCountOf = _ => 10L, + entryFromEarliestOf = _ => ordinal => Some(s"entry-$ordinal"), + earliestRetainedEntryOf = _ => Some("entry-1"), + entryIsOlder = (a, b) => a < b + ) + assertTrue(seeks == Map("stable" -> "entry-6")) ?? s"seeks=$seeks" + }, + test("P2.3: a topic that now retains NOTHING has lost its anchor by definition") { + val seeks = resolveApproximateEntrySeeks( + fraction = 0.5, + topicFqns = Vector("emptied"), + earliest = "earliest", + latest = "latest", + retainedEntryCountOf = _ => 10L, + entryFromEarliestOf = _ => ordinal => Some(s"entry-$ordinal"), + earliestRetainedEntryOf = _ => None, + entryIsOlder = (a, b) => a < b + ) + assertTrue(seeks == Map("emptied" -> "earliest")) ?? s"seeks=$seeks" + }, + test("2,000 distinct topics stay inside the fixed lookup concurrency") { + val topics = (1 to 2_000).map(i => s"topic-$i").toVector + val inFlight = AtomicInteger(0) + val peakInFlight = AtomicInteger(0) + + val answers = boundedParallelTopicLookup( + topics, + operation = "the wide test position", + lookup = topic => + val current = inFlight.incrementAndGet() + peakInFlight.updateAndGet(previous => math.max(previous, current)) + try + Thread.sleep(1) + topic + finally inFlight.decrementAndGet() + ) + + assertTrue( + answers.size == 2_000, + answers.keySet == topics.toSet, + answers.forall((topic, answer) => topic == answer), + peakInFlight.get > 1, + peakInFlight.get <= approximatePositionLookupParallelism, + inFlight.get == 0 + ) ?? s"answers=${answers.size} peak=${peakInFlight.get} inFlight=${inFlight.get}" + }, + test("the wall-clock budget refuses and cancels a lookup set that cannot finish") { + val calls = AtomicInteger(0) + val topics = (1 to 100).map(i => s"topic-$i").toVector + val result = Try( + boundedParallelTopicLookup( + topics, + operation = "'Approximate position (% of data)'", + lookup = _ => + calls.incrementAndGet() + Thread.sleep(10_000) + "unreachable", + budgetMs = 50, + parallelism = 1 + ) + ) + + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + result.failed.toOption.exists(_.getMessage.contains("Approximate position (% of data)")), + result.failed.toOption.exists(_.getMessage.contains("50ms")), + result.failed.toOption.exists(_.getMessage.contains("100 physical topic")), + result.failed.toOption.exists(_.getMessage.contains("Earliest message")), + result.failed.toOption.exists(_.getMessage.contains("Latest message")), + result.failed.toOption.exists(_.getMessage.contains("Specific time")), + calls.get < topics.size + ) ?? s"result=$result calls=${calls.get}" + }, + test("CONCURRENT RESOLUTIONS SHARE ONE GLOBAL WORKER BUDGET - the per-call bound is not a server bound") { + // A fresh 16-thread pool was created for EVERY resolution, so the only bound was a + // local one: six concurrent creates meant six pools, ~96 worker threads and ~96 admin + // requests in flight, with nothing server-wide saying no. Combined with an admission + // check that did not cover in-flight builds, a handful of tabs could do that at will. + // + // Pinned by a latch rather than by timing: every lookup holds its place until + // `globalBound + 1` of them are simultaneously inside, so a peak past the bound is a + // fact the test WAITS for instead of hoping to sample. Nothing is over the bound, the + // latch never trips, the first wave times out once and opens the gate for the rest. + val calls = 6 + val topicsPerCall = 200 + val inFlight = AtomicInteger(0) + val peakInFlight = AtomicInteger(0) + val workerThreads = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + val pastTheBound = java.util.concurrent.CountDownLatch(approximatePositionLookupGlobalParallelism + 1) + val gateOpen = java.util.concurrent.atomic.AtomicBoolean(false) + + val lookup = (topic: String) => + val current = inFlight.incrementAndGet() + peakInFlight.updateAndGet(previous => math.max(previous, current)) + workerThreads.add(Thread.currentThread.getName) + pastTheBound.countDown() + if !gateOpen.get && !pastTheBound.await(2, java.util.concurrent.TimeUnit.SECONDS) then gateOpen.set(true) + inFlight.decrementAndGet() + topic + + val threads = (1 to calls).map(call => + val thread = new Thread( + (() => + boundedParallelTopicLookup( + (1 to topicsPerCall).map(i => s"call-$call-topic-$i").toVector, + operation = s"the shared-budget test position $call", + lookup = lookup, + budgetMs = 120_000 + ) + ): Runnable, + s"cs-shared-budget-$call" + ) + thread.setDaemon(true) + thread + ) + threads.foreach(_.start()) + threads.foreach(_.join(180_000)) + + assertTrue( + threads.forall(!_.isAlive), + peakInFlight.get > 1, // still genuinely parallel - a serial fallback is not the fix + peakInFlight.get <= approximatePositionLookupGlobalParallelism, + workerThreads.size <= approximatePositionLookupGlobalParallelism, + inFlight.get == 0, + // The shared budget's own gauge - what a saturation metric reads - never past its bound. + approximateLookupWorkersInFlight <= approximatePositionLookupGlobalParallelism + ) ?? (s"peak=${peakInFlight.get} globalBound=$approximatePositionLookupGlobalParallelism " + + s"distinctWorkerThreads=${workerThreads.size} inFlight=${inFlight.get} " + + s"sharedGauge=$approximateLookupWorkersInFlight") + }, + test("a non-positive budget refuses before dispatching any lookup") { + val calls = AtomicInteger(0) + val result = Try( + boundedParallelTopicLookup( + Vector("a", "b"), + operation = "'Approximate position (% of data)'", + lookup = topic => + calls.incrementAndGet() + topic, + budgetMs = 0 + ) + ) + assertTrue(result.isFailure, calls.get == 0) + } + ) + + // SEQUENTIAL: the lookup suite measures the SHARED server-wide worker budget, and a sibling + // test resolving positions at the same time would occupy part of the very budget under test. + def spec = + suite(this.getClass.toString)(endpointsSuite, roundingSuite, emptyLogSuite, rejectionSuite, lookupSuite) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/approximatePublishTimePositionTest.scala b/server/src/test/scala/consumer/session_runner/approximatePublishTimePositionTest.scala new file mode 100644 index 000000000..4d9c61e65 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/approximatePublishTimePositionTest.scala @@ -0,0 +1,336 @@ +package consumer.session_runner + +import zio.test.* + +import scala.collection.mutable +import scala.util.Try +import java.util.concurrent.atomic.AtomicInteger + +/** "Start approximately this far through the PUBLISH-TIME RANGE this topic covers" ([[ApproximatePublishTimePosition]]). + * + * The contract is per logical topic: `earliest` is the minimum observed first-entry publish time + * over its partitions, `latest` the maximum observed last-entry publish time, and the cutoff is + * `earliest + fraction * (latest - earliest)`. Every partition is seeked to that one instant. + * Min/max rather than a per-partition quantile is what makes the endpoints exact by construction, + * the mapping monotonic, and the answer independent of how many partitions the topic has. + * + * PURE by construction: the broker sits behind the `timeSpanOf` lookup, so every arrangement below - + * balanced partitions, uneven ones, an idle one, an empty topic, a topic that occupies a single + * instant, both endpoints - is driven with a plain function and no broker. The live-broker + * counterpart is `CsStartFromOutcomesSpec` CS-SF-16..19. + */ +object approximatePublishTimePositionTest extends ZIOSpecDefault: + + import ApproximatePublishTimeSeek.* + + /** A topic whose partitions are named `p0`, `p1`, ... with the spans given. */ + private def topic(spans: (String, (Long, Long))*): (Vector[String], String => Option[TopicPublishTimeSpan]) = + val byName = spans.toMap + (spans.map(_._1).toVector, name => byName.get(name).map((first, last) => TopicPublishTimeSpan(first, last))) + + private def resolve(fraction: Double, spans: (String, (Long, Long))*): ApproximatePublishTimeSeek = + val (partitions, lookup) = topic(spans*) + resolveApproximatePublishTimePosition(fraction, partitions, lookup) + + private def rejected(fraction: Double): Option[String] = + Try(resolve(fraction, "p0" -> (1000L, 2000L))).failed.toOption.map(_.getMessage) + + /** One partition covering 1000..2000ms - the simplest possible arrangement. */ + private val single = Seq("p0" -> (1000L, 2000L)) + + private val endpointsSuite = suite("endpoints")( + test("0.0 is the earliest retained message, not the earliest TIMESTAMP") { + // Both would deliver the same first message when the recorded time is still accurate, + // but only MessageId.earliest keeps working when it is not - and a timestamp seek + // computed from a stale reading would silently skip the head of the log. + assertTrue(resolve(0.0, single*) == Earliest) + }, + test("1.0 seeks to the observed last-entry publish-time boundary") { + // NOT "past the end", which is what 1.0 means in the entry mode. The publish-time mode's high + // endpoint is a timestamp seek and can include every message sharing that millisecond. + assertTrue(resolve(1.0, single*) == Timestamp(2000L)) + }, + test("the endpoints are exact - never derived by interpolating to the ends") { + // Interpolation would reach the same two numbers by arithmetic on doubles. Pinned as + // its own case because the whole point of an endpoint is that it cannot be off by one. + val spread = Seq("p0" -> (1_700_000_000_123L, 1_700_000_086_400_000L)) + assertTrue( + resolve(0.0, spread*) == Earliest, + resolve(1.0, spread*) == Timestamp(1_700_000_086_400_000L) + ) + }, + test("1.0 uses the largest observed last-entry time across the topic's partitions") { + assertTrue(resolve(1.0, "p0" -> (1000L, 5000L), "p1" -> (2000L, 9000L)) == Timestamp(9000L)) + }, + test("0.0 answers without asking the broker anything at all") { + // The earliest end needs no time range, so it must not pay for one: two admin calls per + // partition is the cost this endpoint gets to skip. + val asked = mutable.ListBuffer.empty[String] + val seek = resolveApproximatePublishTimePosition( + 0.0, + Vector("p0", "p1", "p2"), + name => + asked += name + Some(TopicPublishTimeSpan(1000L, 2000L)) + ) + assertTrue(seek == Earliest, asked.isEmpty) ?? s"0.0 looked up $asked" + } + ) + + private val interpolationSuite = suite("the cutoff between the endpoints")( + test("half way through a single partition's range is half way through its time") { + assertTrue(resolve(0.5, "p0" -> (1000L, 2000L)) == Timestamp(1500L)) + }, + test("balanced partitions covering the same range resolve to that range's midpoint") { + // Every partition is seeked to the SAME instant - the position is a property of the + // logical topic, not of whichever partition a message happens to live on. + assertTrue(resolve(0.5, "p0" -> (1000L, 2000L), "p1" -> (1000L, 2000L)) == Timestamp(1500L)) + }, + test("partitions with different spans are combined as min(first) .. max(last)") { + // p0 covers 1000..3000 and p1 covers 2000..5000, so the topic covers 1000..5000 and 25% + // of it is 2000 - inside p1's very first message and a third of the way into p0. + assertTrue( + resolve(0.25, "p0" -> (1000L, 3000L), "p1" -> (2000L, 5000L)) == Timestamp(2000L), + resolve(0.5, "p0" -> (1000L, 3000L), "p1" -> (2000L, 5000L)) == Timestamp(3000L) + ) + }, + test("an IDLE partition does not drag the position backwards") { + // p1 stopped receiving at 1100 while p0 ran on to 9000. A per-partition quantile would + // give p1 a cutoff near its own middle and hand back messages from the far past; taking + // the topic's own min/max puts both partitions at the same instant, and the idle one + // simply has nothing at or after it. This is a limitation the entry mode has to live with + // and this mode is defined to avoid. + assertTrue(resolve(0.5, "p0" -> (1000L, 9000L), "p1" -> (1000L, 1100L)) == Timestamp(5000L)) + }, + test("the answer does not depend on how many partitions the topic has") { + // The same overall range, split three ways and then six ways: identical cutoff. A + // formula that averaged per-partition positions would move here. + val threeWays = Seq("p0" -> (0L, 300L), "p1" -> (100L, 600L), "p2" -> (200L, 900L)) + val sixWays = Seq( + "p0" -> (0L, 100L), + "p1" -> (100L, 200L), + "p2" -> (200L, 400L), + "p3" -> (300L, 600L), + "p4" -> (400L, 700L), + "p5" -> (500L, 900L) + ) + assertTrue(resolve(0.4, threeWays*) == resolve(0.4, sixWays*), resolve(0.4, threeWays*) == Timestamp(360L)) + }, + test("the cutoff is rounded DOWN, so it never lands past the proportion asked for") { + // 1/3 of 10ms is 3.33ms. Rounding down keeps the mapping monotonic and keeps the + // delivered set from starting later than the instant the user pointed at. + assertTrue( + resolve(1.0 / 3.0, "p0" -> (0L, 10L)) == Timestamp(3L), + resolve(2.0 / 3.0, "p0" -> (0L, 10L)) == Timestamp(6L) + ) + }, + test("a larger fraction never resolves to an earlier instant") { + val cutoffs = (0 to 1000).map(_ / 1000.0).map { fraction => + resolve(fraction, "p0" -> (1_700_000_000_000L, 1_700_002_592_000L)) match + case Earliest => Long.MinValue + case Timestamp(atMs) => atMs + } + assertTrue(cutoffs == cutoffs.sorted) ?? "the resolved instant must be monotonic in the fraction" + }, + test("a 30-day range resolves without losing milliseconds to double arithmetic") { + // Epoch millis are ~1.7e12 and a month is ~2.6e9 - both far inside a double's exact + // integer range, but the multiplication has to be done on the SPAN rather than on the + // absolute instants for that to hold. + val start = 1_700_000_000_000L + val thirtyDays = 30L * 24 * 60 * 60 * 1000 + assertTrue( + resolve(0.5, "p0" -> (start, start + thirtyDays)) == Timestamp(start + thirtyDays / 2), + resolve(0.1, "p0" -> (start, start + thirtyDays)) == Timestamp(start + thirtyDays / 10) + ) + }, + test("message density does not change the publish-time interpolation") { + // The motivating topic: a month of retention in which almost everything arrived in the + // final hour. 50% of the TIME RANGE is 15 days back, whatever the message density is - + // that is the whole reason this mode exists next to the entry-position mode. + val start = 1_700_000_000_000L + val thirtyDays = 30L * 24 * 60 * 60 * 1000 + val fifteenDays = thirtyDays / 2 + assertTrue(resolve(0.5, "p0" -> (start, start + thirtyDays)) == Timestamp(start + fifteenDays)) + } + ) + + private val degenerateSuite = suite("topics with no time range to speak of")( + test("a topic no partition can answer for starts from the beginning") { + // Empty: `examineMessage` FAILS on a topic with no entries rather than answering, so + // there is no range at all. Seeking to earliest is what "Earliest message" does on an + // empty topic - the session shows whatever is published from now on. + val (partitions, _) = topic("p0" -> (0L, 0L)) + val nothing = resolveApproximatePublishTimePosition(0.5, partitions, _ => None) + assertTrue(nothing == Earliest) + }, + test("an empty topic resolves to the beginning at EVERY fraction, including 1.0") { + // There is no last-entry boundary to inspect, and on a topic holding nothing "the + // beginning" and "the end" are the same position: whatever arrives next. + val fractions = Vector(0.0, 0.25, 0.5, 0.75, 1.0) + val seeks = fractions.map(f => resolveApproximatePublishTimePosition(f, Vector("p0"), _ => None)) + assertTrue(seeks.forall(_ == Earliest)) ?? s"an empty topic resolved to $seeks" + }, + test("partitions that hold nothing are skipped, not counted as time zero") { + // A partitioned topic where only some partitions were written to: an unanswerable + // partition contributing 0 to the minimum would drag `earliest` back to 1970 and make + // every interior fraction land before the retained messages. + val (partitions, lookup) = topic("p0" -> (4000L, 8000L)) + val withEmpties = resolveApproximatePublishTimePosition(0.5, partitions ++ Vector("p1", "p2"), lookup) + assertTrue(withEmpties == Timestamp(6000L)) + }, + test("a topic that occupies ONE instant starts from the beginning for any interior fraction") { + // first == last: the range has no interior to interpolate into. Every message shares + // that instant, so there is no position that separates them and the honest answer is + // "all of it" - which is what a seek to earliest delivers. No division is involved, so + // this is a definition rather than a guard against a divide-by-zero. + assertTrue( + resolve(0.5, "p0" -> (1500L, 1500L)) == Earliest, + resolve(0.01, "p0" -> (1500L, 1500L)) == Earliest, + resolve(0.99, "p0" -> (1500L, 1500L)) == Earliest + ) + }, + test("a topic that occupies one instant still answers 1.0 with that instant") { + // The observed last-entry time is 1500 and a seek to 1500 delivers it, so the high endpoint + // stays exact even with no range. Everything else published in that same millisecond + // comes too: a timestamp seek is millisecond-granular and cannot separate them. + assertTrue(resolve(1.0, "p0" -> (1500L, 1500L)) == Timestamp(1500L)) + }, + test("a single message is a topic of one instant, and behaves like one") { + assertTrue( + resolve(0.0, "p0" -> (1500L, 1500L)) == Earliest, + resolve(0.5, "p0" -> (1500L, 1500L)) == Earliest, + resolve(1.0, "p0" -> (1500L, 1500L)) == Timestamp(1500L) + ) + }, + test("a range reported backwards by a producer's clock falls back to the beginning") { + // publishTime is stamped by the PRODUCER, so a clock that jumped backwards mid-topic can + // report a first later than the last. There is no interior to interpolate, and a cutoff + // taken from either end could hide messages, so the beginning is the only safe answer. + assertTrue(resolve(0.5, "p0" -> (9000L, 1000L)) == Earliest) + } + ) + + private val rejectionSuite = suite("rejected fractions")( + test("NaN is rejected - every comparison against it is false, so a range check alone lets it through") { + val message = rejected(Double.NaN) + assertTrue(message.exists(_.contains("NaN"))) ?? s"NaN must be rejected with a clear message, got: $message" + }, + test("a fraction below 0 or above 1 is rejected and the message names the value") { + val below = rejected(-0.5) + val above = rejected(1.5) + assertTrue( + below.exists(m => m.contains("Approximate position (% of time)") && m.contains("-0.5") && m.contains("0.0") && m.contains("1.0")), + above.exists(m => m.contains("Approximate position (% of time)") && m.contains("1.5") && m.contains("0.0") && m.contains("1.0")) + ) ?? s"out-of-range fractions must be rejected clearly, got: $below / $above" + }, + test("the message says which of the two modes refused it") { + // Two modes now carry a fraction, and a session can only be fixed if the error names the + // one that rejected it. + val message = rejected(1.5) + assertTrue(message.exists(m => m.contains("time") && !m.contains("data"))) ?? + s"the rejection must name the publish-time mode, got: $message" + }, + test("infinities are rejected") { + assertTrue(rejected(Double.PositiveInfinity).isDefined, rejected(Double.NegativeInfinity).isDefined) + }, + test("a fraction just inside the range is accepted") { + assertTrue(rejected(0.0).isEmpty, rejected(1.0).isEmpty, rejected(0.9999999999).isEmpty) + } + ) + + private val groupingSuite = suite("logical-topic grouping and bounded span lookup")( + test("partitions group under one logical topic while separate topics stay separate") { + val orders = "persistent://public/default/orders" + val audit = "persistent://public/default/audit" + val grouped = groupPhysicalTopicsByLogicalTopic( + Vector(s"$orders-partition-0", audit, s"$orders-partition-1", s"$orders-partition-0", audit) + ) + assertTrue( + grouped == Vector( + orders -> Vector(s"$orders-partition-0", s"$orders-partition-1"), + audit -> Vector(audit) + ) + ) + }, + test("0% groups topics but performs zero publish-time span lookups") { + val calls = AtomicInteger(0) + val orders = "persistent://public/default/orders" + val seeks = resolveApproximatePublishTimeSeeks( + fraction = 0.0, + topicFqns = Vector(s"$orders-partition-0", s"$orders-partition-1"), + timeSpanOf = _ => + calls.incrementAndGet() + throw AssertionError("0% looked up a publish-time span") + ) + assertTrue(calls.get == 0, seeks == Map(orders -> Earliest)) + }, + test("P2.15: SEPARATE topics share ONE cutoff - the session's history window is comparable") { + // The whole point of a "% of time" position on a multi-topic session: the user asked to + // start half way through the history they are looking at, and there is only one such + // history. Interpolating per LOGICAL topic answered a different wall-clock instant for + // each one, so a 50% session over a one-hour topic and a thirty-day topic began at two + // moments a fortnight apart and the merged view was a slice of nothing in particular. + val orders = "persistent://public/default/orders" + val audit = "persistent://public/default/audit" + val spans = Map( + s"$orders-partition-0" -> TopicPublishTimeSpan(1_000L, 2_000L), + s"$orders-partition-1" -> TopicPublishTimeSpan(1_500L, 3_000L), + audit -> TopicPublishTimeSpan(10_000L, 20_000L) + ) + val seeks = resolveApproximatePublishTimeSeeks( + fraction = 0.5, + topicFqns = Vector(s"$orders-partition-0", s"$orders-partition-1", audit), + timeSpanOf = spans.get + ) + + // Pooled across every selected physical topic: 1_000 .. 20_000, so 50% is 10_500. + assertTrue( + seeks == Map(orders -> Timestamp(10_500L), audit -> Timestamp(10_500L)) + ) ?? s"seeks=$seeks" + }, + test("P2.15: 100% is the LATEST boundary seen anywhere in the session, not per topic") { + val orders = "persistent://public/default/orders" + val audit = "persistent://public/default/audit" + val spans = Map( + orders -> TopicPublishTimeSpan(1_000L, 2_000L), + audit -> TopicPublishTimeSpan(10_000L, 20_000L) + ) + val seeks = resolveApproximatePublishTimeSeeks( + fraction = 1.0, + topicFqns = Vector(orders, audit), + timeSpanOf = spans.get + ) + assertTrue(seeks == Map(orders -> Timestamp(20_000L), audit -> Timestamp(20_000L))) ?? s"seeks=$seeks" + }, + test("P2.15: a topic the broker could not describe does not narrow the session's range") { + // `publishTimeSpan` answers None only for a topic that holds nothing. Such a topic + // contributes no boundary and gets the session's cutoff like everything else. + val orders = "persistent://public/default/orders" + val empty = "persistent://public/default/empty" + val spans = Map(orders -> TopicPublishTimeSpan(1_000L, 3_000L)) + val seeks = resolveApproximatePublishTimeSeeks( + fraction = 0.5, + topicFqns = Vector(orders, empty), + timeSpanOf = spans.get + ) + assertTrue(seeks == Map(orders -> Timestamp(2_000L), empty -> Timestamp(2_000L))) ?? s"seeks=$seeks" + }, + test("duplicate target consumers read each physical publish-time span once") { + val calls = AtomicInteger(0) + val orders = "persistent://public/default/orders" + val p0 = s"$orders-partition-0" + val p1 = s"$orders-partition-1" + val seeks = resolveApproximatePublishTimeSeeks( + fraction = 0.5, + topicFqns = Vector(p0, p0, p1, p0, p1), + timeSpanOf = topic => + calls.incrementAndGet() + Option.when(topic == p0)(TopicPublishTimeSpan(1_000L, 2_000L)) + .orElse(Some(TopicPublishTimeSpan(1_500L, 3_000L))), + parallelism = 2 + ) + assertTrue(calls.get == 2, seeks == Map(orders -> Timestamp(2_000L))) + } + ) + + def spec = suite(this.getClass.toString)(endpointsSuite, interpolationSuite, degenerateSuite, rejectionSuite, groupingSuite) diff --git a/server/src/test/scala/consumer/session_runner/batchSizeTest.scala b/server/src/test/scala/consumer/session_runner/batchSizeTest.scala new file mode 100644 index 000000000..8c5c8a580 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/batchSizeTest.scala @@ -0,0 +1,58 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.{MessageId, Schema} +import org.apache.pulsar.client.impl.{BatchMessageIdImpl, MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.nio.ByteBuffer + +/** `messagesInEntryOf` decides how many messages the backward walk counts for one entry, and that + * count drives both the running total and the per-topic overshoot discard. A PRODUCER must not be + * able to inflate it: `X-Pulsar-num-batch-message` lives in `getProperties` next to arbitrary + * producer keys, and the admin client only overwrites it for entries it expands as real batches, so + * a forged value survives on an unbatched message. The non-forgeable witness is the message id - + * the admin client returns a batch id (batch index >= 0) only for a genuine batch. + */ +object batchSizeTest extends ZIOSpecDefault: + + private val topic = "persistent://public/default/batch-size" + + private def message(id: MessageId, properties: Map[String, String]): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + // publish_time is mandatory on MessageMetadata - reading it when unset throws. + md.setPublishTime(1_700_000_000_000L) + properties.foreach((k, v) => md.addProperty().setKey(k).setValue(v)) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap("{}".getBytes("UTF-8")), Schema.BYTES, topic) + msg.setMessageId(id) + msg + + def spec = suite(this.getClass.toString)( + test("a forged batch-size property on an UNBATCHED message is ignored - the entry is one message") { + // THE forgery. An unbatched message (plain id, batch index -1) carrying a huge + // `X-Pulsar-num-batch-message` used to be counted as that many, overshooting the walk and + // letting the per-topic overshoot discard swallow real messages from the stream's head. + val forged = message(new MessageIdImpl(1L, 2L, -1), Map(batchSizeProperty -> "1000000")) + assertTrue(messagesInEntryOf(forged) == 1) ?? s"a forged property was counted as ${messagesInEntryOf(forged)}" + }, + test("a real batch id reports its own batch size, no property needed") { + // The id itself carries the count when the admin client populated it - not a property at all. + val batched = message(new BatchMessageIdImpl(1L, 2L, 0, 0, 50, null), Map.empty) + assertTrue(messagesInEntryOf(batched) == 50) + }, + test("a batch id the admin client left without a size falls back to the verified property") { + // batchSize -1 on the id, but the id CONFIRMS a batch (index 0), so the property is trusted. + val batched = message(new BatchMessageIdImpl(1L, 2L, 0, 0, -1, null), Map(batchSizeProperty -> "100")) + assertTrue(messagesInEntryOf(batched) == 100) + }, + test("a plain unbatched entry with no property is one message") { + assertTrue(messagesInEntryOf(message(new MessageIdImpl(1L, 2L, -1), Map.empty)) == 1) + }, + test("a garbage or non-positive forged size on a batch id falls back to one, never zero or negative") { + // Even where the id confirms a batch, a property that does not parse to a positive number + // is read as one message - the walk then goes DEEPER (over-delivers) rather than swallow. + val negative = message(new BatchMessageIdImpl(1L, 2L, 0, 0, -1, null), Map(batchSizeProperty -> "-5")) + val notANumber = message(new BatchMessageIdImpl(1L, 2L, 0, 0, -1, null), Map(batchSizeProperty -> "lots")) + assertTrue(messagesInEntryOf(negative) == 1, messagesInEntryOf(notANumber) == 1) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/buildConsumerBackoffTest.scala b/server/src/test/scala/consumer/session_runner/buildConsumerBackoffTest.scala new file mode 100644 index 000000000..f7584fbb5 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/buildConsumerBackoffTest.scala @@ -0,0 +1,77 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.PulsarClient +import org.apache.pulsar.client.impl.ConsumerBuilderImpl +import zio.test.* + +import java.util.concurrent.TimeUnit +import scala.util.Try + +/** What `buildConsumer` arms for NEGATIVE-ACKNOWLEDGMENT redelivery. + * + * Regression context: it set `negativeAckRedeliveryDelay(0, SECONDS)`, which the client floors to + * 100ms, so every sustained nack source became a ten-per-second redelivery hammer - a paused + * session nacks everything it receives, and a delivery that throws mid-batch is handed back the + * same way. (The start-from merge used to be the loudest source, nacking whatever it declined at + * its memory cap; it now pauses hot consumers instead and declines nothing.) A session pinned at the cap by one silent stream + * therefore redelivered its whole held set every ~100ms indefinitely - a permanent, silent storm + * of nack/redeliver traffic against the broker. + * + * A DECAYING backoff keeps the cases that should be prompt prompt (the first redelivery is still + * 100ms, so pause/resume feels immediate) while a message bounced over and over backs off toward + * a 10s ceiling, so the storm cools instead of spinning at the floor rate. + * + * The client is real and aimed at a closed port - building a consumer configuration connects to + * nothing. Asserted through `ConsumerBuilderImpl.getConf`, the same configuration `subscribe()` + * would use. + */ +object buildConsumerBackoffTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/nack-backoff" + + private def targetConfig: ConsumerSessionTarget = + ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicFqn))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + def spec = suite(this.getClass.toString)( + test("nack redelivery DECAYS from 100ms to a 10s ceiling instead of hammering at a flat floor") { + val client = PulsarClient.builder.serviceUrl("pulsar://127.0.0.1:1").operationTimeout(2, TimeUnit.SECONDS).build + try + val builder = buildConsumer( + pulsarClient = client, + consumerName = "cs-nack-backoff-0", + topicsToConsume = Vector(topicFqn), + listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + targetConfig = targetConfig + ).toOption.get + val conf = builder.asInstanceOf[ConsumerBuilderImpl[Array[Byte]]].getConf + val backoff = Option(conf.getNegativeAckRedeliveryBackoff) + val delays = (0 to 20).toVector.map(redeliveryCount => backoff.map(_.next(redeliveryCount)).getOrElse(-1L)) + assertTrue( + backoff.isDefined, + delays.head == 100L, + delays.last == 10_000L, + delays.forall(_ <= 10_000L), + delays.zip(delays.tail).forall((sooner, later) => sooner <= later) + ) ?? s"negativeAckRedeliveryBackoff=$backoff delays=$delays" + finally + Try(client.close()) + () + } + ) diff --git a/server/src/test/scala/consumer/session_runner/cleanupQuarantineTest.scala b/server/src/test/scala/consumer/session_runner/cleanupQuarantineTest.scala new file mode 100644 index 000000000..99e153b30 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/cleanupQuarantineTest.scala @@ -0,0 +1,185 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.CompletableFuture +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} + +/** A RELEASE THAT FAILED IS NOT A RELEASE, AND THE HANDLE IS THE ONLY WAY BACK TO IT. + * + * Both cleanup paths used to answer as though a failed release had happened: the partial-build + * unwind swallowed the failure whole, and a target's stop cleared its consumer map whatever + * unsubscribe and close did. A consumer that was mid-reconnect at that instant - the ordinary + * reason either call fails - then kept its subscription, its connection and its listener thread + * for the life of the process, with nothing left holding a reference to close it. + * + * Nothing here needs a broker: the consumer is a proxy that fails a set number of times. + */ +object cleanupQuarantineTest extends ZIOSpecDefault: + + private val topicA = "persistent://public/default/cs-cleanup-a" + private val topicB = "persistent://public/default/cs-cleanup-b" + + /** A consumer whose `unsubscribe` and `close` fail the first `failures` times each - a client + * mid-reconnect, which is exactly when a session is most likely to be stopped. */ + private final class FlakyConsumer(topicFqn: String, failures: Int): + val unsubscribeCalls = AtomicInteger(0) + val closeCalls = AtomicInteger(0) + private val unsubscribesToFail = AtomicInteger(failures) + private val closesToFail = AtomicInteger(failures) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => "cs-cleanup-0" + case "isConnected" => java.lang.Boolean.TRUE + case "unsubscribe" => + unsubscribeCalls.incrementAndGet() + if unsubscribesToFail.getAndUpdate(n => math.max(0, n - 1)) > 0 then + throw new IllegalStateException(s"$topicFqn is reconnecting and cannot unsubscribe") + null + case "close" => + closeCalls.incrementAndGet() + if closesToFail.getAndUpdate(n => math.max(0, n - 1)) > 0 then + throw new IllegalStateException(s"$topicFqn is reconnecting and cannot close") + null + case "pause" => null + case "resume" => null + case "acknowledgeAsync" => CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def targetRunner(consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionTargetRunner = + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumers.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumers.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + def spec = suite("failed cleanup stays addressable and is retried")( + test("P2.2: a consumer that could not be released is retried until it IS released") { + // The broker refuses once - the client is reconnecting - and the session is then + // discarded. Nothing but a retained handle can ever close that consumer afterwards. + val flaky = FlakyConsumer(topicA, failures = 1) + val quarantine = CleanupQuarantine() + val target = targetRunner(Map(topicA -> flaky.consumer)) + + val firstFailures = target.stop(quarantine) + val quarantinedAfterStop = quarantine.pendingLabels + + val stillFailing = quarantine.retryPending() + + assertTrue( + firstFailures.nonEmpty, // vacuity guard: the release really did fail + quarantinedAfterStop.exists(_.contains(topicA)), // the handle was RETAINED, not discarded + stillFailing.isEmpty, // ...and the retry released it + quarantine.pendingLabels.isEmpty, + flaky.unsubscribeCalls.get == 2, + flaky.closeCalls.get == 2 + ) ?? (s"failures=$firstFailures quarantined=$quarantinedAfterStop stillFailing=$stillFailing " + + s"unsubscribes=${flaky.unsubscribeCalls.get} closes=${flaky.closeCalls.get}") + }, + test("P2.2: a consumer that released cleanly is not quarantined, and its peer's failure does not hold it") { + val ok = FlakyConsumer(topicA, failures = 0) + val flaky = FlakyConsumer(topicB, failures = 1) + val quarantine = CleanupQuarantine() + val target = targetRunner(Map(topicA -> ok.consumer, topicB -> flaky.consumer)) + + target.stop(quarantine) + + assertTrue( + quarantine.pendingLabels.size == 1, + quarantine.pendingLabels.exists(_.contains(topicB)), + !quarantine.pendingLabels.exists(_.contains(topicA)) + ) ?? s"quarantined=${quarantine.pendingLabels}" + }, + test("P2.2: a release that keeps failing stays quarantined rather than being forgotten") { + val flaky = FlakyConsumer(topicA, failures = 5) + val quarantine = CleanupQuarantine() + val target = targetRunner(Map(topicA -> flaky.consumer)) + + target.stop(quarantine) + val afterOneRetry = quarantine.retryPending() + + assertTrue(afterOneRetry.size == 1, quarantine.pendingLabels.size == 1) ?? + s"stillFailing=$afterOneRetry pending=${quarantine.pendingLabels}" + }, + test("P2.2: the quarantine is BOUNDED - a server that can never release drops the oldest, not the heap") { + val quarantine = CleanupQuarantine(capacity = 3) + (1 to 10).foreach(i => quarantine.quarantine(s"resource-$i", () => throw new IllegalStateException("never"))) + assertTrue( + quarantine.pendingLabels.size == 3, + quarantine.pendingLabels == Vector("resource-8", "resource-9", "resource-10") + ) ?? s"pending=${quarantine.pendingLabels}" + }, + test("P2.2: the same resource is not quarantined twice by a repeated stop") { + val flaky = FlakyConsumer(topicA, failures = 5) + val quarantine = CleanupQuarantine() + val target = targetRunner(Map(topicA -> flaky.consumer)) + + target.stop(quarantine) + target.stop(quarantine) + + assertTrue(quarantine.pendingLabels.size == 1) ?? s"pending=${quarantine.pendingLabels}" + }, + test("P2.2: a partial build hands back what it could NOT release, and says why on the original failure") { + // `buildAllOrRelease` swallowed release failures outright: the original error propagated + // with no trace of the resources it had left behind, and nothing held them. + val released = scala.collection.mutable.ArrayBuffer[String]() + val stranded = scala.collection.mutable.ArrayBuffer[(String, String)]() + + val err = scala.util.Try( + buildAllOrRelease[String, String]( + inputs = Vector("a", "b", "boom"), + build = input => if input == "boom" then throw new RuntimeException("could not subscribe") else input, + release = resource => + if resource == "b" then throw new IllegalStateException(s"$resource is reconnecting") + released += resource + () + , + onReleaseFailure = (resource, cause) => { stranded += ((resource, cause.getMessage)); () } + ) + ).failed.get + + assertTrue( + err.getMessage == "could not subscribe", // the CAUSE still propagates, not a consequence + released.toVector == Vector("a"), + stranded.toVector.map(_._1) == Vector("b"), // the handle reached the caller + err.getSuppressed.toVector.map(_.getMessage) == Vector("b is reconnecting") + ) ?? s"released=$released stranded=$stranded suppressed=${err.getSuppressed.toVector.map(_.getMessage)}" + } + ) diff --git a/server/src/test/scala/consumer/session_runner/consumerListenerAckIdentityTest.scala b/server/src/test/scala/consumer/session_runner/consumerListenerAckIdentityTest.scala new file mode 100644 index 000000000..ae2639000 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/consumerListenerAckIdentityTest.scala @@ -0,0 +1,333 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{BatchMessageIdImpl, MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters.* + +/** MESSAGE IDENTITY UNDER FAILED PAPERWORK - the two ways a bare `MessageId` lies. + * + * The listener remembers a failed acknowledgment so the broker's redelivery can be finalized as + * paperwork instead of re-decided. That memory used to be keyed by the message id ALONE, and a + * bare id is not an identity here: + * + * - measured against a real broker, EVERY non-persistent message arrives as ledger 0, entry 0 + * - so one failed non-persistent ack marked the topic's NEXT unrelated message as + * "paperwork": acknowledged, never shown. And nothing is ever redelivered on a + * non-persistent topic (nothing is stored), so the marker could never be consumed honestly; + * - one listener serves every topic consumer of its target, and ids do not carry the topic. + * + * The registry is now keyed by (topic, id) and skipped entirely for non-persistent topics, whose + * failed acks are FINAL and inconsequential. + * + * The same suite pins the other explicit lifecycle fact: a delivery the listener SAW fail is + * recorded, and that record - nothing else - licenses the merge to deliver the broker's copy. + */ +object consumerListenerAckIdentityTest extends ZIOSpecDefault: + + private val consumerName = "cs-ack-identity-0" + private val persistentA = "persistent://public/default/ack-id-a" + private val persistentB = "persistent://public/default/ack-id-b" + private val nonPersistent = "non-persistent://public/default/ack-id-np" + + private final class RecordingConsumer(topicFqn: String, failFirstAck: Boolean = false): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + private val failedOnce = java.util.concurrent.atomic.AtomicBoolean(false) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "acknowledgeAsync" => + if failFirstAck && failedOnce.compareAndSet(false, true) then + CompletableFuture.failedFuture(new RuntimeException("broker went away mid-ack")) + else + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, ledgerId: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(ledgerId, entryId, -1)) + msg + + /** A member of a batched entry: same (ledger, entry) as its siblings, distinguished only by + * the batch index - the id shape a batched producer gives every message it sends. + * + * The BATCH SIZE is set, exactly as `ConsumerImpl` sets it on every id it hands a listener. + * It is what says whether the session has seen the whole entry yet, and therefore whether the + * broker can still redeliver it. */ + private def batchMessage( + topicFqn: String, + key: String, + publishTime: Long, + ledgerId: Long, + entryId: Long, + batchIndex: Int, + batchSize: Int = 4 + ): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new BatchMessageIdImpl(ledgerId, entryId, -1, batchIndex, batchSize, null)) + msg + + /** One physical stream of a WIDE target: its own topic, its own consumer, sharing the target's + * single listener - which is exactly the production shape, and the shape the per-target + * decided-member registry has to cover. */ + private def wideStreamTopic(i: Int): String = s"persistent://public/default/ack-id-wide-$i" + + /** Run `body` over `streams` on `threadCount` real threads, released together and JOINED before + * returning. The join is the phase boundary: everything this call does happens strictly before + * whatever the caller does next, so a pause landing between two phases slices every stream's + * entry at exactly the same member - no sleeps, no repetition until something lands. */ + private def acrossStreams(streams: Int, threadCount: Int)(body: Int => Unit): Unit = + val start = CountDownLatch(1) + val workers = (0 until threadCount).map(slot => + val thread = new Thread( + (() => { + start.await(60, TimeUnit.SECONDS) + (0 until streams).filter(_ % threadCount == slot).foreach(body) + }): Runnable, + s"cs-ack-wide-$slot" + ) + thread.setDaemon(true) + thread + ) + workers.foreach(_.start()) + start.countDown() + workers.foreach(_.join(120_000)) + + private def listenerRecording(delivered: ConcurrentLinkedQueue[String], throwOnFirst: Set[String] = Set.empty): ConsumerListener = + val thrown = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + val handler = ConsumerSessionTargetMessageHandler(onNext = msg => + if throwOnFirst.contains(msg.getKey) && thrown.add(msg.getKey) then + throw new io.grpc.StatusRuntimeException(io.grpc.Status.CANCELLED) + delivered.add(msg.getKey) + () + ) + val l = ConsumerListener(handler) + l.startAcceptingNewMessages() + l + + def spec = suite(this.getClass.toString)( + test("a NON-PERSISTENT failed ack is FINAL - the next (0,0) message is delivered, not swallowed as paperwork") { + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered) + val np = RecordingConsumer(nonPersistent, failFirstAck = true) + + // m1 delivers; its ack fails. Nothing is stored on a non-persistent topic, so there + // is nothing to retry - and no marker may be left behind. + l.received(np.consumer, message(nonPersistent, "m1", 100, 0, 0)) + val markersAfterFailure = l.awaitingAckRetryCount + // m2 is an UNRELATED message that happens to carry the same (0,0) id every + // non-persistent message carries. It used to be acknowledged as m1's paperwork. + l.received(np.consumer, message(nonPersistent, "m2", 200, 0, 0)) + + assertTrue( + markersAfterFailure == 0, + delivered.asScala.toVector == Vector("m1", "m2"), + np.handedBack.asScala.toVector.isEmpty // and no pointless nack for the failed ack + ) ?? s"delivered=${delivered.asScala.toVector} markers=$markersAfterFailure" + }, + test("ack-retry markers are TOPIC-QUALIFIED - a same-id message on a sibling topic is untouched") { + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered) + val consumerA = RecordingConsumer(persistentA, failFirstAck = true) + val consumerB = RecordingConsumer(persistentB) + + l.received(consumerA.consumer, message(persistentA, "a1", 100, 7, 5)) // delivered; ack fails -> marker (A, 7:5) + val markersAfterFailure = l.awaitingAckRetryCount + l.received(consumerB.consumer, message(persistentB, "b1", 150, 7, 5)) // same id, OTHER topic: ordinary delivery + val markersAfterSibling = l.awaitingAckRetryCount + l.received(consumerA.consumer, message(persistentA, "a1", 100, 7, 5)) // the real redelivery: paperwork only + + assertTrue( + markersAfterFailure == 1, + markersAfterSibling == 1, // the sibling neither consumed nor grew the marker + l.awaitingAckRetryCount == 0, + delivered.asScala.toVector == Vector("a1", "b1"), // a1 once, b1 once, the redelivery never + consumerA.acknowledged.asScala.count(_ == "a1") == 1 // the redelivery's finalizing ack + ) ?? s"delivered=${delivered.asScala.toVector} markers=${l.awaitingAckRetryCount}" + }, + test("A PAUSE-SLICED BATCH ENTRY: decided members are paperwork on the whole-entry redelivery, never re-shown") { + // The books-balance bug the 50k pause-loop e2e caught: a batched producer packs many + // messages into one broker entry, and per-member acks are client-local bits. Pause + // mid-entry and the gate delivers members 0..1, refuses 2..3 - so the broker + // redelivers the WHOLE entry with a fresh ack set, already-shown members included. + // They used to be re-decided (re-shown, ~half a batch of duplicates per hot pause); + // now their standing decision makes the copies pure paperwork. + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered) + val c = RecordingConsumer(persistentA) + + l.received(c.consumer, batchMessage(persistentA, "m0", 100, 7, 5, 0)) + l.received(c.consumer, batchMessage(persistentA, "m1", 101, 7, 5, 1)) + l.stopAcceptingNewMessages() // the pause lands mid-entry + l.received(c.consumer, batchMessage(persistentA, "m2", 102, 7, 5, 2)) + l.received(c.consumer, batchMessage(persistentA, "m3", 103, 7, 5, 3)) + val refusedAtGate = c.handedBack.asScala.toVector + l.startAcceptingNewMessages() // resume; the broker redelivers the entry entire + (0 to 3).foreach(i => l.received(c.consumer, batchMessage(persistentA, s"m$i", 100L + i, 7, 5, i))) + + assertTrue( + refusedAtGate == Vector("m2", "m3"), + delivered.asScala.toVector == Vector("m0", "m1", "m2", "m3"), // each shown EXACTLY once + c.acknowledged.asScala.count(_ == "m0") == 2, // the original ack + the copy's paperwork + c.acknowledged.asScala.count(_ == "m1") == 2, + c.acknowledged.asScala.count(_ == "m2") == 1, + c.acknowledged.asScala.count(_ == "m3") == 1, + l.decidedBatchEntryCount == 1 + ) ?? s"delivered=${delivered.asScala.toVector} acked=${c.acknowledged.asScala.toVector} refused=$refusedAtGate" + }, + test("A SLICED ENTRY ON EVERY STREAM OF A WIDE TARGET survives to its redelivery, however old it is") { + // The registry evicted by INSERTION ORDER at 512 entries while ONE target may hold + // 1,000 physical streams. A hot pause slices one outstanding entry per stream, so the + // earliest ~488 records were thrown away before their redelivery ever arrived - and the + // whole-entry redelivery then re-showed members this session had already delivered (or, + // in the counted modes, spent a second unit of the user's budget on them). Retention is + // a LIFECYCLE fact, not an age: an entry whose redelivery is still owed cannot be + // evicted at any size. + val streams = 600 // past the 512-entry registry, inside one target's 1,000-stream cap + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered) + val consumers = Vector.tabulate(streams)(i => RecordingConsumer(wideStreamTopic(i))) + + // PHASE 1, gate open: members 0 and 1 of every stream's entry are delivered and decided. + acrossStreams(streams, threadCount = 8)(i => + (0 to 1).foreach(member => + l.received(consumers(i).consumer, batchMessage(wideStreamTopic(i), s"s$i-m$member", 100L + member, 7, 5, member)) + ) + ) + val decidedAfterDelivery = l.decidedBatchEntryCount + + // PHASE 2: the pause lands mid-entry on every stream at once - one sliced entry each. + l.stopAcceptingNewMessages() + acrossStreams(streams, threadCount = 8)(i => + (2 to 3).foreach(member => + l.received(consumers(i).consumer, batchMessage(wideStreamTopic(i), s"s$i-m$member", 100L + member, 7, 5, member)) + ) + ) + val retainedWhileSliced = l.decidedBatchEntryCount + + // PHASE 3: resume, and the broker redelivers the OLDEST sliced entry entire. + l.startAcceptingNewMessages() + (0 to 3).foreach(member => + l.received(consumers(0).consumer, batchMessage(wideStreamTopic(0), s"s0-m$member", 100L + member, 7, 5, member)) + ) + + val oldestDelivered = delivered.asScala.toVector.filter(_.startsWith("s0-")) + val oldestAcknowledged = consumers(0).acknowledged.asScala.toVector + assertTrue( + decidedAfterDelivery == streams, + retainedWhileSliced == streams, // NOT capped at 512: every sliced entry is still owed a redelivery + consumers(0).handedBack.asScala.toVector == Vector("s0-m2", "s0-m3"), + oldestDelivered == Vector("s0-m0", "s0-m1", "s0-m2", "s0-m3"), // each shown EXACTLY once + oldestAcknowledged.count(_ == "s0-m0") == 2, // the original ack plus the copy's paperwork + oldestAcknowledged.count(_ == "s0-m1") == 2, + oldestAcknowledged.count(_ == "s0-m2") == 1, + oldestAcknowledged.count(_ == "s0-m3") == 1 + ) ?? (s"decidedAfterDelivery=$decidedAfterDelivery retainedWhileSliced=$retainedWhileSliced " + + s"oldestDelivered=$oldestDelivered oldestAcked=$oldestAcknowledged " + + s"handedBack=${consumers(0).handedBack.asScala.toVector}") + }, + test("AND IS RECLAIMED once that redelivery completes - retention is a lifecycle, not a bigger number") { + // The other half, and the reason this is not simply a larger constant: an entry the + // broker has retired can never come back, so its record is dead weight and must go. + // Only entries still owed a redelivery are exempt from the cap, so the registry cannot + // grow past (settled cap + one sliced entry per admitted stream). + val streams = 600 + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered) + val consumers = Vector.tabulate(streams)(i => RecordingConsumer(wideStreamTopic(i))) + + acrossStreams(streams, threadCount = 8)(i => + (0 to 1).foreach(member => + l.received(consumers(i).consumer, batchMessage(wideStreamTopic(i), s"s$i-m$member", 100L + member, 7, 5, member)) + ) + ) + l.stopAcceptingNewMessages() + acrossStreams(streams, threadCount = 8)(i => + (2 to 3).foreach(member => + l.received(consumers(i).consumer, batchMessage(wideStreamTopic(i), s"s$i-m$member", 100L + member, 7, 5, member)) + ) + ) + val retainedWhileSliced = l.decidedBatchEntryCount + + // Every sliced entry is now redelivered whole and completed, so nothing is owed. + l.startAcceptingNewMessages() + acrossStreams(streams, threadCount = 8)(i => + (0 to 3).foreach(member => + l.received(consumers(i).consumer, batchMessage(wideStreamTopic(i), s"s$i-m$member", 100L + member, 7, 5, member)) + ) + ) + + // Ordinary traffic afterwards: fresh entries on one stream, each of which may reclaim + // the settled records ahead of it. + (0 until 40).foreach(entry => + (0 to 3).foreach(member => + l.received(consumers(0).consumer, batchMessage(wideStreamTopic(0), s"fresh-$entry-$member", 500L + entry, 9, entry.toLong, member)) + ) + ) + + assertTrue( + retainedWhileSliced == streams, + l.decidedBatchEntryCount <= 512 // reclaimed back inside the cap once nothing is owed + ) ?? s"retainedWhileSliced=$retainedWhileSliced afterSettling=${l.decidedBatchEntryCount}" + }, + test("A FAILED DELIVERY IS AN EXPLICIT TRANSITION: recorded on the nack, claimed by the retry, delivered once") { + // The whole path the merge's duplicate arm rests on, driven through the REAL + // received() pipeline: handler throws once (exactly as a cancelled gRPC call does), + // the listener records the failure and hands the message back, and the broker's + // redelivery - watermarked as a copy - is delivered because of that record alone. + val delivered = ConcurrentLinkedQueue[String]() + val l = listenerRecording(delivered, throwOnFirst = Set("a1")) + val consumerA = RecordingConsumer(persistentA) + val consumerB = RecordingConsumer(persistentB) + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.Ordered( + // Best effort carries EMPTY recorded ends - the continuous rules need none. + Vector( + StartFromStream(startFromStreamId(consumerName, persistentA), EntryPosition.empty), + StartFromStream(startFromStreamId(consumerName, persistentB), EntryPosition.empty) + ), + consumer.session_config.MessageDeliveryOrder.BestEffort + ) + ) + + l.received(consumerA.consumer, message(persistentA, "a1", 100, 1, 0)) + l.received(consumerB.consumer, message(persistentB, "b1", 150, 1, 0)) // both heads: a1 emits - and its send throws + val failuresRecorded = l.failedDeliveryCount + l.received(consumerA.consumer, message(persistentA, "a1", 100, 1, 0)) // the broker's redelivery = the retry + + assertTrue( + failuresRecorded == 1, + l.failedDeliveryCount == 0, // claimed by the retry + consumerA.handedBack.asScala.toVector == Vector("a1"), // the failed attempt, handed back + delivered.asScala.toVector.count(_ == "a1") == 1, // shown exactly once + consumerA.acknowledged.asScala.count(_ == "a1") == 1 + ) ?? s"delivered=${delivered.asScala.toVector} failures=${l.failedDeliveryCount} handedBack=${consumerA.handedBack.asScala.toVector}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/consumerPauseArbiterTest.scala b/server/src/test/scala/consumer/session_runner/consumerPauseArbiterTest.scala new file mode 100644 index 000000000..701c60903 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/consumerPauseArbiterTest.scala @@ -0,0 +1,101 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.AtomicInteger + +/** THE PAUSE ARBITER: one consumer, three legitimate owners, no stomping. + * + * The deterministic failure this class ended, pinned here as its contract: at the skip + * boundary the merge's settle-time resume woke consumers the delivery LIMITER had just held. + * The limiter's flag still said "held", so it never re-paused, and the paced queue grew without + * a bound. With per-reason arbitration a release resumes the consumer only when NO other owner + * still holds it - each mechanism touches its own reason and nothing else. + */ +object consumerPauseArbiterTest extends ZIOSpecDefault: + + private final class CountingConsumer: + val paused = AtomicInteger(0) + val resumed = AtomicInteger(0) + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "pause" => paused.incrementAndGet(); null + case "resume" => resumed.incrementAndGet(); null + case "toString" => "counting-consumer" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private final class ThrowingConsumer: + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "pause" | "resume" => throw new IllegalStateException("consumer is mid-close") + case "toString" => "throwing-consumer" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + def spec = suite(this.getClass.toString)( + test("THE SKIP-BOUNDARY STOMP IS GONE: the merge letting go cannot wake a limiter-held consumer") { + val counting = CountingConsumer() + val arbiter = ConsumerPauseArbiter(counting.consumer) + arbiter.hold(PauseReason.Limiter) // paced queue crossed its watermark + arbiter.hold(PauseReason.Merge) // hot stream at the merge's watermark + arbiter.release(PauseReason.Merge) // the cut releases the merge's hold... + val resumedWhileLimiterHolds = counting.resumed.get + arbiter.release(PauseReason.Limiter) // ...and only the limiter's own release resumes + assertTrue( + resumedWhileLimiterHolds == 0, + counting.resumed.get == 1, + arbiter.heldReasons.isEmpty + ) ?? s"resumedWhileLimiterHolds=$resumedWhileLimiterHolds resumedTotal=${counting.resumed.get}" + }, + test("a user resume does not release the merge's hold on a hot stream") { + val counting = CountingConsumer() + val arbiter = ConsumerPauseArbiter(counting.consumer) + arbiter.hold(PauseReason.User) + arbiter.hold(PauseReason.Merge) + arbiter.release(PauseReason.User) + assertTrue( + counting.resumed.get == 0, + arbiter.heldReasons == Set(PauseReason.Merge) + ) + }, + test("holding the same reason twice needs only one release, and resume fires exactly once") { + val counting = CountingConsumer() + val arbiter = ConsumerPauseArbiter(counting.consumer) + arbiter.hold(PauseReason.Merge) + arbiter.hold(PauseReason.Merge) + arbiter.release(PauseReason.Merge) + assertTrue(counting.resumed.get == 1, arbiter.heldReasons.isEmpty) + }, + test("every hold re-asserts the client-local pause flag - a rogue direct resume is healed") { + val counting = CountingConsumer() + val arbiter = ConsumerPauseArbiter(counting.consumer) + arbiter.hold(PauseReason.Limiter) + counting.consumer.resume() // somebody stomps directly, outside the arbiter + arbiter.hold(PauseReason.Merge) // the next hold of ANY reason re-pauses + assertTrue(counting.paused.get == 2) + }, + test("a throwing client call answers false, and the reason set stays truthful regardless") { + val throwing = ThrowingConsumer() + val arbiter = ConsumerPauseArbiter(throwing.consumer) + val held = arbiter.hold(PauseReason.User) + val reasonsAfterHold = arbiter.heldReasons + val released = arbiter.release(PauseReason.User) + assertTrue( + !held, + reasonsAfterHold == Set(PauseReason.User), + !released, + arbiter.heldReasons.isEmpty + ) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/consumerSessionContextTest.scala b/server/src/test/scala/consumer/session_runner/consumerSessionContextTest.scala new file mode 100644 index 000000000..7436d3c68 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/consumerSessionContextTest.scala @@ -0,0 +1,45 @@ +package consumer.session_runner + +import zio.test.* + +object consumerSessionContextTest extends ZIOSpecDefault: + private def withContext[A](use: ConsumerSessionContext => A): A = + val pool = ConsumerSessionContextPool() + try pool.withContext(0)(use) + finally pool.close() + + def spec = suite(this.getClass.toString)( + test("lastMessage is a live public view of the latest session message") { + withContext { sessionContext => + val absentBeforeFirstMessage = sessionContext.context + .eval("js", "typeof lastMessage === 'undefined'") + .asBoolean() + + sessionContext.setCurrentMessage("""{"key":"first"}""", Right("""{"number":1}""")) + val firstMessageIsVisible = sessionContext.context + .eval( + "js", + s"lastMessage === $CurrentMessageVarName && lastMessage.key === 'first' && lastMessage.value.number === 1" + ) + .asBoolean() + + sessionContext.setCurrentMessage("""{"key":"second"}""", Right("""{"number":2}""")) + val viewFollowedLatestMessage = sessionContext.context + .eval("js", "lastMessage.key === 'second' && lastMessage.value.number === 2") + .asBoolean() + + assertTrue(absentBeforeFirstMessage, firstMessageIsVisible, viewFollowedLatestMessage) + } + }, + test("setting the current message is silent but explicit console output is preserved") { + withContext { sessionContext => + sessionContext.setCurrentMessage("""{"key":"quiet"}""", Right("""{"number":3}""")) + val automaticOutput = sessionContext.getStdout + + sessionContext.runCode("""console.log("user-authored output")""") + val userOutput = sessionContext.getStdout + + assertTrue(automaticOutput.isEmpty, userOutput.contains("[LOG] user-authored output")) + } + } + ) diff --git a/server/src/test/scala/consumer/session_runner/consumerSessionRunnerTest.scala b/server/src/test/scala/consumer/session_runner/consumerSessionRunnerTest.scala new file mode 100644 index 000000000..025f16428 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/consumerSessionRunnerTest.scala @@ -0,0 +1,108 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.admin.PulsarAdmin +import org.apache.pulsar.client.api.PulsarClient +import zio.test.* + +import java.util.concurrent.TimeUnit +import scala.util.Try + +/** `ConsumerSessionRunner.make` is what `ConsumerServiceImpl.createConsumer` reports on: if it + * returns, the session is stored and the client is told Code.OK. + * + * Regression context: it accepted a runner with ZERO consumers. A `MultiTopicSelector` with no + * topics (or one whose topics all failed to resolve, before that was made loud) produced a target + * runner with an empty consumer map; `make` built a session from it and `createConsumer` answered + * OK, so the UI showed a session in state `running` that could never deliver a message and never + * explained why. + * + * The clients are REAL, aimed at a closed port. They construct offline and are never used on this + * path (an empty selection touches neither `getSchemasByTopic` nor `handleStartFrom`), but using + * real objects rather than nulls means an accidental use would surface as a connection error, not + * as an NPE that makes the assertion pass for the wrong reason. + */ +object consumerSessionRunnerTest extends ZIOSpecDefault: + + private def withOfflineClients[A](f: (PulsarClient, PulsarAdmin) => A): A = + val client = PulsarClient.builder + .serviceUrl("pulsar://127.0.0.1:1") + .operationTimeout(2, TimeUnit.SECONDS) + .build + val admin = PulsarAdmin.builder + .serviceHttpUrl("http://127.0.0.1:1") + .connectionTimeout(2, TimeUnit.SECONDS) + .readTimeout(2, TimeUnit.SECONDS) + .requestTimeout(2, TimeUnit.SECONDS) + .build + try f(client, admin) + finally + Try(client.close()) + Try(admin.close()) + + private def target(isEnabled: Boolean, topicFqns: Vector[String]): ConsumerSessionTarget = + ConsumerSessionTarget( + isEnabled = isEnabled, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = topicFqns)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def sessionConfig(targets: Vector[ConsumerSessionTarget]): ConsumerSessionConfig = + ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = targets, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def make(sessionName: String, targets: Vector[ConsumerSessionTarget]): Try[ConsumerSessionRunner] = + withOfflineClients((client, admin) => + Try(ConsumerSessionRunner.make( + pulsarClient = client, + adminClient = admin, + sessionName = sessionName, + sessionConfig = sessionConfig(targets) + )) + ) + + def spec = suite(this.getClass.toString)( + test("a session whose only enabled target resolves to no topics is rejected") { + val result = make("cs-empty-target", Vector(target(isEnabled = true, topicFqns = Vector.empty))) + val message = result.failed.toOption.map(_.getMessage).getOrElse("") + + assertTrue(result.isFailure, message.contains("no topics")) ?? + s"a target that resolves to nothing must not produce a consumer-less session, got: $result" + }, + test("a disabled target does not rescue a session that has no enabled target left") { + // Disabled targets are filtered out before consumers are built, so a session made only + // of them is just as empty - and used to be accepted just as silently. + val result = make("cs-only-disabled", Vector(target(isEnabled = false, topicFqns = Vector("persistent://public/default/t1")))) + val message = result.failed.toOption.map(_.getMessage).getOrElse("") + + assertTrue(result.isFailure, message.contains("no enabled targets")) ?? + s"a session with nothing enabled must be rejected, got: $result" + }, + test("a session with no targets at all is rejected") { + val result = make("cs-no-targets", Vector.empty) + val message = result.failed.toOption.map(_.getMessage).getOrElse("") + + assertTrue(result.isFailure, message.contains("no enabled targets")) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/deliveryBackpressureRaceTest.scala b/server/src/test/scala/consumer/session_runner/deliveryBackpressureRaceTest.scala new file mode 100644 index 000000000..7b38dac64 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryBackpressureRaceTest.scala @@ -0,0 +1,332 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters.* + +/** WHAT KEEPS A LIMITED SESSION ALIVE AND BOUNDED WHEN PAUSE, A FAILED RELEASE AND A COUNTED + * START-FROM MEET. + * + * Three defects live in the seams BETWEEN the limiter's mechanisms, which is why the suites that + * cover each mechanism on its own are all green: + * + * - a Resume landing INSIDE the paused tick found the drain flag still armed, scheduled nothing, + * and then watched the tick give the flag back - a finite backlog stuck forever with the UI + * saying Running (`deliveryRateLimiterTest` pauses, ticks and resumes SEQUENTIALLY, so the + * interleaving cannot occur there); + * - the single retry of a FAILED permit release was armed as an ordinary drain tick, so a user + * pause swallowed it, and Resume only re-arms a drain when something is QUEUED - which it + * never is after the drain that failed the release. Every consumer stayed held by the Limiter + * for the rest of the session (the suite covers failed release and paused re-arming, never + * their intersection); + * - permit-hold eligibility was decided for the WHOLE SESSION, so one latest-n correction that + * never finishes on a disconnected stream refused backpressure for every other stream, and a + * hot peer fed an unbounded queue while the watermarks were declined over and over. + * + * Everything is offline and deterministic: hand-cranked schedulers, a hand-cranked clock, proxy + * consumers, and - for the lost-wakeup race - the core's own monitor as the barrier that pins the + * interleaving. No sleeps, and nothing repeated until it happens to land. + */ +object deliveryBackpressureRaceTest extends ZIOSpecDefault: + + private val consumerName = "cs-backpressure-race" + private val p0 = "persistent://public/default/cs-backpressure-race-0" + private val hotTopic = "persistent://public/default/cs-backpressure-race-hot" + private val stuckTopic = "persistent://public/default/cs-backpressure-race-stuck" + + /** A consumer whose RUNNING state is observable, and whose `resume` can be made to fail the way + * a client mid-reconnect does - the trigger for the release retry this suite is about. */ + private final class RecordingConsumer(topicFqn: String, resumeFailures: Int = 0): + val permitCalls = ConcurrentLinkedQueue[String]() + val isPaused = AtomicBoolean(false) + private val resumesToFail = java.util.concurrent.atomic.AtomicInteger(resumeFailures) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "pause" => + permitCalls.add("pause") + isPaused.set(true) + null + case "resume" => + permitCalls.add("resume") + if resumesToFail.getAndUpdate(n => math.max(0, n - 1)) > 0 then + throw new IllegalStateException("the consumer is reconnecting and cannot take a resume") + isPaused.set(false) + null + case "acknowledgeAsync" => CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + /** A FAT message - the shape the byte watermark exists for, small enough that a broken run's + * unbounded backlog still fits in a test JVM. */ + private val payloadBytes = 4 * 1024 + + private def message(topicFqn: String, key: String, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(1000L + entryId) + md.setPartitionKey(key) + val payload = Array.fill[Byte](payloadBytes)('x'.toByte) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(payload), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetRunner(consumerListener: ConsumerListener, consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumers.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumers.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(target: ConsumerSessionTargetRunner): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-backpressure-race", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + /** THE BARRIER THAT PINS THE LOST-WAKEUP INTERLEAVING, and the reason it is not a poll for + * something that might never happen: the tick thread's paused branch reads the pause flag and + * then UNCONDITIONALLY enters the core's monitor, which this thread is holding. So the only + * state it can reach is BLOCKED, it always reaches it, and reaching it proves the pause flag + * has already been read. Bounded so a regression fails loudly instead of hanging the suite. + */ + private def awaitBlockedOnCoreLock(t: Thread): Unit = + val deadline = java.lang.System.nanoTime() + 30_000_000_000L + while t.getState != Thread.State.BLOCKED && java.lang.System.nanoTime() < deadline do Thread.onSpinWait() + if t.getState != Thread.State.BLOCKED then + throw new AssertionError(s"the paused tick never reached the core lock; its state is ${t.getState}") + + /** Offer fat messages one at a time until the hot consumer's permits are held, or until `cap` - + * the stand-in for "the broker keeps delivering". Answers how many were offered: a session + * whose backpressure works stops WELL below the cap, a session whose backpressure is refused + * runs to it. + */ + private def floodUntilHeld( + limiter: DeliveryRateLimiter[HeldMessage], + consumer: Consumer[Array[Byte]], + isPaused: AtomicBoolean, + listener: ConsumerListener, + cap: Int + ): Int = + var offered = 0 + while offered < cap && !isPaused.get do + limiter.offer(HeldMessage(consumer, message(consumer.getTopic, s"m$offered", offered.toLong), listener)) + offered += 1 + offered + + def spec = suite("delivery backpressure races")( + test("P1.5: a Resume landing INSIDE the paused tick must not lose the backlog's only wakeup") { + // The interleaving the sequential pause/tick/resume test cannot produce: + // 1. the armed tick reads drainingPaused = true; + // 2. Resume clears the flag and calls rearmDrain; + // 3. rearmDrain sees drainScheduled = true and schedules NOTHING; + // 4. the paused tick then clears drainScheduled and exits. + // Queue nonempty, no timer, and nothing further has to arrive - the backlog is stuck + // for good while the session says Running. + val pending = ConcurrentLinkedQueue[Runnable]() + val processed = ConcurrentLinkedQueue[Int]() + val core = DeliveryRateLimiterCore[Int](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (_, task) => { pending.add(task); () }, + process = i => { processed.add(i); () }, + holdPermits = () => true, + releasePermits = () => true + ) + core.setRate(100) + + (1 to 5).foreach(limiter.offer(_)) + val armedTick = pending.poll() // the one timer this backlog armed + limiter.pauseDraining() + + val tickThread = worker("p15-paused-tick")(armedTick.run()) + core.synchronized { + tickThread.start() + // The tick has read "paused" and is parked at the core lock this thread holds. + awaitBlockedOnCoreLock(tickThread) + // The Resume lands exactly here - after the decision, before the flag is handed back. + limiter.resumeDraining() + } + tickThread.join(30_000) + + // Nothing new is offered, and nothing else is required to arrive: whatever timers exist + // now are all the backlog will ever get. + var guard = 0 + while !pending.isEmpty && guard < 50 do + pending.poll().run() + guard += 1 + + assertTrue( + processed.asScala.toVector == (1 to 5).toVector, + core.queuedCount == 0 + ) ?? s"processed=${processed.asScala.toVector} queued=${core.queuedCount} timersLeft=${pending.size}" + }, + test("P1.6: a user pause must not swallow the ONLY retry of a failed permit release") { + // The real arbiter and a real consumer, so the assertion is the one that matters: after + // the user resumes, is the consumer actually RUNNING again? + val recording = RecordingConsumer(p0, resumeFailures = 1) + val l = listener() + val target = targetRunner(l, Map(p0 -> recording.consumer)) + + val pending = ArrayBuffer[(Long, Runnable)]() + var clock = 0L + val core = DeliveryRateLimiterCore[String](nowMs = () => clock, payloadBytesOf = _ => 50L * 1024 * 1024) + val limiter = DeliveryRateLimiter[String]( + core = core, + schedule = (delay, task) => { pending += ((delay, task)); () }, + process = _ => (), + holdPermits = () => target.setPermitHold(true), + releasePermits = () => target.setPermitHold(false) + ) + core.setRate(1000) + + def runAll(): Unit = + val tasks = pending.toVector + pending.clear() + tasks.foreach(_._2.run()) + + (1 to 3).foreach(i => limiter.offer(s"m$i")) // 150 MiB queued: over the byte watermark + val pausedByLimiter = recording.isPaused.get + + clock += 1000 + runAll() // the drain empties the queue; the release throws; ONE retry is armed + val pausedAfterFailedRelease = recording.isPaused.get + + // The user pauses BEFORE the retry fires. Armed as an ordinary drain tick, that retry + // is consumed by the paused branch and never happens. + limiter.pauseDraining() + runAll() + + // Resume. The queue is empty - the drain that failed the release emptied it - so + // rearmDrain schedules nothing and no crossing will ever come to notice. + limiter.resumeDraining() + runAll() + runAll() + + assertTrue( + pausedByLimiter, // the vacuity guard: the hold really did engage + pausedAfterFailedRelease, // and the failed release really did leave it held + !recording.isPaused.get // ...and the user's Resume must have got the consumer back + ) ?? s"calls=${recording.permitCalls.asScala.toVector} pausedNow=${recording.isPaused.get}" + }, + test("P1.7: a stuck latest-n correction on ONE stream must not disable backpressure for a hot one") { + // Latest-n corrects each topic's seek overshoot with a PER-TOPIC discard. One stream is + // disconnected or simply idle and never delivers the messages its correction is waiting + // for; the other keeps supplying visible ones. Session-wide eligibility turned that into + // "no permit hold, ever" - so the hot stream's deliveries piled into an unbounded queue + // with the watermark refused on every crossing. + val hot = RecordingConsumer(hotTopic) + val stuck = RecordingConsumer(stuckTopic) + val l = listener() + l.startFromDiscard = StartFromDiscard.perTopic(Map(stuckTopic -> 3L, hotTopic -> 0L)) + val runner = session(targetRunner(l, Map(hotTopic -> hot.consumer, stuckTopic -> stuck.consumer))) + val limiter = runner.deliveryRateLimiter + + // A low rate limit, and the drain held still so the backlog is exactly what arrived: + // whatever the queue holds at the end, nothing was quietly delivered away. + limiter.core.setRate(1) + limiter.pauseDraining() + + val cap = deliveryRateLimitHoldPermitsAboveQueued * 3 + val offered = floodUntilHeld(limiter, hot.consumer, hot.isPaused, l, cap) + + assertTrue( + l.effectiveDiscard.remaining == 3L, // the vacuity guard: the correction really is stuck + hot.isPaused.get, // the hot stream is what backpressure must reach... + !stuck.isPaused.get, // ...and the stream still counting must be left alone + offered < cap, + limiter.queuedCount <= deliveryRateLimitHoldPermitsAboveQueued, + limiter.queuedCount == offered // nothing was dropped to achieve the bound + ) ?? (s"offered=$offered queued=${limiter.queuedCount} bytes=${limiter.core.queuedBytesCount} " + + s"hotPaused=${hot.isPaused.get} stuckPaused=${stuck.isPaused.get}") + }, + test("P1.7: past the ABSOLUTE ceiling the hold happens anyway, suppression and all") { + // A SHARED "skip first n" that never resolves: any stream can supply the next skipped + // message, so per-stream eligibility correctly protects every one of them - and with + // nothing left to hold, only a hard ceiling stands between one hot stream and the heap. + val hot = RecordingConsumer(hotTopic) + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(5) + val runner = session(targetRunner(l, Map(hotTopic -> hot.consumer))) + val limiter = runner.deliveryRateLimiter + + limiter.core.setRate(1) + limiter.pauseDraining() + + val cap = deliveryRateLimitForceHoldPermitsAboveQueued + deliveryRateLimitHoldPermitsAboveQueued + val offered = floodUntilHeld(limiter, hot.consumer, hot.isPaused, l, cap) + + assertTrue( + l.effectiveDiscard.remaining == 5L, // the vacuity guard: the skip really is unresolved + hot.isPaused.get, + offered < cap, + limiter.queuedCount <= deliveryRateLimitForceHoldPermitsAboveQueued, + limiter.queuedCount == offered // held, not dropped + ) ?? s"offered=$offered queued=${limiter.queuedCount} hotPaused=${hot.isPaused.get}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/deliveryBudgetTest.scala b/server/src/test/scala/consumer/session_runner/deliveryBudgetTest.scala new file mode 100644 index 000000000..bf8ad435f --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryBudgetTest.scala @@ -0,0 +1,200 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.basic_message_filter.targets.{BasicMessageFilterTarget, BasicMessageFilterValueTarget} +import _root_.consumer.message_filter.{JsMessageFilter, MessageFilter, MessageFilterChain, MessageFilterChainMode} +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.jdk.CollectionConverters.* + +/** THE DELIVERY BUDGET'S ONE CONTRACT: "pause after n" means EXACTLY n LOADED - counted at the + * send, after every filter - while processed is free to run ahead. + * + * A client-side threshold can only ever be approximate: a whole chunk lands before the client can + * react, and under a rate limit the first chunk is the full one-second burst - "pause after 10" + * showed a hundred. This suite drives the REAL pipeline (real listener, real GraalVM filter + * chain, real runner send path; only the broker and the wire are replaced) and pins that the + * message spending the last budget unit is the last one sent, that everything behind it survives + * for the next resume, and that a filter dropping most messages makes processed exceed loaded + * without disturbing the loaded count's exactness. + */ +object deliveryBudgetTest extends ZIOSpecDefault: + + private val consumerName = "cs-delivery-budget" + private val p0 = "persistent://public/default/delivery-budget-0" + + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "pause" | "resume" => null + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(n: Int): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(1_700_000_000_000L + n) + md.setPartitionKey(n.toString) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"n":$n}""".getBytes("UTF-8")), Schema.BYTES, p0) + msg.setMessageId(new MessageIdImpl(1L, n.toLong, -1)) + msg + + /** Session-level filter keeping only even `n`: every message is PROCESSED, half are LOADED. */ + private val evenOnly: MessageFilterChain = + MessageFilterChain( + isEnabled = true, + isNegated = false, + mode = MessageFilterChainMode.All, + filters = Vector( + MessageFilter( + isEnabled = true, + isNegated = false, + targetField = BasicMessageFilterTarget(target = BasicMessageFilterValueTarget()), + filter = JsMessageFilter(jsCode = "v => v.n % 2 === 0") + ) + ) + ) + + private def targetRunner(consumerListener: ConsumerListener, consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p0))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p0), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = (consumers).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(target: ConsumerSessionTargetRunner, sessionFilter: MessageFilterChain): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = consumerName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = sessionFilter, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + /** Collects what actually went on the wire. Thread-safe: the drain thread writes it. */ + private final class CollectingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + val loadedValues = ConcurrentLinkedQueue[String]() + override def onNext(value: consumerPb.ResumeResponse): Unit = + value.messages.filter(_.value.isDefined).foreach(m => loadedValues.add(m.value.get)) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + + /** Spin until `condition` holds and keeps holding for 200ms, or the deadline passes - the + * drain runs on the runner's real timer thread, so the test has to meet it in real time. */ + private def awaitStable(deadlineMs: Long = 10_000)(condition: => Boolean): Boolean = + val start = System.nanoTime() + var stableSince = -1L + var done = false + while !done && (System.nanoTime() - start) / 1_000_000 < deadlineMs do + if condition then + if stableSince < 0 then stableSince = System.nanoTime() + else if (System.nanoTime() - stableSince) / 1_000_000 >= 200 then done = true + else stableSince = -1L + if !done then Thread.sleep(20) + done + + def spec = suite("the delivery budget: exactly n loaded")( + test("the send spending the last unit is the last one sent; the tail survives for the next resume") { + val recording = RecordingConsumer(p0) + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val runner = session(targetRunner(listener, Map(p0 -> recording.consumer)), evenOnly) + val observer = CollectingObserver() + + // Budget 3, no rate limit: full speed until the third LOADED message, then a hard stop. + runner.resume(observer, isDebug = false, includeConsumerStats = true, maxMessagesPerSecond = 0, maxMessagesToDeliver = 3) + (1 to 20).foreach(n => listener.received(recording.consumer, message(n))) + + val firstStop = awaitStable() { + observer.loadedValues.size == 3 && runner.deliveryRateLimiter.queuedCount > 0 + } + val loadedAfterFirst = observer.loadedValues.asScala.toVector + val processedAfterFirst = runner.numMessageProcessed + val queuedAfterFirst = runner.deliveryRateLimiter.queuedCount + + // Play again with the same budget: the NEXT three evens, in order, from exactly where + // the drain stopped - nothing lost, nothing repeated. + runner.resume(observer, isDebug = false, includeConsumerStats = true, maxMessagesPerSecond = 0, maxMessagesToDeliver = 3) + val secondStop = awaitStable() { observer.loadedValues.size == 6 } + val loadedAfterSecond = observer.loadedValues.asScala.toVector + + assertTrue(firstStop) && + assertTrue(loadedAfterFirst.map(v => io.circe.parser.parse(v).toOption.get.hcursor.get[Int]("n").toOption.get) == Vector(2, 4, 6)) && + // Processed ran AHEAD of loaded - the filter read the odd ones too - but the loaded + // count stopped at exactly the budget. "100 may be processed, but only 10 loaded." + assertTrue(processedAfterFirst == 6L) && + assertTrue(queuedAfterFirst == 14) && + assertTrue(secondStop) && + assertTrue(loadedAfterSecond.map(v => io.circe.parser.parse(v).toOption.get.hcursor.get[Int]("n").toOption.get) == Vector(2, 4, 6, 8, 10, 12)) + }, + test("a budget WITH a rate limit: the one-second burst cannot blow past n") { + // The user's exact screenshot: rate 100, pause after 10. The bucket starts full, so + // without the budget the first drain hands out 100 at once - the budget must cap that + // very first batch. + val recording = RecordingConsumer(p0) + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val runner = session(targetRunner(listener, Map(p0 -> recording.consumer)), MessageFilterChain.empty) + val observer = CollectingObserver() + + runner.resume(observer, isDebug = false, includeConsumerStats = true, maxMessagesPerSecond = 100, maxMessagesToDeliver = 10) + (1 to 100).foreach(n => listener.received(recording.consumer, message(n))) + + val stopped = awaitStable() { observer.loadedValues.size == 10 } + assertTrue(stopped) && assertTrue(observer.loadedValues.size == 10) + } + ) @@ TestAspect.sequential @@ TestAspect.withLiveClock diff --git a/server/src/test/scala/consumer/session_runner/deliveryLifecycleRaceTest.scala b/server/src/test/scala/consumer/session_runner/deliveryLifecycleRaceTest.scala new file mode 100644 index 000000000..9f8833cc5 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryLifecycleRaceTest.scala @@ -0,0 +1,517 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.basic_message_filter.targets.{BasicMessageFilterTarget, BasicMessageFilterValueTarget} +import _root_.consumer.message_filter.{JsMessageFilter, MessageFilter, MessageFilterChain, MessageFilterChainMode} +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.google.rpc.code.Code +import com.google.rpc.status.Status +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** WHAT A DELIVERY OWES THE BROKER WHEN THE PLAY UNDERNEATH IT MOVES. + * + * Every message this session takes off the broker is either SHOWN to a client and acknowledged, or + * handed back for redelivery. Three lifecycle races broke that bargain, and each loses something + * different: + * + * - a send into an ENDED stream (`failAndComplete` while a delivery was in flight) reported + * success without writing anything, so the caller acknowledged a message no browser ever saw - + * a persistent message gone from the session's subscription for good; + * - the play-generation gate sat in the SESSION callback, after the target handler had already + * advanced the counters, deserialized, leased the JS context and run the user's filters - so a + * second Play made stateful JavaScript observe the same message twice; + * - a FAILED send re-entered that whole pipeline on the redelivery (and on Guaranteed's in-place + * retry), so the browser could receive a message exactly once while the counters and the JS + * state said two or three. + * + * Everything runs offline: `ConsumerListener.received`, `deliverNow`, the guaranteed delivery + * pump, the production target handler installed by `ConsumerSessionTargetRunner.resume` and the + * real session `onNext` closure. Only the broker is replaced, by proxy consumers and hand-built + * `MessageImpl`s. Every interleaving is pinned by latches - never by a sleep, and never by + * repetition until something happens to land. + */ +object deliveryLifecycleRaceTest extends ZIOSpecDefault: + + private val consumerName = "cs-delivery-race-0" + private def topic(i: Int): String = s"persistent://public/default/cs-delivery-race-$i" + private def sid(i: Int): String = startFromStreamId(consumerName, topic(i)) + + /** A connected consumer that records what was acknowledged and what was handed back - the two + * broker-visible dispositions every assertion here is about. */ + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + // A FAR-FUTURE recorded end: a guaranteed resume re-captures the replay + // boundary through this call, and these tests are about mid-replay + // lifecycle races - every hand-built message (small entry ids) must stay + // inside the boundary and no stream may read as finished. + case "getLastMessageIds" => + java.util.List.of[org.apache.pulsar.client.api.MessageId](new MessageIdImpl(1L, 1_000_000L, -1)) + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + /** THE STATEFUL TARGET FILTER, and the whole point of using real JavaScript here: it counts its + * own invocations in a JS global that survives the message, so "the pipeline ran twice for one + * message" is a fact the test reads back out of the session's context rather than infers. It + * retains everything, so the counting is independent of what the filter decides. + * + * A `def`, and that is load-bearing: `JsMessageFilter` memoizes the compiled function against + * the FIRST polyglot context it is evaluated in, so one shared instance would keep counting + * into the first fixture's context and every later fixture would read a run count of zero. */ + private def countingFilter: MessageFilterChain = + MessageFilterChain( + isEnabled = true, + isNegated = false, + mode = MessageFilterChainMode.All, + filters = Vector( + MessageFilter( + isEnabled = true, + isNegated = false, + targetField = BasicMessageFilterTarget(target = BasicMessageFilterValueTarget()), + filter = JsMessageFilter(jsCode = "v => { globalThis.__runs = (globalThis.__runs || 0) + 1; return true }") + ) + ) + ) + + /** Records what the client received, and can be made to FAIL its first write - which is what a + * cancelled or backpressured call does, and the trigger for every retry path here. */ + private final class RecordingObserver(failWrites: Int = 0) extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val frames = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + private val writesToFail = java.util.concurrent.atomic.AtomicInteger(failWrites) + val completed = java.util.concurrent.atomic.AtomicBoolean(false) + + override def onNext(value: consumerPb.ResumeResponse): Unit = + if writesToFail.getAndUpdate(n => math.max(0, n - 1)) > 0 then + throw new IllegalStateException("the client's call could not take this write") + frames.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = completed.set(true) + + def received: Vector[consumerPb.ResumeResponse] = frames.asScala.toVector + /** Messages the browser actually got - a status-only frame is not a delivery. */ + def deliveredMessages: Vector[consumerPb.Message] = received.flatMap(_.messages) + + /** A real session over `topicCount` proxy consumers, with the real target handler and the real + * session `onNext`. `guaranteed` swaps the pass-through ordering layer for the production + * Guaranteed barrier, whose retry is in-place rather than through the broker. */ + private final class Fixture(sessionName: String, topicCount: Int = 1, guaranteed: Boolean = false): + val pool: ConsumerSessionContextPool = ConsumerSessionContextPool() + val listener: ConsumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.startAcceptingNewMessages() + if guaranteed then + // The CREATE-TIME boundary, exactly as production arms it: far-future recorded ends, + // matching what the proxies' getLastMessageIds answers, so every hand-built message + // (small entry ids) stays mid-replay and no stream reads as finished. Since the + // create-is-the-first-Play fix the first Play CONSUMES this boundary instead of + // re-reading the broker, so the fixture must carry it here - relying on a resume-time + // re-capture to install it left the layer boundary-less and every stream finished. + val midReplayStreams = Vector.tabulate(topicCount)(i => StartFromStream(sid(i), EntryPosition(1L, 1_000_000L, -1, 1))) + listener.startFromOrdering = new StartFromOrdering[HeldMessage]( + Some(GlobalSkipMerge[HeldMessage]( + streamIds = Vector.tabulate(topicCount)(sid), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => 0L, + policy = OrderingPolicy.GuaranteedOnly, + graceMs = 500L + )), + midReplayStreams.map(s => s.id -> s).toMap + ) + + val consumers: Vector[RecordingConsumer] = Vector.tabulate(topicCount)(i => RecordingConsumer(topic(i))) + private val consumersByTopic: Map[String, Consumer[Array[Byte]]] = + Vector.tabulate(topicCount)(i => topic(i) -> consumers(i).consumer).toMap + + val target: ConsumerSessionTargetRunner = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumersByTopic.keys.toVector)), + messageFilterChain = countingFilter, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumersByTopic.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumersByTopic, + pauseArbiters = consumersByTopic.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + val runner: ConsumerSessionRunner = ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + /** One message arriving on a Pulsar listener thread, through the real delivery path. */ + def deliver(partition: Int, key: String, publishTime: Long, entryId: Long): Unit = + listener.received(consumers(partition).consumer, message(topic(partition), key, publishTime, entryId)) + + /** How many times the user's stateful filter has run, read straight out of the session's + * JS context - the oracle for "the pipeline ran again". */ + def jsRuns: Int = pool.withContext(0)(_.runCode("globalThis.__runs || 0")).trim.toInt + + def acknowledged: Vector[String] = consumers.flatMap(_.acknowledged.asScala.toVector) + def handedBack: Vector[String] = consumers.flatMap(_.handedBack.asScala.toVector) + + /** Put a LATCH between `deliverNow` and the production target handler, so a delivery can be + * held mid-flight while a lifecycle event lands. Installed after `resume`, which is what + * installs the handler being wrapped; the wrapper keeps its own reference, so a later + * resume replacing the field cannot rescue a delivery already inside it - exactly like a + * Pulsar listener thread descheduled just as a second Play arrives. */ + def latchHandler(entered: CountDownLatch, release: CountDownLatch): Unit = + val installed = listener.targetMessageHandler.onNext + listener.targetMessageHandler.onNext = msg => + entered.countDown() + release.await(60, TimeUnit.SECONDS) + installed(msg) + + private val await = 60_000L + + private val terminalStreamSuite = suite("a send into a stream that has ENDED is a failed delivery, not a silent success")( + test("A DELIVERY IN FLIGHT ACROSS failAndComplete IS HANDED BACK - never acknowledged, never counted as read") { + // The shape from production: one target's Resume throws, the service ends the client's + // stream through `failAndComplete`, and another target's listener thread is at that + // moment inside a delivery. `sendResponse` treated the ended stream as a successful + // no-op, so `deliverNow` went straight on to acknowledge - and a persistent message + // left this session's subscription without a browser ever seeing it. + // + // The handler here calls `sendResponse` directly: this test is about the SEND crossing + // the terminal boundary and what `deliverNow` then does with the broker, and wiring it + // this way keeps the claim independent of the generation gate in front of it (pinned + // separately below). + val f = Fixture("cs-terminal-inflight") + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + + val inFlight = CountDownLatch(1) + val release = CountDownLatch(1) + f.listener.targetMessageHandler.onNext = _ => + inFlight.countDown() + release.await(60, TimeUnit.SECONDS) + f.runner.sendResponse(observer, Seq(consumerPb.Message(numMessageProcessed = 1L)), Vector.empty) + + val delivering = worker("pulsar-listener-0")(f.deliver(0, "m1", 100L, 0L)) + delivering.start() + val reachedTheSend = inFlight.await(30, TimeUnit.SECONDS) + + // A later target could not be resumed; the service ends the stream through the runner. + f.runner.failAndComplete(Status(code = Code.FAILED_PRECONDITION.value, message = "target 1 could not be resumed")) + release.countDown() + delivering.join(await) + + val lostToTheEndedStream = observer.deliveredMessages + + // The client creates the session again, and the broker redelivers what was handed back. + val replacement = Fixture("cs-terminal-inflight-2") + val secondObserver = RecordingObserver() + replacement.runner.resume(secondObserver, isDebug = false) + replacement.deliver(0, "m1", 100L, 0L) + + assertTrue( + reachedTheSend, + f.acknowledged.isEmpty, // NOT acknowledged: nothing was written for it + f.handedBack == Vector("m1"), // handed back for redelivery instead + f.listener.consumedBounds.isEmpty, // and Topic Positions did not advance over it + lostToTheEndedStream.isEmpty, + secondObserver.deliveredMessages.size == 1, // exactly once on the replacement + replacement.acknowledged == Vector("m1"), + replacement.handedBack.isEmpty + ) ?? (s"acknowledged=${f.acknowledged} handedBack=${f.handedBack} bounds=${f.listener.consumedBounds.keySet} " + + s"lost=${lostToTheEndedStream.size} replacementFrames=${secondObserver.deliveredMessages.size} " + + s"replacementAcked=${replacement.acknowledged}") + }, + test("ENDING THE STREAM INVALIDATES THE PLAY FIRST, so the doomed delivery never runs the pipeline") { + // The other half of the same fix. Refusing at the send keeps the MESSAGE, but the + // delivery would still have spent the whole pipeline getting there - both processed + // counters, the deserialization, the JS lease and the user's stateful filters - all for + // a response that can no longer be written to anybody. `failAndComplete` bumps the play + // generation before it ends the stream, exactly as `stop` does, so the in-flight + // handler dies at the target's gate instead. + val f = Fixture("cs-terminal-generation") + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + + val inFlight = CountDownLatch(1) + val release = CountDownLatch(1) + f.latchHandler(inFlight, release) + + val delivering = worker("pulsar-listener-0")(f.deliver(0, "m1", 100L, 0L)) + delivering.start() + val inFlightNow = inFlight.await(30, TimeUnit.SECONDS) + f.runner.failAndComplete(Status(code = Code.FAILED_PRECONDITION.value, message = "target 1 could not be resumed")) + release.countDown() + delivering.join(await) + + assertTrue( + inFlightNow, + f.jsRuns == 0, // no user JavaScript ran for a delivery nobody could receive + f.target.stats.messageProcessed.get == 0L, + f.runner.numMessageProcessed == 0L, + f.acknowledged.isEmpty, + f.handedBack == Vector("m1"), + observer.deliveredMessages.isEmpty + ) ?? (s"jsRuns=${f.jsRuns} processed=${f.target.stats.messageProcessed.get} " + + s"acked=${f.acknowledged} handedBack=${f.handedBack}") + }, + test("a data send into an ended stream REFUSES, so its caller takes the failure path") { + // The unit-level statement of the same rule. A progress-only frame may still be dropped + // silently - losing it costs a progress bar - but a frame carrying a message is a + // delivery, and a delivery that was not written must never look like one that was. + val f = Fixture("cs-terminal-send") + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.runner.stop() + + val dataSendRefused = Try(f.runner.sendResponse(observer, Seq(consumerPb.Message()), Vector.empty)).isFailure + val progressPushSurvived = Try(f.runner.sendResponse(observer, Seq.empty, Vector.empty)).isSuccess + + assertTrue( + dataSendRefused, + progressPushSurvived, + observer.completed.get, + observer.deliveredMessages.isEmpty + ) ?? s"dataSendRefused=$dataSendRefused progressPushSurvived=$progressPushSurvived frames=${observer.received.size}" + } + ) + + private val generationGateSuite = suite("a superseded play does no work at all - not even the target's")( + test("A SECOND PLAY CROSSING AN IN-FLIGHT DELIVERY MUTATES NOTHING, and the message is shown exactly once afterwards") { + // The gate used to live in the SESSION callback, which the target reaches only after + // incrementing both processed counters, deserializing the message, leasing the JS + // context and running the target's filters. So a second Play (or a Stop) crossing an + // in-flight delivery left the stale attempt's state changes behind: the message was + // nacked and redelivered, and the user's stateful JavaScript saw it TWICE for one row + // on screen. + val f = Fixture("cs-superseded") + val first = RecordingObserver() + f.runner.resume(first, isDebug = false) + + val inFlight = CountDownLatch(1) + val release = CountDownLatch(1) + f.latchHandler(inFlight, release) + + val delivering = worker("pulsar-listener-0")(f.deliver(0, "m1", 100L, 0L)) + delivering.start() + val inFlightNow = inFlight.await(30, TimeUnit.SECONDS) + + // The user pressed Play again while that delivery was mid-flight. + val second = RecordingObserver() + f.runner.resume(second, isDebug = false) + release.countDown() + delivering.join(await) + + val jsRunsAfterStaleAttempt = f.jsRuns + val processedAfterStaleAttempt = f.target.stats.messageProcessed.get + val sessionProcessedAfterStaleAttempt = f.runner.numMessageProcessed + + // The broker redelivers what the stale attempt handed back; THIS play delivers it. + f.deliver(0, "m1", 100L, 0L) + + assertTrue( + inFlightNow, + jsRunsAfterStaleAttempt == 0, // the superseded attempt ran no user JavaScript + processedAfterStaleAttempt == 0L, // and moved no counter + sessionProcessedAfterStaleAttempt == 0L, + f.handedBack == Vector("m1"), // it was handed back instead + first.deliveredMessages.isEmpty, // nothing reached the replaced stream + f.jsRuns == 1, // ONE run of the stateful filter for one message, in total + f.target.stats.messageProcessed.get == 1L, + f.runner.numMessageProcessed == 1L, + f.runner.numMessageSent == 1L, + second.deliveredMessages.size == 1, + f.acknowledged == Vector("m1") + ) ?? (s"staleAttempt: jsRuns=$jsRunsAfterStaleAttempt processed=$processedAfterStaleAttempt " + + s"sessionProcessed=$sessionProcessedAfterStaleAttempt; afterwards: jsRuns=${f.jsRuns} " + + s"processed=${f.target.stats.messageProcessed.get} sent=${f.runner.numMessageSent} " + + s"acked=${f.acknowledged} handedBack=${f.handedBack} frames=${second.deliveredMessages.size}") + }, + test("a STOP crossing an in-flight delivery is the same rule: nothing mutated, the message handed back") { + // Counters only, deliberately: `stop` also CLOSES the session's JS contexts, so the + // run-count global cannot be read back afterwards - and a delivery that reached the + // pool at all would have died on the closed context rather than at the gate, which is + // exactly the reading this asserts against. + val f = Fixture("cs-stopped-midflight") + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + + val inFlight = CountDownLatch(1) + val release = CountDownLatch(1) + f.latchHandler(inFlight, release) + + val delivering = worker("pulsar-listener-0")(f.deliver(0, "m1", 100L, 0L)) + delivering.start() + val inFlightNow = inFlight.await(30, TimeUnit.SECONDS) + Try(f.runner.stop()) + release.countDown() + delivering.join(await) + + assertTrue( + inFlightNow, + f.target.stats.messageProcessed.get == 0L, + f.runner.numMessageProcessed == 0L, + f.acknowledged.isEmpty, + f.handedBack == Vector("m1"), + observer.deliveredMessages.isEmpty + ) ?? (s"processed=${f.target.stats.messageProcessed.get} " + + s"acked=${f.acknowledged} handedBack=${f.handedBack}") + } + ) + + private val failedSendSuite = suite("a retry re-SENDS - it does not re-run the pipeline")( + test("BEST EFFORT: a failed send then a redelivery leaves the stateful filter run ONCE and the counters at one") { + // The browser can receive a message exactly once and still see the counters - and the + // user's accumulated JS state - as if it had been processed twice, because the failed + // attempt had already run the whole target and session pipeline before the send was + // known to have failed. What the retry owes is the SAME response, sent again. + val f = Fixture("cs-failed-send-best-effort") + val observer = RecordingObserver(failWrites = 1) + f.runner.resume(observer, isDebug = false) + + f.deliver(0, "m1", 100L, 0L) // the send fails; the message is handed back + val handedBackAfterFailure = f.handedBack + val acknowledgedAfterFailure = f.acknowledged + + f.deliver(0, "m1", 100L, 0L) // the broker redelivers it + + assertTrue( + handedBackAfterFailure == Vector("m1"), + acknowledgedAfterFailure.isEmpty, + f.jsRuns == 1, // ONE run of the user's stateful filter for one message + f.target.stats.messageProcessed.get == 1L, + f.runner.numMessageProcessed == 1L, + f.runner.numMessageSent == 1L, + observer.deliveredMessages.size == 1, // and exactly one row on screen + f.acknowledged == Vector("m1"), + f.runner.preparedDeliveryCount == 0 // the memo is released by the send that took + ) ?? (s"jsRuns=${f.jsRuns} processedByTarget=${f.target.stats.messageProcessed.get} " + + s"processed=${f.runner.numMessageProcessed} sent=${f.runner.numMessageSent} " + + s"frames=${observer.deliveredMessages.size} acked=${f.acknowledged} handedBack=${f.handedBack} " + + s"pending=${f.runner.preparedDeliveryCount}") + }, + test("GUARANTEED: the in-place retry of a failed send is a re-send, not a second pass") { + // Guaranteed never hands a failed head back to the broker - it keeps it and retries it + // on the next pump. Same rule, different retry mechanism: the pipeline must not run + // again, so nothing is counted twice and no stateful filter sees the message twice. + val f = Fixture("cs-failed-send-guaranteed", topicCount = 2, guaranteed = true) + val observer = RecordingObserver(failWrites = 1) + f.runner.resume(observer, isDebug = false) + + f.deliver(0, "m1", 100L, 0L) // held: stream 1 has not spoken yet + f.deliver(1, "m2", 200L, 0L) // m1 is now the safe head - and its send fails + + val acknowledgedAfterFailure = f.acknowledged + val jsRunsAfterFailure = f.jsRuns + + f.listener.pumpGuaranteedDelivery() // the retry tick + + assertTrue( + acknowledgedAfterFailure.isEmpty, + jsRunsAfterFailure == 1, + f.handedBack.isEmpty, // Guaranteed keeps the head; the broker is never asked again + f.jsRuns == 1, + f.target.stats.messageProcessed.get == 1L, + f.runner.numMessageProcessed == 1L, + f.runner.numMessageSent == 1L, + observer.deliveredMessages.size == 1, + f.acknowledged == Vector("m1"), + f.runner.preparedDeliveryCount == 0 + ) ?? (s"jsRuns=${f.jsRuns} (after failure $jsRunsAfterFailure) " + + s"processedByTarget=${f.target.stats.messageProcessed.get} sent=${f.runner.numMessageSent} " + + s"frames=${observer.deliveredMessages.size} acked=${f.acknowledged} handedBack=${f.handedBack} " + + s"pending=${f.runner.preparedDeliveryCount}") + }, + test("a retry under a NEW play is not replayed - the superseded attempt's response belongs to the stream that is gone") { + // The two fixes compose in one direction that matters: a prepared response is only ever + // re-sent to the play that prepared it. A second Play means new stats, a new observer + // and possibly a new client, so the retry there re-runs the pipeline once - and the + // counters end at one run per play, never at a stale frame from the previous one. + val f = Fixture("cs-retry-across-plays") + val first = RecordingObserver(failWrites = 1) + f.runner.resume(first, isDebug = false) + f.deliver(0, "m1", 100L, 0L) // prepared under play 1, send failed, handed back + + val second = RecordingObserver() + f.runner.resume(second, isDebug = false) // play 2 + f.deliver(0, "m1", 100L, 0L) // the redelivery lands under the new play + + assertTrue( + f.jsRuns == 2, // once per play - the first play's run is not undoable + f.runner.numMessageSent == 2L, + first.deliveredMessages.isEmpty, + second.deliveredMessages.size == 1, + f.acknowledged == Vector("m1") + ) ?? (s"jsRuns=${f.jsRuns} sent=${f.runner.numMessageSent} first=${first.deliveredMessages.size} " + + s"second=${second.deliveredMessages.size} acked=${f.acknowledged}") + } + ) + + def spec = suite(this.getClass.toString)(terminalStreamSuite, generationGateSuite, failedSendSuite) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/deliveryOrderSwitchTest.scala b/server/src/test/scala/consumer/session_runner/deliveryOrderSwitchTest.scala new file mode 100644 index 000000000..5769130bc --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryOrderSwitchTest.scala @@ -0,0 +1,362 @@ +package consumer.session_runner + +import consumer.session_config.MessageDeliveryOrder +import zio.test.* + +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch} +import java.util.concurrent.atomic.AtomicBoolean +import scala.jdk.CollectionConverters.* + +/** CHANGING THE DELIVERY ORDER OF A SESSION THAT IS ALREADY RUNNING. + * + * Guaranteed - the product default since 2026-08-11 - can wait on an + * idle source forever. The wait is disclosed; this is what makes it ACTIONABLE - one call + * switches the live session to Best effort and the messages the barrier was holding are + * RELEASED, in the new order, exactly once. Nothing is re-read from the broker, nothing is + * dropped, nothing arrives twice. + * + * Two levels are pinned here: the pure decision matrix (which direction is honoured, which is + * refused and why), and the merge's own switch (what happens to the held set, the counted budget + * and the flow-control marks). The listener/consumer path is pinned in + * [[liveDeliveryOrderSwitchTest]] and the RPC surface in `consumerServiceDeliveryOrderTest`. + * + * Time is injected and moves only when a test says so. + */ +object deliveryOrderSwitchTest extends ZIOSpecDefault: + + private val a = "cs@persistent://public/default/switch-a" + private val b = "cs@persistent://public/default/switch-b" + private val c = "cs@persistent://public/default/switch-c" + + private val graceMs = 500L + + private final class MonoClock: + @volatile var monoMs: Long = 0L + def advance(ms: Long): Unit = monoMs += ms + + /** The refusal reason, or a legible complaint when the switch was not refused at all. */ + private def reasonOf(switch: DeliveryOrderSwitch): String = switch match + case DeliveryOrderSwitch.Refused(reason) => reason + case other => s"NOT REFUSED: $other" + + private final class Fixture( + streamIds: Vector[String], + policy: OrderingPolicy = OrderingPolicy.GuaranteedOnly, + budget: Long = 0, + pauseStreamAt: Int = startFromMergePauseStreamAt, + resumeStreamAt: Int = startFromMergeResumeStreamAt + ): + val clock = MonoClock() + private var nextEntryId = Map.empty[String, Long].withDefaultValue(0L) + val merge = GlobalSkipMerge[String]( + streamIds = streamIds, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(budget), + nowMs = () => clock.monoMs, + pauseStreamAt = pauseStreamAt, + resumeStreamAt = resumeStreamAt, + policy = policy, + graceMs = graceMs + ) + private val out = Vector.newBuilder[(String, StartFromOutcome)] + + def offer(streamId: String, publishTime: Long, value: String): Unit = + val entryId = nextEntryId(streamId) + nextEntryId = nextEntryId.updated(streamId, entryId + 1) + val key = MessageOrderKey(publishTime, streamId, ledgerId = 1L, entryId = entryId, batchIndex = -1) + out ++= merge.offer(streamId, key, atBacklogEnd = false, payload = value) + + /** The unit-level stand-in for the listener's guaranteed pump: peek and commit until + * nothing is safe, every send succeeding. */ + def pumpAll(): Vector[String] = + val delivered = Vector.newBuilder[String] + var going = true + while going do + merge.peekGuaranteed() match + case Some(v) => delivered += v; merge.commitGuaranteed() + case None => going = false + delivered.result() + + def switchToBestEffort(): Vector[String] = + val released = merge.relaxGuaranteedToBestEffort() + out ++= released + released.collect { case (v, StartFromOutcome.Deliver) => v } + + def sweep(): Unit = out ++= merge.sweepStalled() + + def delivered: Vector[String] = out.result().collect { case (v, StartFromOutcome.Deliver) => v } + def dropped: Vector[String] = out.result().collect { case (v, StartFromOutcome.Drop) => v } + + def spec = suite(this.getClass.toString)( + // ------------------------------------------------------------------ the decision matrix + test("the ONE honoured direction is Guaranteed -> Best effort") { + assertTrue( + deliveryOrderSwitchFor(MessageDeliveryOrder.Guaranteed, MessageDeliveryOrder.BestEffort) + == DeliveryOrderSwitch.RelaxToBestEffort + ) + }, + test("asking for the order the session already delivers in is a no-op, not an error") { + // Two clients, or one impatient one, must not turn a repeated click into a failure. + assertTrue( + deliveryOrderSwitchFor(MessageDeliveryOrder.Guaranteed, MessageDeliveryOrder.Guaranteed) == DeliveryOrderSwitch.AlreadyThere, + deliveryOrderSwitchFor(MessageDeliveryOrder.BestEffort, MessageDeliveryOrder.BestEffort) + == DeliveryOrderSwitch.AlreadyThere, + deliveryOrderSwitchFor(MessageDeliveryOrder.AsReceived, MessageDeliveryOrder.AsReceived) == DeliveryOrderSwitch.AlreadyThere + ) + }, + test("PROMOTION TO GUARANTEED IS REFUSED, because it cannot be honestly delivered mid-stream") { + // Guaranteed promises that Dekaf introduced no cross-stream disorder into what the user + // has SEEN. A session already running under a weaker order has emitted past silent + // streams and cannot un-emit; granting the promise now would be a lie about messages + // already on screen. The refusal must say what to do instead. + val fromBestEffort = deliveryOrderSwitchFor(MessageDeliveryOrder.BestEffort, MessageDeliveryOrder.Guaranteed) + val fromFastest = deliveryOrderSwitchFor(MessageDeliveryOrder.AsReceived, MessageDeliveryOrder.Guaranteed) + assertTrue( + fromBestEffort.isInstanceOf[DeliveryOrderSwitch.Refused], + fromFastest.isInstanceOf[DeliveryOrderSwitch.Refused], + reasonOf(fromBestEffort).contains("Start the session again") + ) ?? s"fromBestEffort=$fromBestEffort fromFastest=$fromFastest" + }, + test("SWITCHING TO FASTEST IS REFUSED: it is the absence of the merge, not a weaker order") { + // The same layer also resolves a counted start-from cut, so removing it mid-session + // could change WHICH messages the session shows - the exact failure that ruled out + // recreating the session. Fastest stays a configuration choice. + val fromGuaranteed = deliveryOrderSwitchFor(MessageDeliveryOrder.Guaranteed, MessageDeliveryOrder.AsReceived) + val fromBestEffort = deliveryOrderSwitchFor(MessageDeliveryOrder.BestEffort, MessageDeliveryOrder.AsReceived) + assertTrue( + fromGuaranteed.isInstanceOf[DeliveryOrderSwitch.Refused], + fromBestEffort.isInstanceOf[DeliveryOrderSwitch.Refused], + reasonOf(fromGuaranteed).contains("Play") + ) ?? s"fromGuaranteed=$fromGuaranteed fromBestEffort=$fromBestEffort" + }, + + // ------------------------------------------------------------- the merge's own switch + test("THE HELD SET IS RELEASED, EXACTLY ONCE, IN BEST-EFFORT ORDER - not dropped, not re-read") { + // The stall this exists for: c never speaks, so the guaranteed barrier holds + // everything the other two streams delivered, for as long as it takes. Arrival order + // is deliberately not publish-time order, so a release that merely dumped the queues + // would be visibly wrong. + val f = Fixture(Vector(a, b, c)) + f.offer(a, 100, "a-100") + f.offer(a, 300, "a-300") + f.offer(b, 200, "b-200") + f.offer(b, 400, "b-400") + f.offer(a, 500, "a-500") + val pumpedWhileGuaranteed = f.pumpAll() + val heldBehindTheSilentStream = f.merge.heldCount + + // The user has been staring at a stalled session; every held head is long past its + // best-effort residence by the time they click. + f.clock.advance(graceMs + 1) + val released = f.switchToBestEffort() + + assertTrue( + pumpedWhileGuaranteed.isEmpty, // the barrier delivered nothing while c was silent + heldBehindTheSilentStream == 5, + released == Vector("a-100", "b-200", "a-300", "b-400", "a-500"), + f.delivered == released, // exactly once: the release IS the whole delivery + f.delivered.distinct == f.delivered, + f.dropped.isEmpty, // nothing was thrown away to make the switch + f.merge.heldCount == 0, + !f.merge.isGuaranteedOrdering, // the barrier is gone... + f.merge.isContinuousOrdering, // ...but the session is still ordered + f.merge.peekGuaranteed().isEmpty + ) ?? s"pumped=$pumpedWhileGuaranteed released=$released delivered=${f.delivered} held=${f.merge.heldCount}" + }, + test("messages arriving AFTER the switch obey the new rules - held for the grace, then out") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") // b silent: the barrier holds it + f.clock.advance(graceMs + 1) + val released = f.switchToBestEffort() + + // A fresh arrival on the still-silent-peer side: best effort holds it for its own + // residence and no longer waits for b indefinitely. + f.offer(a, 700, "a-700") + val heldWhileFresh = f.delivered + f.clock.advance(graceMs + 1) + f.sweep() + + assertTrue( + released == Vector("a-100"), + heldWhileFresh == Vector("a-100"), + f.delivered == Vector("a-100", "a-700"), + f.merge.heldCount == 0 + ) ?? s"released=$released delivered=${f.delivered}" + }, + test("a peek left in flight across the switch is released ONCE, and a late commit cannot double it") { + // Defence in depth: production holds the ordering lock across peek/commit, so a switch + // cannot land between them. If one ever did, the head must not both be released by the + // switch and popped again by the stale commit. + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + f.offer(b, 200, "b-200") + val peeked = f.merge.peekGuaranteed() + f.clock.advance(graceMs + 1) + val released = f.switchToBestEffort() + f.merge.commitGuaranteed() // stale: the barrier it belonged to no longer exists + + assertTrue( + peeked.contains("a-100"), + released == Vector("a-100", "b-200"), + f.delivered.distinct == f.delivered, + f.merge.heldCount == 0 + ) ?? s"peeked=$peeked released=$released held=${f.merge.heldCount}" + }, + test("switching MID-CUT leaves the counted start-from budget exactly where it was") { + // The cut is defined over the merged stream and must stay exact whatever the delivery + // order does afterwards: the switch may not spend, refund or skip a single unit. + val f = Fixture(Vector(a, b), policy = OrderingPolicy.ExactCutThenGuaranteed, budget = 3) + f.offer(a, 10, "a-10") + f.offer(b, 20, "b-20") // resolves a-10 as the first drop + f.offer(a, 30, "a-30") // resolves b-20 as the second + val remainingBeforeSwitch = f.merge.progressDiscard.map(_.remaining) + + val releasedMidCut = f.switchToBestEffort() + val remainingAfterSwitch = f.merge.progressDiscard.map(_.remaining) + + // The cut carries on exactly as before, and only then does ordering continue. + f.offer(b, 40, "b-40") + f.clock.advance(graceMs + 1) + f.sweep() + + assertTrue( + remainingBeforeSwitch.contains(1L), + releasedMidCut.isEmpty, // the exact rule still waits for b - the switch did not skip it + remainingAfterSwitch.contains(1L), + f.dropped == Vector("a-10", "b-20", "a-30"), // exactly the 3 the user asked to skip + f.merge.progressDiscard.map(_.remaining).contains(0L), + f.delivered == Vector("b-40"), + !f.merge.isGuaranteedOrdering, + f.merge.isContinuousOrdering + ) ?? s"before=$remainingBeforeSwitch after=$remainingAfterSwitch dropped=${f.dropped} delivered=${f.delivered}" + }, + test("flow control survives the switch: what the watermark paused is released when the drain empties it") { + val f = Fixture(Vector(a, b), pauseStreamAt = 5, resumeStreamAt = 1) + (1 to 6).foreach(i => f.offer(a, 100L + i, s"a-$i")) // b is blind; a races past its watermark + val pausedUnderGuaranteed = f.merge.desiredPausedStreams + f.clock.advance(graceMs + 1) + val released = f.switchToBestEffort() + + assertTrue( + pausedUnderGuaranteed == Set(a), + released.size == 6, + released == (1 to 6).map(i => s"a-$i").toVector, + f.merge.desiredPausedStreams.isEmpty, // the drain released the backpressure + f.merge.heldCount == 0 + ) ?? s"pausedUnderGuaranteed=$pausedUnderGuaranteed released=$released desired=${f.merge.desiredPausedStreams}" + }, + test("relaxing a layer that is not guaranteed changes nothing at all") { + val bestEffort = Fixture(Vector(a, b), policy = OrderingPolicy.BestEffortOnly) + bestEffort.offer(a, 100, "a-100") + val released = bestEffort.switchToBestEffort() + val passThrough = StartFromOrdering.passThrough[String] + + assertTrue( + released.isEmpty, + bestEffort.merge.heldCount == 1, // still held by its own residence, untouched + bestEffort.merge.isContinuousOrdering, + !bestEffort.merge.isGuaranteedOrdering, + passThrough.relaxGuaranteedToBestEffort().isEmpty + ) ?? s"released=$released held=${bestEffort.merge.heldCount}" + }, + test("THE SWITCH IS SAFE AGAINST CONCURRENT DELIVERY: nothing is lost and nothing is doubled") { + // The production discipline modelled exactly: every decision - an offer, a pump cycle, + // the switch itself - runs under the session's ORDERING lock, and the merge's own + // monitor nests inside it. Offer threads, a delivery pump and the switch all run at + // once; afterwards every single message must have come out exactly once. + val streams = Vector(a, b, c) + val perStream = 400 + val orderingLock = new Object + val clock = MonoClock() + val merge = GlobalSkipMerge[String]( + streamIds = streams, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.monoMs, + policy = OrderingPolicy.GuaranteedOnly, + graceMs = graceMs + ) + val delivered = ConcurrentLinkedQueue[String]() + val otherOutcomes = ConcurrentLinkedQueue[String]() + def record(resolved: Vector[(String, StartFromOutcome)]): Unit = + resolved.foreach { + case (v, StartFromOutcome.Deliver) => delivered.add(v) + case (v, _) => otherOutcomes.add(v) + } + + val start = CountDownLatch(1) + val switched = AtomicBoolean(false) + val done = AtomicBoolean(false) + + val offerers = streams.zipWithIndex.map { (streamId, s) => + val runnable: Runnable = () => + start.await() + var i = 0 + while i < perStream do + val key = MessageOrderKey(i.toLong * 10 + s, streamId, 1L, i.toLong, -1) + orderingLock.synchronized(record(merge.offer(streamId, key, atBacklogEnd = false, payload = f"m-$s-$i%04d"))) + i += 1 + val t = Thread(runnable, s"offer-$s") + t.setDaemon(true) + t + } + val pump = { + val runnable: Runnable = () => + start.await() + while !done.get do + orderingLock.synchronized { + var going = true + while going do + merge.peekGuaranteed() match + case Some(v) => delivered.add(v); merge.commitGuaranteed() + case None => going = false + } + // Outside the lock, so the pump cannot starve the offerers or the switch. + Thread.`yield`() + val t = Thread(runnable, "guaranteed-pump") + t.setDaemon(true) + t + } + val switcher = { + val runnable: Runnable = () => + start.await() + // Let real work pile up behind the barrier first, but never wait forever: the + // switch has to happen for this test to be about anything. + val deadlineNanos = System.nanoTime() + 10_000_000_000L + while merge.heldCount < 50 && System.nanoTime() < deadlineNanos do Thread.`yield`() + orderingLock.synchronized(record(merge.relaxGuaranteedToBestEffort())) + switched.set(true) + val t = Thread(runnable, "order-switcher") + t.setDaemon(true) + t + } + + val workers = offerers :+ pump :+ switcher + workers.foreach(_.start()) + start.countDown() + (offerers :+ switcher).foreach(_.join(120_000)) + done.set(true) + pump.join(120_000) + + // Everything still held is released by the ordinary best-effort residence bound. + clock.advance(graceMs + 1) + var draining = true + while draining do + val resolved = orderingLock.synchronized(merge.sweepStalled()) + record(resolved) + draining = resolved.nonEmpty + + val out = delivered.asScala.toVector + val expected = streams.indices.flatMap(s => (0 until perStream).map(i => f"m-$s-$i%04d")).toSet + assertTrue( + switched.get, + workers.forall(!_.isAlive), + otherOutcomes.asScala.toVector.isEmpty, // no drops, no requeues: nobody offered a copy + out.size == expected.size, + out.toSet == expected, // nothing lost + out.distinct.size == out.size, // nothing doubled + merge.heldCount == 0 + ) ?? (s"delivered=${out.size}/${expected.size} duplicates=${out.size - out.distinct.size} " + + s"missing=${(expected -- out.toSet).size} held=${merge.heldCount} other=${otherOutcomes.asScala.toVector.take(5)}") + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/deliveryRateLimiterTest.scala b/server/src/test/scala/consumer/session_runner/deliveryRateLimiterTest.scala new file mode 100644 index 000000000..218e50a30 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryRateLimiterTest.scala @@ -0,0 +1,296 @@ +package consumer.session_runner + +import zio.test.* + +import scala.collection.mutable.ArrayBuffer + +/** The delivery rate limiter's decisions, driven with a hand-cranked clock and a hand-cranked + * scheduler - no threads, no sleeps, every tick explicit. + * + * The properties worth pinning, in the order a message meets them: the bucket starts full so the + * first screenful is instant; the sustained rate is exact over any window; order is the queue's + * order under every interleaving; the watermarks fire their callbacks exactly once per crossing; + * a refused permit hold retries instead of sticking; a user pause stops the drain without losing + * the backlog; and one failing delivery costs exactly itself. + */ +object deliveryRateLimiterTest extends ZIOSpecDefault: + + /** A core with a manual clock. Tests advance `clock` and call the drain protocol by hand. */ + private final class ManualCore(initialRate: Long): + var clock: Long = 0L + val core = DeliveryRateLimiterCore[Int](nowMs = () => clock) + core.setRate(initialRate) + + /** A limiter whose timer is a list: `runPending()` is the scheduler thread. Delays are recorded + * so the pacing decisions themselves can be asserted. */ + private final class ManualLimiter(initialRate: Long, holdAnswers: Iterator[Boolean] = Iterator.continually(true)): + var clock: Long = 0L + val processed = ArrayBuffer[Int]() + val scheduledDelays = ArrayBuffer[Long]() + var holdCalls = 0 + var releaseCalls = 0 + var throwOn: Set[Int] = Set.empty + + private val pending = ArrayBuffer[Runnable]() + + val core = DeliveryRateLimiterCore[Int](nowMs = () => clock) + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (delayMs, task) => { scheduledDelays += delayMs; pending += task }, + process = i => { if throwOn.contains(i) then throw RuntimeException(s"boom on $i"); processed += i; () }, + holdPermits = () => { holdCalls += 1; holdAnswers.next() }, + releasePermits = () => { releaseCalls += 1; true } + ) + core.setRate(initialRate) + + def runPending(): Unit = + val tasks = pending.toVector + pending.clear() + tasks.foreach(_.run()) + + def hasPending: Boolean = pending.nonEmpty + + def spec = suite("delivery rate limiter")( + suite("the core's arithmetic")( + test("the bucket starts FULL: the first second's worth drains at once") { + // A session capped at 100/s that begins with "latest 50" paints all 50 immediately - + // the user asked for exactly those. The cap shapes what follows, not the first paint. + val m = ManualCore(100) + (1 to 500).foreach(i => m.core.offer(i)) + val first = m.core.beginDrain() + m.core.finishDrain() + assertTrue(first == (1 to 100).toVector) + }, + test("a returned delivery token refills the bucket by one, never past its size") { + // The guaranteed pump's failed-send refund: the send never happened, so the + // token it acquired must come back - and an over-refund must clamp at the + // bucket, exactly like requeueFront's refund on the queued path. + val m = ManualCore(1) + val first = m.core.tryAcquireDeliveryToken() + val exhausted = m.core.tryAcquireDeliveryToken() + m.core.returnDeliveryToken() + m.core.returnDeliveryToken() // a second refund must not bank a second token + val afterRefund = m.core.tryAcquireDeliveryToken() + val clamped = m.core.tryAcquireDeliveryToken() + assertTrue(first, !exhausted, afterRefund, !clamped) + }, + test("the sustained rate is EXACT: each elapsed second earns exactly the rate") { + val m = ManualCore(100) + (1 to 500).foreach(i => m.core.offer(i)) + m.core.beginDrain(); m.core.finishDrain() // the initial burst of 100 + + val perSecond = (1 to 4).map { _ => + m.clock += 1000 + val batch = m.core.beginDrain() + m.core.finishDrain() + batch.size + } + assertTrue(perSecond == Vector(100, 100, 100, 100)) && + assertTrue(m.core.queuedCount == 0) + }, + test("idle time cannot bank more than one second's burst") { + val m = ManualCore(100) + m.core.offer(0) + m.core.beginDrain(); m.core.finishDrain() + // A minute of silence, then a flood: the bucket is capped at the rate, so the first + // drain answers with 100, not with 6000 saved-up tokens. + m.clock += 60_000 + (1 to 300).foreach(i => m.core.offer(i)) + val batch = m.core.beginDrain() + assertTrue(batch.size == 100) + }, + test("FIFO: an offer made during a drain lands BEHIND everything queued") { + val inline = ManualCore(0).core.offer(1) + assertTrue(inline.processNow) + + // A backlog built under a limit, then the limit lifted MID-DRAIN: the unlimited + // shortcut must still be refused while the drain is in flight, or the new message + // would run concurrently with it and could overtake the batch into the session's + // stateful filters. + val m = ManualCore(5) + (2 to 4).foreach(i => m.core.offer(i)) + val batch = m.core.beginDrain() + m.core.setRate(0) + val duringDrain = m.core.offer(5) + m.core.finishDrain() + assertTrue(!duringDrain.processNow) && + assertTrue(batch == Vector(2, 3, 4)) && + assertTrue(m.core.beginDrain() == Vector(5)) + }, + test("rate 0 with a backlog refuses the inline shortcut and flushes everything") { + // The backlog exists because the rate was JUST lowered to 0 with messages queued: the + // flush must stay ordered, so new offers join the queue until it has drained. + val m = ManualCore(100) + (1 to 150).foreach(i => m.core.offer(i)) + m.core.setRate(0) + val outcome = m.core.offer(151) + val flush = m.core.beginDrain() + m.core.finishDrain() + assertTrue(!outcome.processNow) && assertTrue(flush == (1 to 151).toVector) + }, + test("one timer per backlog: a second offer never arms a second drain") { + val m = ManualCore(10) + val first = m.core.offer(1) + val second = m.core.offer(2) + assertTrue(first.scheduleDrainAfterMs.isDefined) && + assertTrue(second.scheduleDrainAfterMs.isEmpty) + }, + test("a cancelled timer can be re-armed, and re-arming an armed one is refused") { + val m = ManualCore(10) + m.core.offer(1) + m.core.cancelScheduledDrain() + val rearmed = m.core.rearmDrain() + val again = m.core.rearmDrain() + assertTrue(rearmed.isDefined) && assertTrue(again.isEmpty) + }, + test("the next-drain delay is the exact token wait, floored against timer churn") { + // rate 100: the next token is 10ms away, but waking every 10ms burns a thread on + // timers, so the wait is floored and the batch grows to match - the tokens keep + // accruing while asleep, so the floor costs no throughput. + val fast = ManualCore(100) + (1 to 200).foreach(i => fast.core.offer(i)) + fast.core.beginDrain() + val fastNext = fast.core.finishDrain().rescheduleAfterMs + + // rate 1: the exact wait (a full second) is what gets scheduled - no busy ticks. + val slow = ManualCore(1) + slow.core.offer(1); slow.core.offer(2) + slow.core.beginDrain() + val slowNext = slow.core.finishDrain().rescheduleAfterMs + + assertTrue(fastNext.contains(deliveryRateLimitMinRescheduleDelayMs)) && + assertTrue(slowNext.contains(1000L)) + }, + test("one drain is capped, and the remainder reschedules immediately") { + // A fat bucket must not hold the drainer - and with it the session's single JS + // context - for an unbounded stretch. + val m = ManualCore(10_000) + (1 to 2000).foreach(i => m.core.offer(i)) + val batch = m.core.beginDrain() + val next = m.core.finishDrain().rescheduleAfterMs + assertTrue(batch.size == deliveryRateLimitMaxDrainBatch) && assertTrue(next.contains(0L)) + } + ), + suite("the wrapper's side effects")( + test("under a limit, nothing is processed at offer time; the drain releases FIFO") { + val m = ManualLimiter(100) + (1 to 5).foreach(m.limiter.offer(_)) + val beforeTick = m.processed.toVector + m.runPending() + assertTrue(beforeTick.isEmpty) && assertTrue(m.processed.toVector == (1 to 5).toVector) + }, + test("unlimited passes straight through on the calling thread") { + val m = ManualLimiter(0) + (1 to 5).foreach(m.limiter.offer(_)) + assertTrue(m.processed.toVector == (1 to 5).toVector) && assertTrue(!m.hasPending) + }, + test("the permit hold fires ONCE at the high watermark and releases ONCE at the low") { + val m = ManualLimiter(1_000_000) + (1 to deliveryRateLimitHoldPermitsAboveQueued + 50).foreach(m.limiter.offer(_)) + val holdsAfterCrossing = m.holdCalls + // Drain it down; the bucket is huge, so only the per-tick cap bounds each batch. + while m.core.queuedCount > 0 do m.runPending() + assertTrue(holdsAfterCrossing == 1) && + assertTrue(m.holdCalls == 1) && + assertTrue(m.releaseCalls == 1) && + assertTrue(m.processed.size == deliveryRateLimitHoldPermitsAboveQueued + 50) + }, + test("a REFUSED hold retries on the next offer - suppression must not stick") { + // The runner refuses holds while a counted start-from is still resolving. When the + // refusal's reason has passed, the very next crossing offer must re-assert the hold, + // not believe a stale flag. + val m = ManualLimiter(1_000_000, holdAnswers = Iterator(false, true)) + (1 to deliveryRateLimitHoldPermitsAboveQueued).foreach(m.limiter.offer(_)) + val afterRefusal = m.holdCalls + m.limiter.offer(0) + assertTrue(afterRefusal == 1) && assertTrue(m.holdCalls == 2) + }, + test("a user pause stops the drain; resume re-arms it; the backlog survives both") { + val m = ManualLimiter(100) + (1 to 5).foreach(m.limiter.offer(_)) + m.limiter.pauseDraining() + m.runPending() // the tick that was already armed fires into the pause and must no-op + val processedWhilePaused = m.processed.toVector + m.limiter.resumeDraining() + m.runPending() + assertTrue(processedWhilePaused.isEmpty) && assertTrue(m.processed.toVector == (1 to 5).toVector) + }, + test("a throwing delivery costs exactly itself") { + val m = ManualLimiter(100) + m.throwOn = Set(2) + (1 to 3).foreach(m.limiter.offer(_)) + m.runPending() + assertTrue(m.processed.toVector == Vector(1, 3)) + }, + test("stop clears the backlog for good - re-arming afterwards delivers nothing") { + val m = ManualLimiter(100) + (1 to 5).foreach(m.limiter.offer(_)) + m.limiter.stop() + m.limiter.resumeDraining() + m.runPending() + assertTrue(m.processed.isEmpty) && assertTrue(m.core.queuedCount == 0) + }, + test("the held flag stays set until the limiter itself releases - nobody can stomp it any more") { + // The old contract reset the flag on a user resume, because a user resume used to + // wake every consumer wholesale. Holds now land on per-consumer pause ARBITERS + // (reason Limiter), which a user resume cannot release - so the flag stays + // truthful on its own, and a second crossing must NOT re-call the hold hook. + val m = ManualLimiter(1_000_000) + (1 to deliveryRateLimitHoldPermitsAboveQueued).foreach(m.limiter.offer(_)) + m.limiter.offer(0) + assertTrue(m.holdCalls == 1) + } + ), + suite("the delivery budget's machinery")( + test("forceQueue closes the unlimited inline shortcut - a budget needs the queue") { + val m = ManualCore(0) + m.core.setForceQueue(true) + val outcome = m.core.offer(1) + assertTrue(!outcome.processNow) && assertTrue(m.core.queuedCount == 1) + }, + test("requeueFront puts the unprocessed tail back AT THE HEAD, order intact, tokens refunded") { + val m = ManualCore(10) + (1 to 10).foreach(i => m.core.offer(i)) + val batch = m.core.beginDrain() // takes all 10, spends all 10 tokens + m.core.finishDrain() + // Pretend the budget stopped after 3: the remaining 7 go back untouched. + m.core.requeueFront(batch.drop(3)) + // The refund matters: without it those 7 would be double-charged on the next drain. + val next = m.core.beginDrain() + assertTrue(batch.size == 10) && assertTrue(next == (4 to 10).toVector) + }, + test("a stop DURING a drain delivers exactly up to the stop and requeues the rest in order") { + // The wrapper checks the pause flag between items; the process callback itself + // flips it at the third delivery - exactly what the runner's send-site budget does. + var deliveredCount = 0 + val stopAt = 3 + val limiterHolder = scala.collection.mutable.ArrayBuffer[DeliveryRateLimiter[Int]]() + val core = DeliveryRateLimiterCore[Int](nowMs = () => 0L) + val pending = scala.collection.mutable.ArrayBuffer[Runnable]() + val processed = scala.collection.mutable.ArrayBuffer[Int]() + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (_, task) => { pending += task; () }, + process = i => { + processed += i + deliveredCount += 1 + if deliveredCount == stopAt then limiterHolder.head.pauseDraining() + }, + holdPermits = () => true, + releasePermits = () => true + ) + limiterHolder += limiter + core.setRate(100) + (1 to 10).foreach(limiter.offer(_)) + pending.toVector.foreach(_.run()); pending.clear() + + val afterStop = processed.toVector + // Resume: the tail is exactly where it was, and drains in order. + limiter.resumeDraining() + pending.toVector.foreach(_.run()); pending.clear() + + assertTrue(afterStop == Vector(1, 2, 3)) && + assertTrue(processed.toVector == (1 to 10).toVector) + } + ) + ) diff --git a/server/src/test/scala/consumer/session_runner/deliveryRateLimiterWiringTest.scala b/server/src/test/scala/consumer/session_runner/deliveryRateLimiterWiringTest.scala new file mode 100644 index 000000000..3fc3cbddf --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/deliveryRateLimiterWiringTest.scala @@ -0,0 +1,518 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters.* + +/** The rate limiter wired into the REAL delivery path: `ConsumerListener.received` with proxy + * consumers, a hand-cranked drain, and a real runner for the permit arbitration. Only the broker + * is replaced. + * + * What must hold end to end, not just inside the limiter: a limited Deliver reaches nobody until + * the drain and is ACKNOWLEDGED at delivery, not at receipt; DROPS ignore the limit entirely, so + * a counted skip positions at full speed; a delivery that fails at the observer is handed back + * exactly as the unlimited path would; the user's pause outranks the limiter's permit hold; and a + * hold is refused while start-from counting is still resolving. + */ +object deliveryRateLimiterWiringTest extends ZIOSpecDefault: + + private val consumerName = "cs-rate-limit-wiring" + private val p0 = "persistent://public/default/rate-limit-wiring-0" + + /** A consumer recording acks, nacks AND permit calls - the last two are this suite's subject. */ + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + val permitCalls = ConcurrentLinkedQueue[String]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "pause" => permitCalls.add("pause"); null + case "resume" => permitCalls.add("resume"); null + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(key: String, entryId: Long): MessageImpl[Array[Byte]] = messageOn(p0, key, 1000L + entryId, entryId) + + private def messageOn(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** A listener recording deliveries, with an optional per-key throw to play the cancelled + * client. Gate OPEN, pass-through ordering - the plain live-tail shape. */ + private def listener(delivered: ConcurrentLinkedQueue[String], throwOn: Set[String] = Set.empty): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => + if throwOn.contains(msg.getKey) then throw io.grpc.StatusRuntimeException(io.grpc.Status.CANCELLED) + delivered.add(msg.getKey) + () + )) + l.startAcceptingNewMessages() + l + + /** A limiter with a hand-cranked timer, wired the way the runner wires it. */ + private final class ManualWiring(rate: Long, holdPermits: () => Boolean = () => true, releasePermits: () => Boolean = () => true): + private val pending = ArrayBuffer[Runnable]() + val core = DeliveryRateLimiterCore[HeldMessage](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[HeldMessage]( + core = core, + schedule = (_, task) => { pending += task; () }, + process = held => held.listener.deliverNow(held), + holdPermits = holdPermits, + releasePermits = releasePermits + ) + core.setRate(rate) + + def drain(): Unit = + val tasks = pending.toVector + pending.clear() + tasks.foreach(_.run()) + + private def targetRunner(consumerListener: ConsumerListener, consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p0))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p0), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = (consumers).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(target: ConsumerSessionTargetRunner): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-rate-limit-wiring", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + def spec = suite("delivery rate limiter wired into the delivery path")( + test("a limited Deliver reaches nobody until the drain, then arrives in order, ACKED AT DELIVERY") { + val delivered = ConcurrentLinkedQueue[String]() + val recording = RecordingConsumer(p0) + val l = listener(delivered) + val wiring = ManualWiring(rate = 100) + l.deliveryRateLimiter = Some(wiring.limiter) + + (1 to 3).foreach(i => l.received(recording.consumer, message(s"m$i", i.toLong))) + val deliveredBeforeDrain = delivered.asScala.toVector + val ackedBeforeDrain = recording.acknowledged.asScala.toVector + wiring.drain() + + // Nothing moved before the drain - not the handler, and NOT the acks: a message + // acknowledged at receipt would be lost to a session that stopped before its delivery. + assertTrue(deliveredBeforeDrain.isEmpty) && + assertTrue(ackedBeforeDrain.isEmpty) && + assertTrue(delivered.asScala.toVector == Vector("m1", "m2", "m3")) && + assertTrue(recording.acknowledged.asScala.toVector == Vector("m1", "m2", "m3")) + }, + test("DROPS ignore the limit: a counted skip positions at full speed under any rate") { + val delivered = ConcurrentLinkedQueue[String]() + val recording = RecordingConsumer(p0) + val l = listener(delivered) + l.startFromDiscard = StartFromDiscard.shared(2) + val wiring = ManualWiring(rate = 1) // absurdly tight, to prove drops never touch it + l.deliveryRateLimiter = Some(wiring.limiter) + + (1 to 3).foreach(i => l.received(recording.consumer, message(s"m$i", i.toLong))) + + // The two drops were acknowledged IMMEDIATELY, with no drain ever run; only the third + // message - the first the user will see - sits waiting for the limiter. + assertTrue(recording.acknowledged.asScala.toVector == Vector("m1", "m2")) && + assertTrue(delivered.asScala.isEmpty) && + assertTrue(wiring.limiter.queuedCount == 1) + }, + test("a delivery failing at the observer is handed back; the batch behind it still delivers") { + val delivered = ConcurrentLinkedQueue[String]() + val recording = RecordingConsumer(p0) + val l = listener(delivered, throwOn = Set("m2")) + val wiring = ManualWiring(rate = 100) + l.deliveryRateLimiter = Some(wiring.limiter) + + (1 to 3).foreach(i => l.received(recording.consumer, message(s"m$i", i.toLong))) + wiring.drain() + + assertTrue(delivered.asScala.toVector == Vector("m1", "m3")) && + assertTrue(recording.handedBack.asScala.toVector == Vector("m2")) && + assertTrue(recording.acknowledged.asScala.toVector == Vector("m1", "m3")) + }, + test("a permit release cannot resume a USER-paused session - the User reason still holds") { + // The gate-shut REFUSAL this used to pin is gone by design: with per-reason + // arbitration the limiter may record its hold any time, because releasing it can + // never wake a consumer another owner still holds. What survives - and what actually + // protected the user - is pinned here: pause, cycle a permit hold and release, and + // the consumer must never see a resume. + val recording = RecordingConsumer(p0) + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val target = targetRunner(l, Map(p0 -> recording.consumer)) + + target.pause() // the user's hold + target.setPermitHold(true) + target.setPermitHold(false) + + assertTrue(!recording.permitCalls.asScala.toVector.contains("resume")) ?? + s"calls=${recording.permitCalls.asScala.toVector}" + }, + test("a REFUSED aggregate hold rolls its partial holds back - nothing stays paused for a hold that failed") { + // holdPermits aggregates several consumers; one throwing makes the whole answer + // false while the others already paused. Without the rollback those partial holds + // sat in their arbiters with the limiter believing it owned nothing - paused forever. + var holds = 0 + var releases = 0 + val pending = ArrayBuffer[Runnable]() + var clock = 0L + val core = DeliveryRateLimiterCore[String](nowMs = () => clock, payloadBytesOf = _ => 50L * 1024 * 1024) + val limiter = DeliveryRateLimiter[String]( + core = core, + schedule = (_, task) => { pending += task; () }, + process = _ => (), + holdPermits = () => { holds += 1; false }, // the aggregate always refuses + releasePermits = () => { releases += 1; true } + ) + core.setRate(1000) + + (1 to 3).foreach(i => limiter.offer(s"m$i")) // crosses the byte mark + val afterFirstCrossing = (holds, releases) + limiter.offer("m4") // still over the mark: the un-set flag lets the next offer retry + + assertTrue( + afterFirstCrossing == (1, 1), // refused hold -> immediate rollback release + holds == 2, // and the retry happened + releases == 2 + ) ?? s"holds=$holds releases=$releases" + }, + test("a release that FAILS ONCE is retried on its own tick - a drained queue produces no other chance") { + var releaseAttempts = 0 + val pending = ArrayBuffer[(Long, Runnable)]() + var clock = 0L + val core = DeliveryRateLimiterCore[String](nowMs = () => clock, payloadBytesOf = _ => 50L * 1024 * 1024) + val limiter = DeliveryRateLimiter[String]( + core = core, + schedule = (delay, task) => { pending += ((delay, task)); () }, + process = _ => (), + holdPermits = () => true, + releasePermits = () => { releaseAttempts += 1; releaseAttempts > 1 } // first release fails + ) + core.setRate(1000) + (1 to 3).foreach(i => limiter.offer(s"m$i")) // byte crossing: permits held + + clock += 1000 + def runAll(): Unit = { val t = pending.toVector; pending.clear(); t.foreach(_._2.run()) } + runAll() // the drain empties the queue; the release attempt FAILS; a retry tick is armed + val retryArmed = pending.nonEmpty && pending.forall(_._1 == deliveryRateLimitReleaseRetryDelayMs) + runAll() // the retry succeeds + + assertTrue( + releaseAttempts == 2, + retryArmed, + core.queuedCount == 0 + ) ?? s"attempts=$releaseAttempts retryArmed=$retryArmed pending=${pending.size}" + }, + test("WHILE PAUSED, the unlimited inline path is refused - the message queues for the resume") { + var processed = 0 + val pending = ArrayBuffer[Runnable]() + val core = DeliveryRateLimiterCore[String](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[String]( + core = core, + schedule = (_, task) => { pending += task; () }, + process = _ => processed += 1, + holdPermits = () => true, + releasePermits = () => true + ) + // Rate 0 = the unlimited inline path... unless the user has the session paused: a + // message arriving between the pause RPC and the consumers actually stopping used to + // be processed (and acknowledged) inline - delivery after pause. + limiter.pauseDraining() + limiter.offer("m1") + val processedWhilePaused = processed + val queuedWhilePaused = core.queuedCount + + limiter.resumeDraining() + val tasks = pending.toVector; pending.clear(); tasks.foreach(_.run()) + + assertTrue( + processedWhilePaused == 0, + queuedWhilePaused == 1, + processed == 1 + ) ?? s"processedWhilePaused=$processedWhilePaused queued=$queuedWhilePaused processed=$processed" + }, + test("QUEUED BYTES hold the permits when the count never would - the fat-message watermark") { + // 2000 queued 5 MB payloads is 10 GB the count watermark calls perfectly fine. The + // byte watermark is the other half of the same backpressure: three fat payloads must + // hold permits, and draining them must let go. + var holds = 0 + var releases = 0 + val pending = ArrayBuffer[Runnable]() + var clock = 0L + val core = DeliveryRateLimiterCore[String](nowMs = () => clock, payloadBytesOf = _ => 50L * 1024 * 1024) + val limiter = DeliveryRateLimiter[String]( + core = core, + schedule = (_, task) => { pending += task; () }, + process = _ => (), + holdPermits = () => { holds += 1; true }, + releasePermits = () => { releases += 1; true } + ) + core.setRate(1000) // any non-zero rate forces the queue path + + limiter.offer("a") + limiter.offer("b") + val holdsAtTwo = holds // 100 MiB queued: still under the 128 MiB mark + limiter.offer("c") // 150 MiB: over it, at a count of THREE + val holdsAtThree = holds + + clock += 1000 + val tasks = pending.toVector; pending.clear() + tasks.foreach(_.run()) // the drain empties the queue: bytes back under the low mark + + assertTrue( + holdsAtTwo == 0, + holdsAtThree == 1, + holds == 1, + releases == 1, + core.queuedBytesCount == 0L + ) ?? s"holds=$holds releases=$releases queuedBytes=${core.queuedBytesCount}" + }, + test("with no other owner, a permit hold pauses and its release resumes - exactly once each") { + val recording = RecordingConsumer(p0) + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val target = targetRunner(l, Map(p0 -> recording.consumer)) + l.startAcceptingNewMessages() + + target.setPermitHold(true) + target.setPermitHold(false) + + assertTrue(recording.permitCalls.asScala.toVector == Vector("pause", "resume")) ?? + s"calls=${recording.permitCalls.asScala.toVector}" + }, + test("the runner refuses a hold while start-from counting is in flight, then grants it") { + val recording = RecordingConsumer(p0) + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + val runner = session(targetRunner(l, Map(p0 -> recording.consumer))) + // A real rate, so offers actually QUEUE - at 0 everything passes inline, the watermark + // is never reached, and this whole test measures nothing. + runner.deliveryRateLimiter.core.setRate(1) + + // A counted skip still has 5 to drop: the limiter crossing its watermark must NOT pause + // the consumers - a throttled-quiet stream is indistinguishable from the silent stream + // the merge gives up on. + l.startFromDiscard = StartFromDiscard.shared(5) + (1 to deliveryRateLimitHoldPermitsAboveQueued + 1) + .foreach(i => runner.deliveryRateLimiter.offer(HeldMessage(recording.consumer, message(s"m$i", i.toLong), l))) + val pausesDuringResolution = recording.permitCalls.asScala.count(_ == "pause") + val queuedDuringResolution = runner.deliveryRateLimiter.queuedCount + + // The skip finishes; the very next crossing offer must re-assert the hold. + (1 to 5).foreach(_ => l.startFromDiscard.claim(p0)) + runner.deliveryRateLimiter.offer(HeldMessage(recording.consumer, message("late", 9999L), l)) + + // The vacuity guard first: the backlog really did cross the watermark while refused. + assertTrue(queuedDuringResolution >= deliveryRateLimitHoldPermitsAboveQueued) && + assertTrue(pausesDuringResolution == 0) && + assertTrue(recording.permitCalls.asScala.count(_ == "pause") == 1) + }, + test("TWO TOPICS through one limiter: per-topic order holds, acks land on the right consumer") { + // One target consuming two topics has ONE listener and one limiter; the limit is per + // session, so both topics share the queue. Per-topic relative order must survive it, + // and each delivery must acknowledge on the consumer it arrived through. + val p1 = "persistent://public/default/rate-limit-wiring-1" + val delivered = ConcurrentLinkedQueue[String]() + val c0 = RecordingConsumer(p0) + val c1 = RecordingConsumer(p1) + val l = listener(delivered) + val wiring = ManualWiring(rate = 100) + l.deliveryRateLimiter = Some(wiring.limiter) + + l.received(c0.consumer, messageOn(p0, "a1", 100L, 0L)) + l.received(c1.consumer, messageOn(p1, "b1", 110L, 0L)) + l.received(c0.consumer, messageOn(p0, "a2", 120L, 1L)) + l.received(c1.consumer, messageOn(p1, "b2", 130L, 1L)) + wiring.drain() + + val out = delivered.asScala.toVector + assertTrue(out == Vector("a1", "b1", "a2", "b2")) && + assertTrue(c0.acknowledged.asScala.toVector == Vector("a1", "a2")) && + assertTrue(c1.acknowledged.asScala.toVector == Vector("b1", "b2")) + }, + test("the MERGE'S GLOBAL ORDER survives the limiter, even when arrival order fights it") { + // Two topics, arrival deliberately INVERTED against publish time: all of p0 arrives + // before any of p1. A global skip of 1 must drop the globally-oldest (a1, which arrived + // first but is only oldest by publish time), and everything DELIVERED must come out in + // the merge's key order - b1 (pt 200) before a2 (pt 300) - not in arrival order, and + // the limiter's queue must preserve exactly that decision order through its drain. + val p1 = "persistent://public/default/rate-limit-wiring-merge-1" + val delivered = ConcurrentLinkedQueue[String]() + val c0 = RecordingConsumer(p0) + val c1 = RecordingConsumer(p1) + val l = listener(delivered) + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip(1, Vector( + StartFromStream(startFromStreamId(consumerName, p0), EntryPosition(1L, 1L, -1, 1)), + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition(1L, 1L, -1, 1)) + )) + ) + val wiring = ManualWiring(rate = 100) + l.deliveryRateLimiter = Some(wiring.limiter) + + l.received(c0.consumer, messageOn(p0, "a1", 100L, 0L)) + l.received(c0.consumer, messageOn(p0, "a2", 300L, 1L)) + l.received(c1.consumer, messageOn(p1, "b1", 200L, 0L)) + l.received(c1.consumer, messageOn(p1, "b2", 400L, 1L)) + wiring.drain() + + // a1 was DROPPED - acknowledged immediately, never delivered, never rate limited. + assertTrue(delivered.asScala.toVector == Vector("b1", "a2", "b2")) && + assertTrue(c0.acknowledged.asScala.headOption.contains("a1")) && + assertTrue(!delivered.asScala.toVector.contains("a1")) + }, + test("GUARANTEED spends no rate token while it is waiting for another topic") { + val p1 = "persistent://public/default/rate-limit-wiring-guaranteed-1" + val delivered = ConcurrentLinkedQueue[String]() + val c0 = RecordingConsumer(p0) + val c1 = RecordingConsumer(p1) + val l = listener(delivered) + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.Ordered( + // Far-future recorded ends: these streams are MID-REPLAY - the barrier must + // wait on them, and nothing here is past the boundary. + Vector( + StartFromStream(startFromStreamId(consumerName, p0), EntryPosition(1L, 1_000_000L, -1, 1)), + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition(1L, 1_000_000L, -1, 1)) + ), + _root_.consumer.session_config.MessageDeliveryOrder.Guaranteed + ) + ) + val wiring = ManualWiring(rate = 1) + l.deliveryRateLimiter = Some(wiring.limiter) + + // p0 alone is not safe: Guaranteed still waits for p1. That wait must leave the one + // initial token untouched, so p0 can be delivered as soon as p1 supplies its head. + l.received(c0.consumer, messageOn(p0, "a", 100L, 0L)) + val beforeP1 = delivered.asScala.toVector + l.received(c1.consumer, messageOn(p1, "b", 200L, 0L)) + + assertTrue(beforeP1.isEmpty) && + assertTrue(delivered.asScala.toVector == Vector("a")) && + assertTrue(c0.acknowledged.asScala.toVector == Vector("a")) + }, + test("GUARANTEED hands the rate token back when the send fails - the retry is not charged twice") { + // rate 1 and a hand clock that never moves: the bucket holds exactly one token, ever. + // The first send attempt FAILS (a cancelled client); the head stays in the merge and + // the pump's next tick retries it. Without the refund that retry finds an empty + // bucket - the failed attempt burned the only token - and the head sits for a full + // simulated second on top of the latency the failure already cost. + val p1 = "persistent://public/default/rate-limit-wiring-guaranteed-refund-1" + val delivered = ConcurrentLinkedQueue[String]() + val c0 = RecordingConsumer(p0) + val c1 = RecordingConsumer(p1) + var failNext = true + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => + if failNext then + failNext = false + throw io.grpc.StatusRuntimeException(io.grpc.Status.CANCELLED) + delivered.add(msg.getKey) + () + )) + l.startAcceptingNewMessages() + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.Ordered( + // Far-future recorded ends: mid-replay streams, nothing past the boundary. + Vector( + StartFromStream(startFromStreamId(consumerName, p0), EntryPosition(1L, 1_000_000L, -1, 1)), + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition(1L, 1_000_000L, -1, 1)) + ), + _root_.consumer.session_config.MessageDeliveryOrder.Guaranteed + ) + ) + val wiring = ManualWiring(rate = 1) + l.deliveryRateLimiter = Some(wiring.limiter) + + l.received(c0.consumer, messageOn(p0, "a", 100L, 0L)) // waits on p1 + l.received(c1.consumer, messageOn(p1, "b", 200L, 0L)) // both heads: the pump tries "a" and the send fails + val deliveredAfterFailure = delivered.asScala.toVector + l.sweepStartFromStall() // the pump's retry timer - it must find the refunded token + + assertTrue( + deliveredAfterFailure.isEmpty, // the failed attempt delivered nothing... + delivered.asScala.toVector == Vector("a"), // ...and the retry needed no fresh token + c0.acknowledged.asScala.toVector == Vector("a") + ) ?? s"afterFailure=$deliveredAfterFailure delivered=${delivered.asScala.toVector}" + }, + test("resume installs the SHARED limiter on the listener, with the requested rate") { + val delivered = ConcurrentLinkedQueue[String]() + val recording = RecordingConsumer(p0) + val l = listener(delivered) + val runner = session(targetRunner(l, Map(p0 -> recording.consumer))) + + val before = l.deliveryRateLimiter + + runner.resume(new io.grpc.stub.StreamObserver[com.tools.teal.pulsar.ui.api.v1.consumer.ResumeResponse] { + override def onNext(value: com.tools.teal.pulsar.ui.api.v1.consumer.ResumeResponse): Unit = () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + }, isDebug = false, includeConsumerStats = true, maxMessagesPerSecond = 123) + + assertTrue(before.isEmpty) && + assertTrue(l.deliveryRateLimiter.contains(runner.deliveryRateLimiter)) && + assertTrue(runner.deliveryRateLimiter.core.rate == 123L) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/globalStartFromTest.scala b/server/src/test/scala/consumer/session_runner/globalStartFromTest.scala new file mode 100644 index 000000000..b606764c5 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/globalStartFromTest.scala @@ -0,0 +1,1115 @@ +package consumer.session_runner + +import org.apache.pulsar.client.impl.{BatchMessageIdImpl, MessageIdImpl} +import zio.test.* + +/** The two GLOBAL start-from contracts: "skip the first n" and "the latest n" are counted over ALL + * physical topics of a session, ordered by publish time - not per partition. + * + * Contract change context: both modes used to be per physical topic. "Latest 2" on a 3-partition + * topic delivered SIX messages (the last 2 of each partition), and "skip 5" dropped whichever 5 + * arrived first out of the broker's arbitrary interleaving. Both are now defined over the merged + * stream, ordered by publish time. + * + * The two algorithms differ, and the memory profile is the reason. Skip-N is a streaming k-way + * merge holding ONE message per topic, because n has deliberately no cap and buffering n messages + * would let a typed number exhaust the heap. Latest-N buffers NOTHING at all: its cut is resolved + * from entry METADATA before delivery begins (`resolveLatestN` walks backward from each topic's + * end), so every consumer simply starts in the right place - the bounded top-n heap it once was + * is gone, and with it the only start-from path whose memory grew with a number the user typed. + * The `latestSuite` below pins the resolver. + * + * Everything here is driven with plain values through the pure state machines: no broker, and no + * mock. + */ +object globalStartFromTest extends ZIOSpecDefault: + + private val p0 = "persistent://public/default/t-partition-0" + private val p1 = "persistent://public/default/t-partition-1" + private val p2 = "persistent://public/default/t-partition-2" + + /** One delivered message as the ordering layers see it. `value` is what the user would read. */ + private final case class Arrival(streamId: String, key: MessageOrderKey, atBacklogEnd: Boolean, value: String) + + private def at(publishTime: Long, topicFqn: String, entryId: Long, batchIndex: Int = -1): MessageOrderKey = + MessageOrderKey(orderTime = publishTime, topicFqn = topicFqn, ledgerId = 1L, entryId = entryId, batchIndex = batchIndex) + + /** A partition's whole backlog: entry ids run 0..k-1 and the LAST message is flagged as the end + * of the pre-existing backlog, exactly as the runtime flags it. + */ + private def backlog(topicFqn: String, messages: (Long, String)*): Vector[Arrival] = + messages.toVector.zipWithIndex.map { case ((publishTime, value), entryId) => + Arrival(topicFqn, at(publishTime, topicFqn, entryId), atBacklogEnd = entryId == messages.size - 1, value) + } + + private final case class Run( + dropped: Vector[String], + delivered: Vector[String], + maxHeld: Int, + leftHeld: Int + ) + + private def drive[M <: StartFromMerge[String]](merge: M, arrivals: Vector[Arrival]): Run = + var maxHeld = 0 + val out = arrivals.flatMap { arrival => + val resolved = merge.offer(arrival.streamId, arrival.key, arrival.atBacklogEnd, arrival.value) + maxHeld = maxHeld max merge.heldCount + resolved + } + Run( + dropped = out.collect { case (value, StartFromOutcome.Drop) => value }, + delivered = out.collect { + case (value, StartFromOutcome.Deliver) => value + case (value, StartFromOutcome.DeliverOutOfOrder) => value + }, + maxHeld = maxHeld, + leftHeld = merge.heldCount + ) + + private def skip(n: Long, streams: Vector[String], arrivals: Vector[Arrival], maxHeld: Int = startFromMergeMaxHeld): Run = + drive(GlobalSkipMerge[String](streams, drainedAtStart = Set.empty, discard = StartFromDiscard.shared(n), maxHeld = maxHeld), arrivals) + + /** Round-robin across partitions - the shape the broker actually delivers a partitioned topic + * in, and the shape under which "first n delivered" and "globally first n" differ. + */ + private def interleave(streams: Vector[Arrival]*): Vector[Arrival] = + val rounds = streams.toVector + val longest = rounds.map(_.size).maxOption.getOrElse(0) + (0 until longest).toVector.flatMap(i => rounds.flatMap(stream => stream.lift(i))) + + private val orderSuite = suite("the total order")( + test("publish time decides first") { + val earlier = at(100, p2, entryId = 9) + val later = at(101, p0, entryId = 0) + assertTrue(MessageOrderKey.ordering.lt(earlier, later)) + }, + test("a publish-time tie is broken by topic name, then by position in the log") { + // Ties are the COMMON case: a fast producer stamps many messages with one millisecond. + val a = at(100, p0, entryId = 5) + val b = at(100, p1, entryId = 0) + val c = at(100, p1, entryId = 1) + assertTrue(MessageOrderKey.ordering.lt(a, b), MessageOrderKey.ordering.lt(b, c)) + }, + test("messages inside one batched entry order by batch index") { + val first = at(100, p0, entryId = 3, batchIndex = 0) + val second = at(100, p0, entryId = 3, batchIndex = 1) + assertTrue(MessageOrderKey.ordering.lt(first, second)) + }, + test("an unbatched message sorts before batch index 0 of the same entry, never equal to it") { + val unbatched = at(100, p0, entryId = 3) + val batched = at(100, p0, entryId = 3, batchIndex = 0) + assertTrue(MessageOrderKey.ordering.lt(unbatched, batched), unbatched != batched) + }, + test("the order is TOTAL - two different shuffles of the same messages sort identically") { + // Without a full tiebreak this is exactly what flakes: the result would depend on the + // order the broker happened to deliver in. + val keys = Vector( + at(100, p0, 0), at(100, p1, 0), at(100, p1, 1), at(100, p2, 0), + at(101, p0, 1), at(101, p0, 2), at(99, p2, 7) + ) + val oneWay = scala.util.Random(1).shuffle(keys).sorted(MessageOrderKey.ordering) + val otherWay = scala.util.Random(2).shuffle(keys).sorted(MessageOrderKey.ordering) + assertTrue(oneWay == otherWay, oneWay.distinct.size == keys.size) + } + ) + + private val backlogEndSuite = suite("finding the end of the pre-existing backlog")( + test("a message before the last entry is still backlog") { + assertTrue(!isPastBacklogEnd(EntryPosition(1, 4, -1, 1), EntryPosition(1, 9, -1, 1))) + }, + test("the last entry of the log ends the backlog") { + assertTrue(isPastBacklogEnd(EntryPosition(1, 9, -1, 1), EntryPosition(1, 9, -1, 1))) + }, + test("a message published after the session started is past the end") { + assertTrue(isPastBacklogEnd(EntryPosition(1, 12, -1, 1), EntryPosition(1, 9, -1, 1))) + }, + test("a later ledger is past the end even with a smaller entry id") { + // Entry ids restart at 0 in each ledger, so comparing entry ids alone would declare a + // fresh ledger to be backlog forever and stall the merge. + assertTrue(isPastBacklogEnd(EntryPosition(2, 0, -1, 1), EntryPosition(1, 9, -1, 1))) + }, + test("a batched end is reached only at its own batch index, not at the start of its entry") { + val end = EntryPosition(1, 9, 4, 5) + assertTrue( + !isPastBacklogEnd(EntryPosition(1, 9, 0, 5), end), + !isPastBacklogEnd(EntryPosition(1, 9, 3, 5), end), + isPastBacklogEnd(EntryPosition(1, 9, 4, 5), end) + ) + }, + test("an end reported without a batch index still waits for the whole final batch") { + // getLastMessageId may answer with a bare entry id even when that entry is a batch; + // ending at batch index 0 would throw away the rest of the newest batch. + val end = EntryPosition(1, 9, -1, 1) + assertTrue( + !isPastBacklogEnd(EntryPosition(1, 9, 0, 10), end), + !isPastBacklogEnd(EntryPosition(1, 9, 8, 10), end), + isPastBacklogEnd(EntryPosition(1, 9, 9, 10), end) + ) + }, + test("MessageId.earliest resolves to EXACTLY the canonical empty position") { + // VERIFIED against Pulsar 3.2.1: an empty topic answers getLastMessageIds with + // MessageId.earliest, whose ledger and entry are -1 and whose batch size is 0. Mapping + // that to (-1, -1, -1, 1) left it merely NEAR the empty position and not equal to it - + // and "nothing retained" is recognised by equality, so an empty partition was waited on + // forever. A live 3-partition "latest 2" with one empty partition delivered ZERO. + val fromEarliest = EntryPosition.of(org.apache.pulsar.client.api.MessageId.earliest) + val fromNegativeId = EntryPosition.of(new MessageIdImpl(-1L, -1L, -1)) + assertTrue(fromEarliest == EntryPosition.empty, fromNegativeId == EntryPosition.empty) ?? + s"earliest -> $fromEarliest, (-1,-1) -> $fromNegativeId, empty is ${EntryPosition.empty}" + }, + test("an empty topic is drained by anything at all") { + assertTrue(isPastBacklogEnd(EntryPosition(0, 0, -1, 1), EntryPosition.empty)) + }, + test("a plain message id becomes an unbatched position") { + val position = EntryPosition.of(new MessageIdImpl(7L, 3L, 0)) + assertTrue(position == EntryPosition(7L, 3L, -1, 1)) + }, + test("a batched message id keeps its index and its batch size") { + val position = EntryPosition.of(new BatchMessageIdImpl(7L, 3L, 0, 2, 10, null)) + assertTrue(position == EntryPosition(7L, 3L, 2, 10)) + } + ) + + /** Three partitions whose publish times interleave, so the global order is NOT the per-partition + * order and not the delivery order either. Globally, by publish time: + * a1 a2 b1 c1 a3 b2 c2 b3 c3 + */ + private val threePartitions = Vector( + backlog(p0, 10L -> "a1", 20L -> "a2", 50L -> "a3"), + backlog(p1, 30L -> "b1", 60L -> "b2", 80L -> "b3"), + backlog(p2, 40L -> "c1", 70L -> "c2", 90L -> "c3") + ) + + private val globalOrder = Vector("a1", "a2", "b1", "c1", "a3", "b2", "c2", "b3", "c3") + + private val skipSuite = suite("skip the globally-first n")( + test("drops the globally-first n by publish time, not the first n the broker delivered") { + // THE contract change. Round-robin delivery offers a1 b1 c1 a2 b2 c2 ..., so the old + // arrival-order counter dropped a1 b1 c1 - two of which are NOT among the three oldest. + val run = skip(3, Vector(p0, p1, p2), interleave(threePartitions*)) + assertTrue( + run.dropped == globalOrder.take(3), + run.delivered == globalOrder.drop(3) + ) ?? s"dropped=${run.dropped} delivered=${run.delivered}" + }, + test("the count is exactly n, whatever the interleaving") { + val counts = Vector(0, 1, 4, 8, 9).map(n => n -> skip(n, Vector(p0, p1, p2), interleave(threePartitions*))) + assertTrue(counts.forall((n, run) => run.dropped.size == n && run.delivered.size == 9 - n)) ?? + s"${counts.map((n, run) => s"n=$n dropped=${run.dropped.size}")}" + }, + test("the messages left are exactly the globally-latest ones, in order") { + val run = skip(6, Vector(p0, p1, p2), interleave(threePartitions*)) + assertTrue(run.delivered == Vector("c2", "b3", "c3"), run.dropped == globalOrder.take(6)) + }, + test("the messages HELD at the cut are released in global order") { + // Scoped deliberately: this is a claim about the batch the merge was still holding when + // the budget ran out, which it sorts before releasing. It is NOT a claim about + // everything that arrives afterwards - see the post-cut suite below. + // + // The cut lands the moment the LAST unit is claimed (dropping a1 then a2), so the + // sorted batch is b1 c1 - what was held at that instant - and everything after passes + // through in ARRIVAL order (b2 c2 a3 b3 c3). The old code kept merging until a later + // claim ANSWERED no, which happened to widen the sorted batch; that wait is exactly + // what held boundary messages hostage when the Nth drop emptied a waited-for stream. + val run = skip(2, Vector(p0, p1, p2), interleave(threePartitions*)) + assertTrue( + run.dropped == globalOrder.take(2), + run.delivered == Vector("b1", "c1") ++ Vector("b2", "c2", "a3", "b3", "c3") + ) ?? s"dropped=${run.dropped} delivered=${run.delivered}" + }, + test("skipping more than the session holds delivers nothing and drops all of it") { + val run = skip(100, Vector(p0, p1, p2), interleave(threePartitions*)) + assertTrue(run.delivered.isEmpty, run.dropped.size == 9) + }, + test("HOLDS AT MOST ONE MESSAGE PER TOPIC - never n") { + // The memory contract. n has no cap: a merge that buffered n to sort it would let a + // typed number exhaust the heap. + val big = Vector( + backlog(p0, (1L to 400L).map(i => (i * 2, s"a$i"))*), + backlog(p1, (1L to 400L).map(i => (i * 2 + 1, s"b$i"))*) + ) + val run = skip(700, Vector(p0, p1), interleave(big*)) + assertTrue(run.maxHeld <= 2, run.dropped.size == 700, run.leftHeld == 0) ?? + s"held up to ${run.maxHeld} messages for a skip of 700" + }, + test("a topic that has drained its backlog does not stall the merge") { + // p1 holds one old message and nothing else. Without the +infinity rule the merge would + // wait forever for a second p1 head and the session would show nothing. + val arrivals = interleave( + backlog(p0, 10L -> "a1", 20L -> "a2", 30L -> "a3"), + backlog(p1, 5L -> "b1") + ) + val run = skip(2, Vector(p0, p1), arrivals) + assertTrue(run.dropped == Vector("b1", "a1"), run.delivered == Vector("a2", "a3"), run.leftHeld == 0) + }, + test("a topic that was already empty at session start never stalls the merge") { + val merge = GlobalSkipMerge[String](Vector(p0, p1), drainedAtStart = Set(p1), StartFromDiscard.shared(1)) + val run = drive(merge, backlog(p0, 10L -> "a1", 20L -> "a2")) + assertTrue(run.dropped == Vector("a1"), run.delivered == Vector("a2")) + }, + test("the same messages arriving in a different interleaving give the same answer") { + // Determinism: without the full tiebreak the answer would follow the delivery order. + val tied = Vector( + backlog(p0, 100L -> "a1", 100L -> "a2"), + backlog(p1, 100L -> "b1", 100L -> "b2"), + backlog(p2, 100L -> "c1", 100L -> "c2") + ) + val roundRobin = skip(3, Vector(p0, p1, p2), interleave(tied*)) + val oneAtATime = skip(3, Vector(p0, p1, p2), tied.flatten) + assertTrue(roundRobin.dropped == oneAtATime.dropped, roundRobin.dropped == Vector("a1", "a2", "b1")) ?? + s"roundRobin=${roundRobin.dropped} oneAtATime=${oneAtATime.dropped}" + }, + test("a single stream is exact without holding anything after the skip") { + val run = skip(2, Vector(p0), backlog(p0, 10L -> "a1", 20L -> "a2", 30L -> "a3")) + assertTrue(run.dropped == Vector("a1", "a2"), run.delivered == Vector("a3"), run.leftHeld == 0) + }, + test("a silent stream cannot make the merge grow without bound - the hot SOURCE is paused, not bounced") { + // p1 never delivers and never reaches its backlog end, so the merge can never know + // whether p1 holds something older than p0's head. At the watermark it marks p0 for + // PAUSE: in production the consumer stops delivering, so growth stops at the watermark + // plus the in-flight overshoot. Nothing is handed back - a declined message's + // redelivery races its own successors, which was the order bug - and nothing is + // decided while a stream that could still contribute has not spoken. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(5), maxHeld = 4) + (1L to 4L).foreach(i => merge.offer(p0, at(i, p0, i - 1), atBacklogEnd = false, s"a$i")) + val pausedAtCap = merge.desiredPausedStreams + // The pause is asynchronous, so a few in-flight messages still land: ACCEPTED, never bounced. + val inFlight = merge.offer(p0, at(5, p0, 4), atBacklogEnd = false, "a5") + assertTrue( + pausedAtCap == Set(p0), + inFlight.isEmpty, + merge.heldCount == 5, + merge.desiredPausedStreams == Set(p0), + !merge.desiredPausedStreams.contains(p1) // the BLIND stream is never paused + ) ?? s"pausedAtCap=$pausedAtCap inFlight=$inFlight held=${merge.heldCount}" + }, + test("AT THE MEMORY CAP THE MERGE MUST NOT GUESS which messages to drop") { + // The counterexample the cap used to answer wrongly: with n = 1 and room for two held + // messages, p0 delivers 100 and 200 while p1 is delayed. Advancing at the cap dropped + // p0/100 - and then p1 answered with 1, which was GLOBALLY EARLIEST and should have + // been the one message dropped, but was delivered instead. + // + // The count was exact either way; the SET was not, and the set is the contract. + val arrivals = Vector( + Arrival(p0, at(100, p0, 0), atBacklogEnd = false, "a1"), + Arrival(p0, at(200, p0, 1), atBacklogEnd = false, "a2"), + Arrival(p1, at(1, p1, 0), atBacklogEnd = true, "b1") + ) + val run = skip(1, Vector(p0, p1), arrivals, maxHeld = 2) + assertTrue( + run.dropped == Vector("b1"), + run.delivered == Vector("a1", "a2") + ) ?? s"dropped=${run.dropped} delivered=${run.delivered}" + }, + test("arrivals past the watermark are ACCEPTED - the pause has a bounded overshoot, never a bounce") { + // The pause takes effect asynchronously, so whatever was already in flight still + // lands. Accepting it is safe (it is in its stream's append order) and refusing it + // was the order bug. When the delayed stream finally speaks, everything held resolves + // exactly as if the stream had never been slow at all. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1), maxHeld = 2) + merge.offer(p0, at(100, p0, 0), atBacklogEnd = false, "a1") + merge.offer(p0, at(200, p0, 1), atBacklogEnd = false, "a2") + val pastWatermark = merge.offer(p0, at(300, p0, 2), atBacklogEnd = false, "a3") + val pausedBefore = merge.desiredPausedStreams + val afterDelayed = merge.offer(p1, at(1, p1, 0), atBacklogEnd = true, "b1") + assertTrue( + pastWatermark.isEmpty, // accepted and held, not bounced + pausedBefore == Set(p0), + afterDelayed == Vector( + "b1" -> StartFromOutcome.Drop, + "a1" -> StartFromOutcome.Deliver, + "a2" -> StartFromOutcome.Deliver, + "a3" -> StartFromOutcome.Deliver + ), + merge.desiredPausedStreams.isEmpty // the cut releases the pause + ) ?? s"pastWatermark=$pastWatermark pausedBefore=$pausedBefore afterDelayed=$afterDelayed" + }, + test("the stream the merge is BLIND on is never paused - it is what unblocks the merge") { + // Pausing everything at the cap would deadlock: the merge is waiting for exactly this + // stream. A blind stream has an empty queue, so no watermark can ever mark it. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1), maxHeld = 1) + merge.offer(p0, at(100, p0, 0), atBacklogEnd = false, "a1") + val fromBlind = merge.offer(p1, at(50, p1, 0), atBacklogEnd = true, "b1") + assertTrue(fromBlind == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Deliver)) ?? + s"the blind stream was refused at the cap and the merge could never advance: $fromBlind" + }, + test("progress is read off the merge's own budget") { + val discard = StartFromDiscard.shared(4) + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, discard) + drive(merge, interleave(backlog(p0, 10L -> "a1", 20L -> "a2"), backlog(p1, 30L -> "b1", 40L -> "b2"))) + assertTrue(merge.progressDiscard.map(_.total) == Some(4L), merge.progressDiscard.map(_.remaining) == Some(0L)) + }, + test("n = 0 delivers everything and holds nothing") { + val run = skip(0, Vector(p0, p1, p2), interleave(threePartitions*)) + assertTrue(run.dropped.isEmpty, run.delivered.size == 9, run.maxHeld == 0) + } + ) + + /** A topic described from its END: element 0 is the LAST entry, as `(publish time, messages in + * that entry)`. Entry ids are `"#"`, so an assertion names the topic and how far + * back the walk went. + */ + private def entries(spec: Map[String, Vector[(Long, Int)]]): String => Long => Option[LogEntry[String]] = + topicFqn => + k => + val log = spec.getOrElse(topicFqn, Vector.empty) + Option.when(k >= 1 && k <= log.size) { + val (publishTime, messages) = log(k.toInt - 1) + LogEntry(s"$topicFqn#$k", publishTime, messages) + } + + /** One unbatched entry per message, newest first. */ + private def unbatched(publishTimes: Long*): Vector[(Long, Int)] = publishTimes.toVector.map(_ -> 1) + + /** The entry order the walk uses on the `entries` labels: `#k` is the k-th entry from the END, so + * a larger ordinal is strictly OLDER. The production comparator is `MessageIdImpl.compareTo`; + * here the label carries the same information. */ + private def olderByHashOrdinal(a: String, b: String): Boolean = a.split("#").last.toInt > b.split("#").last.toInt + + /** The three partitions above as stored logs - newest entry first. */ + private val threeLogs = Map( + p0 -> unbatched(50L, 20L, 10L), + p1 -> unbatched(80L, 60L, 30L), + p2 -> unbatched(90L, 70L, 40L) + ) + + private def cut(n: Long, spec: Map[String, Vector[(Long, Int)]]): Map[String, LatestNSeek[String]] = + resolveLatestN(n, spec.keys.toVector.sorted, entries(spec), olderByHashOrdinal) + + /** THE LAST N, RESOLVED FROM ENTRY METADATA AND NOT FROM DELIVERED MESSAGES. + * + * Contract change context: "latest 2" on a 3-partition topic used to deliver SIX messages (the + * last 2 of each partition). It was then narrowed to the globally-last two by a top-n HEAP of + * delivered messages - which made the session's memory a number the user typed, let live + * traffic evict the historical tail it was supposed to be picking from, and showed nothing at + * all until every partition had drained. + * + * The cut is now computed BEFORE anything is delivered, by one merged backward walk over entry + * metadata: take whichever topic's current entry was published latest, count its messages, step + * that topic back one entry, repeat until n are accounted for. Every consumer then starts in + * the right place and simply streams. Memory is one cursor per topic; nothing is buffered. + */ + private val latestSuite = suite("resolve the cut for the globally-last n")( + test("cuts exactly n across the whole session, not n per partition") { + // THE contract change. Only the two partitions holding the newest messages contribute; + // the third is ANCHORED at its inspected tail - that entry is delivered and dropped - + // so it shows no history and still keeps anything published after the inspection. + assertTrue( + cut(2, threeLogs) == Map( + p0 -> LatestNSeek.FromEntry(s"$p0#1", 1L), + p1 -> LatestNSeek.FromEntry(s"$p1#1", 0L), + p2 -> LatestNSeek.FromEntry(s"$p2#1", 0L) + ) + ) ?? s"${cut(2, threeLogs)}" + }, + test("the cut is the global tail by publish time, spread over whichever partitions hold it") { + // b2(60) c2(70) b3(80) c3(90) - so p1 and p2 each go two entries back, p0 none. + assertTrue( + cut(4, threeLogs) == Map( + p0 -> LatestNSeek.FromEntry(s"$p0#1", 1L), + p1 -> LatestNSeek.FromEntry(s"$p1#2", 0L), + p2 -> LatestNSeek.FromEntry(s"$p2#2", 0L) + ) + ) ?? s"${cut(4, threeLogs)}" + }, + test("a partition holding only older messages is ANCHORED at its inspected tail") { + // Never at EARLIEST - that would show its whole log - and never at seek-time LATEST + // either: the seek happens after the inspection, so "latest" would silently jump any + // message published in between, where every CONTRIBUTING partition kept its concurrent + // appends. The anchor is the tail the walk actually saw; that one entry is delivered + // and dropped (a per-topic head-drop), which is exactly "everything after it". + assertTrue( + cut(1, threeLogs)(p0) == LatestNSeek.FromEntry(s"$p0#1", 1L), + cut(1, threeLogs)(p1) == LatestNSeek.FromEntry(s"$p1#1", 1L) + ) + }, + test("a non-contributing topic's BATCHED tail is anchored and dropped WHOLE") { + // The anchor drop is counted in messages, so a 10-message batch tail costs a + // 10-message head-drop - the seek can only land on the entry boundary. + val spec = Map(p0 -> Vector(100L -> 10), p1 -> unbatched(200L)) + assertTrue( + cut(1, spec) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 10L), p1 -> LatestNSeek.FromEntry(s"$p1#1", 0L)) + ) ?? s"${cut(1, spec)}" + }, + test("the cut reaches back into a partition only as far as it has to") { + // 5 asked for: c3(90) b3(80) c2(70) b2(60) a3(50) - p0 contributes its newest entry. + assertTrue( + cut(5, threeLogs) == Map( + p0 -> LatestNSeek.FromEntry(s"$p0#1", 0L), + p1 -> LatestNSeek.FromEntry(s"$p1#2", 0L), + p2 -> LatestNSeek.FromEntry(s"$p2#2", 0L) + ) + ) ?? s"${cut(5, threeLogs)}" + }, + test("HOLDS ONE ENTRY PER TOPIC - the answer never grows with n") { + // The memory contract, and the whole reason the heap is gone. 900 messages over three + // partitions, any n: the answer is three positions. + val big = Map( + p0 -> unbatched((1L to 300L).reverse.map(_ * 3)*), + p1 -> unbatched((1L to 300L).reverse.map(_ * 3 + 1)*), + p2 -> unbatched((1L to 300L).reverse.map(_ * 3 + 2)*) + ) + assertTrue(cut(10, big).size == 3, cut(500, big).size == 3) + }, + test("the walk asks the broker O(n / batch size) times, not once per partition per message") { + // Strictly cheaper than the per-topic walks this replaced, which cost that PER TOPIC. + var lookups = 0 + val counting: String => Long => Option[LogEntry[String]] = topicFqn => + k => + lookups += 1 + entries(threeLogs)(topicFqn)(k) + resolveLatestN(2, Vector(p0, p1, p2), counting, olderByHashOrdinal) + // Three to prime the cursors, then one step after taking c3. + assertTrue(lookups == 4) ?? s"$lookups lookups" + }, + test("a batched entry contributes all its messages, and the overshoot is discarded at the head") { + // A seek can only land on an entry boundary, so the only exact way to reach the n-th + // message inside a batch is to seek to its entry and drop what precedes it. + val batched = Map(p0 -> Vector(100L -> 10, 90L -> 10)) + assertTrue(cut(3, batched) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 7L))) ?? s"${cut(3, batched)}" + }, + test("the overshoot lands on the topic the walk STOPPED on, and on no other") { + // p2 contributes two unbatched entries (90, 85); the walk then takes p1's newest entry, + // a batch of 10, for the single message still wanted - so 9 are dropped from p1's head + // and NOTHING from p2's, whose two entries were both wanted in full. + val mixed = Map(p1 -> Vector(80L -> 10), p2 -> unbatched(90L, 85L, 70L)) + assertTrue( + cut(3, mixed) == Map(p1 -> LatestNSeek.FromEntry(s"$p1#1", 9L), p2 -> LatestNSeek.FromEntry(s"$p2#2", 0L)) + ) ?? s"${cut(3, mixed)}" + }, + test("a session holding fewer than n messages shows all of it - each topic anchored, not Everything") { + // Exhausted CONTRIBUTORS keep their oldest-entry anchor so the retention re-check + // covers them; `Everything` is reserved for a topic that held nothing when inspected. + val small = Map(p0 -> unbatched(10L), p1 -> unbatched(20L)) + assertTrue( + cut(50, small) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 0L), p1 -> LatestNSeek.FromEntry(s"$p1#1", 0L)) + ) ?? s"${cut(50, small)}" + }, + test("an empty partition contributes nothing and cannot hold the cut back") { + // The heap had to wait for every partition to reach its recorded end before it could + // release anything, so an empty or stalled partition showed the user zero messages. + // A partition with no entries simply answers nothing here. + val withEmpty = Map(p0 -> unbatched(30L, 20L, 10L), p1 -> Vector.empty[(Long, Int)]) + // The empty partition maps to EVERYTHING: it held nothing when inspected, so whatever + // it holds at seek time arrived after the inspection - live traffic the session shows. + assertTrue(cut(2, withEmpty) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#2", 0L), p1 -> LatestNSeek.Everything)) + }, + test("a publish-time tie is cut deterministically, by topic name") { + // The same tiebreak [[MessageOrderKey]] uses, so an entry-level cut and a message-level + // order cannot disagree. + val tied = Map(p0 -> unbatched(100L, 100L), p1 -> unbatched(100L, 100L)) + assertTrue( + cut(2, tied) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 1L), p1 -> LatestNSeek.FromEntry(s"$p1#2", 0L)) + ) ?? s"${cut(2, tied)}" + }, + test("a clamping broker is detected within a bounded number of steps, not by the first repeat") { + // `examineMessage` CLAMPS rather than fails on the earliest side; a Pulsar version that + // clamped on the latest side too would otherwise make the running total grow forever. + // + // CHANGED EXPECTATION, deliberately: the old code concluded "exhausted" the instant an + // entry id repeated and stopped in exactly 3 lookups. That same first repeat is what a + // SINGLE concurrent append also produces (the moving anchor - see `movingAnchorSuite`), + // so reading it as exhaustion was the whole-backlog bug. A clamp is now told apart from + // an append by ONE VERIFICATION LOOKUP at the k that produced the last accepted entry - + // a clamped end never moves, a grown end answers a newer entry - so it still resolves + // to `Everything`, one lookup later than the old first-repeat guard. + var lookups = 0 + val clamping: String => Long => Option[LogEntry[String]] = topicFqn => + k => + lookups += 1 + val clamped = k.min(2).max(1) + Some(LogEntry(s"$topicFqn#$clamped", 100L - clamped, 3)) + val resolved = resolveLatestN(100, Vector(p0), clamping, olderByHashOrdinal) + assertTrue( + resolved == Map(p0 -> LatestNSeek.FromEntry(s"$p0#2", 0L)), + lookups > 3, + lookups <= maxLatestNReanchorSteps + 4 + ) ?? s"resolved=$resolved lookups=$lookups (bound $maxLatestNReanchorSteps)" + }, + test("a walk that outlives its TIME budget fails with guidance instead of grinding on") { + // N bounds the request; TIME bounds the cost - an unbatched topic pays one broker + // lookup per entry, and n alone cannot tell a thousand lookups from ten million. + var clock = 0L + val slowLog = Map(p0 -> unbatched((1L to 100L).reverse.map(_ * 10)*)) + val slow: String => Long => Option[LogEntry[String]] = topicFqn => + k => + clock += 1_000L // each lookup costs a second + entries(slowLog)(topicFqn)(k) + val outcome = scala.util.Try( + resolveLatestN(50, Vector(p0), slow, olderByHashOrdinal, resolveBudgetMs = 5_000L, nowMs = () => clock) + ) + assertTrue( + outcome.isFailure, + outcome.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + outcome.failed.toOption.exists(_.getMessage.contains("budget")) + ) ?? s"$outcome" + }, + test("a walk inside its time budget is untouched by it") { + var clock = 0L + val counting: String => Long => Option[LogEntry[String]] = topicFqn => + k => + clock += 10L + entries(threeLogs)(topicFqn)(k) + val resolved = resolveLatestN(2, Vector(p0, p1, p2), counting, olderByHashOrdinal, resolveBudgetMs = 5_000L, nowMs = () => clock) + assertTrue(resolved.values.count { case LatestNSeek.FromEntry(_, _) => true; case _ => false } == 3) + }, + test("n = 0 asks the broker nothing and positions everything at the live tail") { + var lookups = 0 + val counting: String => Long => Option[LogEntry[String]] = topicFqn => + k => + lookups += 1 + entries(threeLogs)(topicFqn)(k) + val resolved = resolveLatestN(0, Vector(p0, p1), counting, olderByHashOrdinal) + assertTrue(resolved == Map(p0 -> LatestNSeek.Nothing, p1 -> LatestNSeek.Nothing), lookups == 0) + }, + test("one topic named twice is walked once") { + // A session's topic vector is the concatenation of every enabled target's resolved + // topics, and two targets may legitimately select the same topic. + var lookups = 0 + val counting: String => Long => Option[LogEntry[String]] = topicFqn => + k => + lookups += 1 + entries(threeLogs)(topicFqn)(k) + val resolved = resolveLatestN(1, Vector(p0, p0, p0), counting, olderByHashOrdinal) + assertTrue(resolved == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 0L)), lookups == 1) ?? s"lookups=$lookups" + } + ) + + /** A log with STABLE absolute entry ids `e1..eN` (e1 oldest), read the way + * `examineMessage(topic, "latest", k)` reads it: the k-th entry counted back from the CURRENT + * end. `appendsBefore` names the lookup ordinals at which a producer appends one entry FIRST - + * i.e. the anchor the walk counts back from moves forward under it, exactly as it does when a + * real producer writes during session creation. A frozen-log lambda cannot express this, which + * is why every existing latest-n test missed the moving anchor. + */ + private def growingLog(startEntries: Long, appendsBefore: Map[Int, Long] = Map.empty): (String => Long => Option[LogEntry[String]], () => Int) = + var total = startEntries + var lookups = 0 + val lambda: String => Long => Option[LogEntry[String]] = _ => + k => + lookups += 1 + total += appendsBefore.getOrElse(lookups, 0L) + val idx = total - k + 1 // 1-based index from the start; 1 is the oldest retained entry + Option.when(idx >= 1) { LogEntry(s"e$idx", idx * 10, 1) } + (lambda, () => lookups) + + /** Older = smaller absolute index, which is how `MessageIdImpl.compareTo` orders real entry ids + * (ledger, then entry). Shared by the growing-log tests so the walk can tell a strictly-older + * answer from a re-anchored one. */ + private def olderByAbsoluteId(a: String, b: String): Boolean = a.drop(1).toLong < b.drop(1).toLong + + private val movingAnchorSuite = suite("the backward walk survives a log that grows under it")( + test("a SINGLE append per gap is not misread as exhaustion - the whole backlog bug") { + // e1..e10 at start; the last 3 are e8, e9, e10. One entry is appended just before the + // SECOND lookup, so `latest, 2` answers e10 AGAIN (the anchor moved forward by one). + // The old guard read that repeated id as "exhausted", fell back to EARLIEST, and a + // request for the last 3 delivered the entire backlog while reporting success. + val (log, _) = growingLog(startEntries = 10, appendsBefore = Map(2 -> 1L)) + val resolved = resolveLatestN(3, Vector(p0), log, olderByAbsoluteId) + assertTrue(resolved == Map(p0 -> LatestNSeek.FromEntry("e8", 0L))) ?? + s"a single concurrent append turned 'latest 3' into ${resolved(p0)}" + }, + test("a BURST of appends does not make the cursor jump forward and double-count") { + // e1..e10 at start (last 4 = e7..e10); two entries land before the second lookup, so + // `latest, 2` answers e11 - NEWER than the e10 just taken. Taking it walks the cursor + // FORWARD off the contiguous suffix and double-counts, cutting the history too shallow + // (the old code stopped at e9, hiding e7 and e8). The cut must still be the four that + // were newest AT START, namely e7..e10; the two live appends stream in behind them. + val (log, _) = growingLog(startEntries = 10, appendsBefore = Map(2 -> 2L)) + val resolved = resolveLatestN(4, Vector(p0), log, olderByAbsoluteId) + assertTrue(resolved == Map(p0 -> LatestNSeek.FromEntry("e7", 0L))) ?? + s"a burst of concurrent appends cut 'latest 4' at ${resolved(p0)} instead of e7" + }, + test("a log that OUTRUNS the walk for the whole bound fails LOUDLY, naming the topic") { + // One entry lands before EVERY lookup, so the anchor moves exactly as fast as the walk + // steps and `latest, k` keeps answering at or above the entry just taken, forever. + // There is no honest "last n" on such a topic at this moment, and both silent endings + // answer a question nobody asked: classifying it as exhaustion seeks EARLIEST (the + // whole backlog as a successful session - the original defect), and stopping short + // delivers fewer than n as success. Refusing is the only honest outcome, and it names + // the topic so the user knows which producer to quiet down. + val (log, lookups) = growingLog(startEntries = 200, appendsBefore = (1 to 1_000).map(i => i -> 1L).toMap) + val outcome = scala.util.Try(resolveLatestN(5, Vector(p0), log, olderByAbsoluteId)) + assertTrue( + outcome.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + outcome.failed.toOption.exists(_.getMessage.contains(p0)), + // The refusal is BOUNDED: a prime, one ambiguous repeat plus its verification, and + // at most the re-anchor budget of catch-up lookups - never an unbounded chase. + lookups() <= maxLatestNReanchorSteps + 8 + ) ?? s"outcome=$outcome after ${lookups()} lookups" + } + ) + + /** THE CONTRACT, STATED RATHER THAN APOLOGISED FOR: APPEND ORDER WITHIN A PARTITION, PUBLISH + * TIME ACROSS PARTITIONS. + * + * Pulsar preserves APPEND order within a partition and stamps `publishTime` from the PRODUCER's + * clock. Those two are not the same thing: several producers writing one partition, or one + * producer whose clock steps back, append publish times that run backwards inside a single log. + * + * Neither counting mode reads a whole log, and neither can: "skip first n" is a streaming merge + * over per-stream heads, "latest n" is a backward walk over per-topic entry cursors, and both + * would have to buffer or scan an entire partition to notice that it is not in clock order. + * That is O(topic) at any n, which is the cost both designs exist to avoid. + * + * So the guarantee is: the COUNT is exact without qualification, and WHICH messages make the + * cut is exact as far as each log really is in publish-time order. These tests pin that as the + * real behaviour - including the case where it visibly differs from an exact publish-time + * answer - so no documentation, UI label or downstream test can claim more than the code does. + * The UI labels ("Skip first n messages", "Latest n messages") must not imply otherwise. + */ + private val nonMonotonicSuite = suite("the contract: append order within a partition")( + test("a stream whose publish times run backwards is NOT resorted") { + // p0's second message is stamped OLDER than its first - two producers, or one clock + // that stepped back. Globally by publish time the oldest message is a2 (1), so an + // order-by-publish-time contract would drop a2. The merge drops b1 (50) instead, + // because within p0 it takes the log's order as given and a2 is not yet a head. + val arrivals = Vector( + Arrival(p0, at(100, p0, 0), atBacklogEnd = false, "a1"), + Arrival(p1, at(50, p1, 0), atBacklogEnd = true, "b1"), + Arrival(p0, at(1, p0, 1), atBacklogEnd = true, "a2") + ) + val run = skip(1, Vector(p0, p1), arrivals) + assertTrue( + run.dropped == Vector("b1"), + run.delivered == Vector("a1", "a2") + ) ?? s"dropped=${run.dropped} delivered=${run.delivered}" + }, + test("an INTERIOR clock reversal is not resorted either, and the count survives it") { + // Not merely at the tail: p0's middle message is the oldest thing in the session. An + // exact publish-time contract would drop a2(1) and a3(5); the merge drops a1(100) and + // a2(1), because a3 is not a head until a2 has been taken. + val arrivals = Vector( + Arrival(p0, at(100, p0, 0), atBacklogEnd = false, "a1"), + Arrival(p0, at(1, p0, 1), atBacklogEnd = false, "a2"), + Arrival(p0, at(5, p0, 2), atBacklogEnd = true, "a3"), + Arrival(p1, at(200, p1, 0), atBacklogEnd = true, "b1") + ) + val run = skip(2, Vector(p0, p1), arrivals) + assertTrue( + run.dropped == Vector("a1", "a2"), + run.delivered == Vector("a3", "b1") + ) ?? s"dropped=${run.dropped} delivered=${run.delivered}" + }, + test("the COUNT is exact even when a stream's clock runs backwards") { + // The half of the contract that does survive: n messages are dropped, whatever the + // producer clocks did. Only WHICH n is approximate. + val arrivals = Vector( + Arrival(p0, at(100, p0, 0), atBacklogEnd = false, "a1"), + Arrival(p1, at(50, p1, 0), atBacklogEnd = false, "b1"), + Arrival(p0, at(1, p0, 1), atBacklogEnd = true, "a2"), + Arrival(p1, at(2, p1, 1), atBacklogEnd = true, "b2") + ) + val counts = Vector(0, 1, 2, 3, 4).map(n => n -> skip(n, Vector(p0, p1), arrivals)) + assertTrue(counts.forall((n, run) => run.dropped.size == n && run.delivered.size == 4 - n)) ?? + s"${counts.map((n, run) => s"n=$n dropped=${run.dropped.size} delivered=${run.delivered.size}")}" + }, + test("LATEST-N CUTS BY APPEND POSITION TOO - a buried high-timestamp message is not found") { + // p0's newest ENTRY is stamped 1 while an older entry of the same log is stamped 100. + // An exact publish-time answer for "the latest 1" would be that buried a1(100). The + // walk compares each topic's CURRENT entry, so it sees p0 offering 1, prefers p1's + // 50, and never looks deeper into p0. + // + // Deliberate, and it is the same limit as everywhere else here: finding that message + // would mean scanning the whole log. Recorded as behaviour so the contract and the code + // cannot drift apart - NOT as the answer an exact publish-time contract would give. + val reversed = Map(p0 -> unbatched(1L, 100L), p1 -> unbatched(50L)) + assertTrue( + cut(1, reversed) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 1L), p1 -> LatestNSeek.FromEntry(s"$p1#1", 0L)) + ) ?? s"${cut(1, reversed)}" + }, + test("the COUNT of a latest-n cut is exact even when a log's clock ran backwards") { + val reversed = Map(p0 -> unbatched(1L, 100L), p1 -> unbatched(50L)) + // Two asked for: p1's only entry, then p0's newest. Exactly two messages, whatever the + // clocks did. + assertTrue( + cut(2, reversed) == Map(p0 -> LatestNSeek.FromEntry(s"$p0#1", 0L), p1 -> LatestNSeek.FromEntry(s"$p1#1", 0L)) + ) ?? s"${cut(2, reversed)}" + } + ) + + /** WHAT HAPPENS AFTER THE SKIP'S BUDGET IS SPENT. + * + * The merge exists to decide WHICH n messages are dropped. Once the budget is spent it stops + * merging entirely and every later message is passed straight through, because continuing to + * merge would mean holding a message from every stream for the whole life of the session - + * unbounded memory for a session that is now just streaming. + * + * So the delivery SEQUENCE after the cut is the order the brokers delivered in, one listener + * thread per physical topic. Pinned here so no documentation or downstream test can claim a + * global ordering that the code deliberately does not provide. + */ + private val postCutSuite = suite("delivery order after the cut")( + test("the budget's LAST unit finishes the skip immediately - no N+1st head is waited for") { + // Skip 1 over two streams; the drop EMPTIES p1, which is not at its backlog end. The + // old code kept `dropping` true until a LATER claim answered no, so it went blind on + // p1 and held a1 hostage - for up to the whole stall window - to learn something the + // spent budget had already decided. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1)) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") // held; p1 blind + val resolved = merge.offer(p1, at(5, p1, 0), atBacklogEnd = false, "b1") + assertTrue( + resolved == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Deliver), + merge.heldCount == 0 + ) ?? s"resolved=$resolved held=${merge.heldCount}" + }, + test("settleIfDone flips exactly when the budget is spent and nothing is held - and only then") { + // The settled flag is what lets the session drop its ordering lock for the rest of its + // life, so it must never flip early - and it only flips when ASKED, after the settling + // batch has been fully handled by the listener. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1)) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + merge.settleIfDone() + val midSkip = merge.isSettled // still dropping, one message held: must stay false + merge.offer(p1, at(5, p1, 0), atBacklogEnd = true, "b1") // spends the budget, drains a1 + val beforeAsked = merge.isSettled // one-way, but only settleIfDone may flip it + merge.settleIfDone() + assertTrue(!midSkip, !beforeAsked, merge.isSettled) + }, + test("after the cut, a later message can be delivered before an older one from another stream") { + // Both streams are individually monotonic, so this is not the clock problem above: it + // is simply that nothing is held back once the skip is done. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1)) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + val atCut = merge.offer(p1, at(5, p1, 0), atBacklogEnd = true, "b1") + val afterA = merge.offer(p0, at(100, p0, 1), atBacklogEnd = false, "a2") + val afterB = merge.offer(p1, at(30, p1, 1), atBacklogEnd = false, "b2") + + assertTrue( + atCut == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Deliver), + afterA == Vector("a2" -> StartFromOutcome.Deliver), + afterB == Vector("b2" -> StartFromOutcome.Deliver), + merge.heldCount == 0 + ) ?? s"a2 (published at 100) was delivered before b2 (published at 30): atCut=$atCut afterA=$afterA afterB=$afterB" + }, + test("nothing at all is held once the budget is spent") { + // The memory half of the same decision, and the reason it is not going to change. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(1)) + merge.offer(p1, at(5, p1, 0), atBacklogEnd = true, "b1") + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + (1 to 500).foreach(i => merge.offer(p0, at(1000L + i, p0, i.toLong), atBacklogEnd = false, s"a$i")) + assertTrue(merge.heldCount == 0) + } + ) + + /** A silent waited-for stream must not hold the merge forever. */ + private val stallSuite = suite("a stream that never speaks is bounded and surfaced, not waited on forever")( + test("RESETTING the stall clock hands a silent stream a fresh window - paused time proves nothing") { + // The runner resets on RESUME: a pause holds every source, so a stream that was blind + // for one second before a five-minute pause must get a full window after it - the old + // wall-clock accounting abandoned it on the first sweep after resume. + var clock = 1_000L + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(1), + maxHeld = 2, + stallWindowMs = startFromMergeStallWindowMs, + nowMs = () => clock + ) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") // held; p1 blind + clock += startFromMergeStallWindowMs - 1_000 // a long user pause elapses + merge.resetStallClock() // resume + clock += 2_000 // two REAL seconds after resume + val justAfterResume = merge.sweepStalled() // old code: gave up here (31s elapsed) + val stillWaiting = merge.waitingOn.contains(p1) + clock += startFromMergeStallWindowMs + 1 // a full window of real silence + val afterRealWindow = merge.sweepStalled() + assertTrue( + justAfterResume.isEmpty, + stillWaiting, + afterRealWindow == Vector("a1" -> StartFromOutcome.Drop), + merge.heldCount == 0 + ) ?? s"justAfterResume=$justAfterResume stillWaiting=$stillWaiting afterRealWindow=$afterRealWindow" + }, + test("a stream silent past the give-up window is abandoned and the merge advances") { + // p1's backlog was trimmed after the session recorded its end, so it delivers nothing. + // p0 reaches the watermark and is marked for PAUSE (its in-flight tail still lands). + // Once p1 has stayed silent past the window the merge stops waiting for it, drains + // what it held, and carries on - instead of holding a paused world forever in silence. + var clock = 1_000L + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(1), + maxHeld = 2, + stallWindowMs = startFromMergeStallWindowMs, + nowMs = () => clock + ) + val a1 = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") // held; p1 blind + val a2 = merge.offer(p0, at(20, p0, 1), atBacklogEnd = false, "a2") // held; watermark reached + val a3 = merge.offer(p0, at(30, p0, 2), atBacklogEnd = false, "a3") // in-flight overshoot: accepted, p0 marked for pause + val pausedWhileBlind = merge.desiredPausedStreams + val waitingBefore = merge.waitingOn + + clock += startFromMergeStallWindowMs + 1 // p1 still silent, past the window + val afterGiveUp = merge.offer(p0, at(40, p0, 3), atBacklogEnd = false, "a4") + + assertTrue( + a1.isEmpty, + a2.isEmpty, + a3.isEmpty, + pausedWhileBlind == Set(p0), + waitingBefore == Set(p1), + afterGiveUp == Vector( + "a1" -> StartFromOutcome.Drop, + "a2" -> StartFromOutcome.Deliver, + "a3" -> StartFromOutcome.Deliver, + "a4" -> StartFromOutcome.Deliver + ), + !merge.waitingOn.contains(p1) // p1 was abandoned; p0 is momentarily empty but still legitimately waited for + ) ?? s"pausedWhileBlind=$pausedWhileBlind waitingBefore=$waitingBefore afterGiveUp=$afterGiveUp stillWaiting=${merge.waitingOn}" + }, + test("the SWEEP gives up with NO further offer - the last backlog message has nobody behind it") { + // p0 delivered everything it had and ended its backlog; p1 was trimmed and never + // speaks. The offer-driven check can never run again - there are no offers left - so + // only the time-driven sweep can honour the window. It used to hang forever here. + var clock = 1_000L + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(1), + maxHeld = 2, + stallWindowMs = startFromMergeStallWindowMs, + nowMs = () => clock + ) + val a1 = merge.offer(p0, at(10, p0, 0), atBacklogEnd = true, "a1") // p0's LAST message; p1 blind + val beforeSweep = merge.sweepStalled() // window not yet passed: nothing moves + + clock += startFromMergeStallWindowMs + 1 + val swept = merge.sweepStalled() + + assertTrue( + a1.isEmpty, + beforeSweep.isEmpty, + swept == Vector("a1" -> StartFromOutcome.Drop), + merge.heldCount == 0 + ) ?? s"beforeSweep=$beforeSweep swept=$swept held=${merge.heldCount}" + }, + test("a stream that speaks before the window is NOT given up on - a slow stream is not cut") { + var clock = 1_000L + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(1), + maxHeld = 2, + stallWindowMs = startFromMergeStallWindowMs, + nowMs = () => clock + ) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + clock += startFromMergeStallWindowMs - 1 // just under the window + val stillWaiting = merge.waitingOn + // p1 finally speaks: its message is the globally-earliest, so it is dropped and a1 delivered. + val fromP1 = merge.offer(p1, at(5, p1, 0), atBacklogEnd = true, "b1") + assertTrue( + stillWaiting == Set(p1), + fromP1 == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Deliver) + ) ?? s"stillWaiting=$stillWaiting fromP1=$fromP1" + } + ) + + /** The exactness limit at the cap boundary - documented, not fixed (see [[startFromMergeMaxHeld]]). */ + private val capBoundarySuite = suite("the watermark keeps the COUNT exact but not always WHICH n")( + test("a NON-MONOTONIC log can still cross streams at the boundary - the append-order contract, not the flow control") { + // p0's log is not in publish-time order (two producers, or a clock that stepped back): + // its entries append as A(10), B(40), C(20). All three are accepted - nothing is + // declined any more - and the drops follow HEAD order: A(10) against b1(50), then + // B(40), because C sits BEHIND B in its own log. The COUNT is exactly 2; the SET is + // the append-order answer: the globally-earliest two by publish time are A(10) and + // C(20), but C cannot be seen past B. This is [[MessageOrderKey]]'s documented + // contract surfacing - reordering a stream against its own log would mean reading the + // whole log - and no watermark or pause changes it. Pinned so the limit stays + // documented rather than believed away. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(2), maxHeld = 2) + val a = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "A") + val b = merge.offer(p0, at(40, p0, 1), atBacklogEnd = false, "B") + val c = merge.offer(p0, at(20, p0, 2), atBacklogEnd = false, "C") // accepted past the watermark + val resolved = merge.offer(p1, at(50, p1, 0), atBacklogEnd = true, "b1") + val droppedSet = resolved.collect { case (v, StartFromOutcome.Drop) => v }.toSet + assertTrue( + a.isEmpty, + b.isEmpty, + c.isEmpty, + resolved == Vector( + "A" -> StartFromOutcome.Drop, + "B" -> StartFromOutcome.Drop, + "C" -> StartFromOutcome.Deliver, + "b1" -> StartFromOutcome.Deliver + ), + droppedSet == Set("A", "B"), // the limit: C(20) should have been dropped instead of B(40) + !droppedSet.contains("C") + ) ?? s"resolved=$resolved" + } + ) + + /** FLOW CONTROL REPLACED DECLINING, AND WITH IT THE WHOLE OVERTAKE CLASS. + * + * A declined message came back through broker redelivery while its successors kept arriving, + * so a stream could re-enter the merge out of its own append order - the one premise a k-way + * merge cannot survive. The floor guard held the door for the FIRST declined message, but + * successors declined BY THE GUARD were not remembered: two of them returning out of order + * could still spend the budget's last unit on the wrong message, on a perfectly monotonic + * stream. Nothing is declined now - hot sources are PAUSED - so there is nothing to return + * out of order, and the counterexample is structurally impossible: the first test drives the + * exact sequence that used to break. + */ + private val flowControlSuite = suite("hot sources are paused, and a stream can no longer overtake itself")( + test("THE OLD COUNTEREXAMPLE IS GONE: the exact set survives a full merge on a monotonic stream") { + // Skip 5, room for 2. Under declining: t7 bounced at the cap, t8 bounced by the floor + // guard UNREMEMBERED, t7's return cleared the floor, and t9 could then spend the last + // unit while t8 was still in flight - dropped {t1,t5,t6,t7,t9}, t8 delivered. Under + // pause the arrivals are simply accepted in order and the dropped set is exactly the + // first five of the merged stream: {t1,t5,t6,t7,t8}. + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.shared(5), maxHeld = 2) + val h5 = merge.offer(p1, at(50, p1, 0), atBacklogEnd = false, "t5") + val h6 = merge.offer(p1, at(60, p1, 1), atBacklogEnd = false, "t6") + val h7 = merge.offer(p1, at(70, p1, 2), atBacklogEnd = false, "t7") // past the watermark: accepted, p1 marked for pause + val burst = merge.offer(p0, at(10, p0, 0), atBacklogEnd = true, "t1") + val t8 = merge.offer(p1, at(80, p1, 3), atBacklogEnd = false, "t8") + val t9 = merge.offer(p1, at(90, p1, 4), atBacklogEnd = false, "t9") + + val dropped = (burst ++ t8 ++ t9).collect { case (v, StartFromOutcome.Drop) => v } + assertTrue( + h5.isEmpty, + h6.isEmpty, + h7.isEmpty, + merge.desiredPausedStreams.isEmpty, // budget spent at t8: the pause lifted with it + burst == Vector( + "t1" -> StartFromOutcome.Drop, + "t5" -> StartFromOutcome.Drop, + "t6" -> StartFromOutcome.Drop, + "t7" -> StartFromOutcome.Drop + ), + t8 == Vector("t8" -> StartFromOutcome.Drop), + t9 == Vector("t9" -> StartFromOutcome.Deliver), + dropped.toSet == Set("t1", "t5", "t6", "t7", "t8") + ) ?? s"burst=$burst t8=$t8 t9=$t9 paused=${merge.desiredPausedStreams}" + }, + test("a stream paused for its own queue RESUMES once it drains below the low watermark") { + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + Set.empty, + StartFromDiscard.shared(10), + pauseStreamAt = 3, + resumeStreamAt = 1 + ) + merge.offer(p0, at(30, p0, 0), atBacklogEnd = false, "a1") + merge.offer(p0, at(40, p0, 1), atBacklogEnd = false, "a2") + merge.offer(p0, at(50, p0, 2), atBacklogEnd = false, "a3") + val pausedAtHigh = merge.desiredPausedStreams + // p1's NEWER head lets the merge drain p0's whole queue - well under the low watermark. + merge.offer(p1, at(100, p1, 0), atBacklogEnd = true, "b1") + val afterDrain = merge.desiredPausedStreams + assertTrue( + pausedAtHigh == Set(p0), + afterDrain.isEmpty, + merge.heldCount == 1 // only b1: p0 drained and is blind again, so its pause lifted + ) ?? s"pausedAtHigh=$pausedAtHigh afterDrain=$afterDrain held=${merge.heldCount}" + }, + test("the BYTE watermark pauses a fat stream that the count watermarks would never notice") { + val merge = GlobalSkipMerge[String]( + Vector(p0, p1), + Set.empty, + StartFromDiscard.shared(5), + pauseBytesAt = 10L, + resumeBytesAt = 5L, + payloadBytesOf = (v: String) => v.length.toLong + ) + merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "aaaaaa") // 6 bytes: under + val underBytes = merge.desiredPausedStreams + merge.offer(p0, at(20, p0, 1), atBacklogEnd = false, "bbbbbb") // 12 bytes total: over + val overBytes = merge.desiredPausedStreams + assertTrue( + underBytes.isEmpty, + overBytes == Set(p0), + merge.heldBytesCount == 12L, + !overBytes.contains(p1) // blind, and empty-queued: never paused + ) ?? s"under=$underBytes over=$overBytes heldBytes=${merge.heldBytesCount}" + } + ) + + /** A broker unload redelivers everything un-acked while the originals may still be HELD in the + * merge or already decided. Re-deciding a copy would spend a second budget unit on one + * message - count exact, set short. The guard is the per-stream APPEND position watermark: + * within one stream it only grows (publish time does not have to), so "at or below" is + * exactly "offered before". + */ + private val duplicateSuite = suite("a redelivered duplicate never spends a second budget unit")( + test("a duplicate of a message still HELD is handed back - no claim, and no acknowledgment either") { + // Acknowledging the copy was the loss path: the ack burned the message id, so an + // original whose DELIVERY later failed could never be redelivered. The copy is + // requeued instead; the broker's backoff retries it until the original settles. + val discard = StartFromDiscard.shared(2) + val merge = GlobalSkipMerge[String](Vector(p0, p1), Set.empty, discard) + val first = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + val copy = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1-copy") // unload redelivery + val resolved = merge.offer(p1, at(5, p1, 0), atBacklogEnd = true, "b1") + assertTrue( + first.isEmpty, + copy == Vector("a1-copy" -> StartFromOutcome.Requeue), // decided NOTHING - handed back + resolved == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Drop), + discard.remaining == 0L // exactly two claims: b1 and a1 - the copy spent nothing + ) ?? s"copy=$copy resolved=$resolved remaining=${discard.remaining}" + }, + test("a duplicate of a message already DECIDED is handed back without a claim") { + // Requeue is safe here too: a successfully acknowledged original produces no more + // copies, and one whose ACK failed is finalized by the listener's paperwork check + // before the merge is ever asked - so a copy reaching this arm always still has an + // open fate somewhere downstream, and deciding nothing is the only safe answer. + val discard = StartFromDiscard.shared(2) + val merge = GlobalSkipMerge[String](Vector(p0), Set.empty, discard) + val dropped = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1") + val copy = merge.offer(p0, at(10, p0, 0), atBacklogEnd = false, "a1-copy") + val next = merge.offer(p0, at(20, p0, 1), atBacklogEnd = true, "a2") + assertTrue( + dropped == Vector("a1" -> StartFromOutcome.Drop), + copy == Vector("a1-copy" -> StartFromOutcome.Requeue), + next == Vector("a2" -> StartFromOutcome.Drop), + discard.remaining == 0L // exactly two claims: a1 and a2, never the copy + ) ?? s"dropped=$dropped copy=$copy next=$next remaining=${discard.remaining}" + }, + test("a batched entry's LATER piece is not mistaken for a duplicate of an earlier one") { + // Same ledger and entry, higher batch index: a legitimate successor, not a copy. + val discard = StartFromDiscard.shared(1) + val merge = GlobalSkipMerge[String](Vector(p0), Set.empty, discard) + val first = merge.offer(p0, at(10, p0, 0, batchIndex = 0), atBacklogEnd = false, "a1#0") + val second = merge.offer(p0, at(10, p0, 0, batchIndex = 1), atBacklogEnd = true, "a1#1") + assertTrue( + first == Vector("a1#0" -> StartFromOutcome.Drop), + second == Vector("a1#1" -> StartFromOutcome.Deliver), + discard.remaining == 0L + ) ?? s"first=$first second=$second" + } + ) + + private val sharedCounterSuite = suite("the merge refuses a counter it could never claim")( + test("a PerTopic discard is refused at construction - the merge claims by STREAM id") { + // `advance` claims `discard.claim(streamId)`, and a stream id ("consumer@topic") is not + // a topic FQN. Only a SHARED counter matches any key; a PerTopic counter would never + // find its key, never claim, and the skip would silently drop nothing at all. The + // session wiring only ever hands the merge a shared counter today - this makes wiring + // anything else fail at construction instead of never-claiming. + val outcome = scala.util.Try( + GlobalSkipMerge[String](Vector(p0, p1), Set.empty, StartFromDiscard.perTopic(Map(p0 -> 3L))) + ) + assertTrue( + outcome.isFailure, + outcome.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) ?? s"a PerTopic counter was accepted: $outcome" + } + ) + + def spec = suite(this.getClass.toString)( + orderSuite, + backlogEndSuite, + skipSuite, + latestSuite, + movingAnchorSuite, + nonMonotonicSuite, + postCutSuite, + stallSuite, + capBoundarySuite, + flowControlSuite, + duplicateSuite, + sharedCounterSuite + ) diff --git a/server/src/test/scala/consumer/session_runner/handleStartFromTest.scala b/server/src/test/scala/consumer/session_runner/handleStartFromTest.scala new file mode 100644 index 000000000..eccebff05 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/handleStartFromTest.scala @@ -0,0 +1,142 @@ +package consumer.session_runner + +import consumer.start_from.{DateTimeUnit, RelativeDateTime} +import zio.test.* + +import java.time.{ZoneId, ZonedDateTime} + +/** `resolveRelativeDateTime` turns a "N units ago" selection into the timestamp a consumer seeks to. + * + * Regression context: rounding used `ZonedDateTime.truncatedTo`, which REJECTS any unit larger than + * a day - so Week/Month/Year with "round to unit start" threw UnsupportedTemporalTypeException. + * Three of the seven units were broken for an ordinary UI selection, surfacing only as a generic + * FAILED_PRECONDITION on the whole session. + * + * A frozen `now` keeps every case deterministic (the production call passes ZonedDateTime.now()). + */ +object handleStartFromTest extends ZIOSpecDefault: + + // A Wednesday, mid-month, mid-year, with non-zero time-of-day so truncation is observable. + private val now = ZonedDateTime.of(2026, 7, 15, 13, 47, 29, 123_000_000, ZoneId.of("UTC")) + + private def resolve(value: Int, unit: DateTimeUnit, rounded: Boolean): ZonedDateTime = + resolveRelativeDateTime(RelativeDateTime(value = value, unit = unit, isRoundedToUnitStart = rounded), now) + + private val allUnits = List( + DateTimeUnit.Second, + DateTimeUnit.Minute, + DateTimeUnit.Hour, + DateTimeUnit.Day, + DateTimeUnit.Week, + DateTimeUnit.Month, + DateTimeUnit.Year + ) + + def spec = suite(this.getClass.toString)( + test("every unit resolves when rounding is requested") { + // The regression itself: Week/Month/Year used to throw here. + val failures = allUnits.flatMap { unit => + scala.util.Try(resolve(1, unit, rounded = true)).failed.toOption.map(t => s"$unit -> ${t.getClass.getSimpleName}") + } + assertTrue(failures.isEmpty) ?? s"units that threw while rounding: ${failures.mkString(", ")}" + }, + test("every unit resolves without rounding") { + val failures = allUnits.flatMap { unit => + scala.util.Try(resolve(1, unit, rounded = false)).failed.toOption.map(t => s"$unit -> ${t.getClass.getSimpleName}") + } + assertTrue(failures.isEmpty) ?? s"units that threw: ${failures.mkString(", ")}" + }, + test("unrounded units subtract exactly, preserving time-of-day") { + assertTrue( + resolve(30, DateTimeUnit.Second, rounded = false) == now.minusSeconds(30), + resolve(30, DateTimeUnit.Minute, rounded = false) == now.minusMinutes(30), + resolve(5, DateTimeUnit.Hour, rounded = false) == now.minusHours(5), + resolve(3, DateTimeUnit.Day, rounded = false) == now.minusDays(3), + resolve(2, DateTimeUnit.Week, rounded = false) == now.minusWeeks(2), + resolve(2, DateTimeUnit.Month, rounded = false) == now.minusMonths(2), + resolve(1, DateTimeUnit.Year, rounded = false) == now.minusYears(1) + ) + }, + test("rounding to the start of a second/minute/hour zeroes the finer fields") { + val sec = resolve(0, DateTimeUnit.Second, rounded = true) + val min = resolve(0, DateTimeUnit.Minute, rounded = true) + val hour = resolve(0, DateTimeUnit.Hour, rounded = true) + assertTrue( + sec.getNano == 0, + min.getSecond == 0 && min.getNano == 0, + hour.getMinute == 0 && hour.getSecond == 0 && hour.getNano == 0 + ) + }, + test("rounding to the start of a day zeroes the time of day") { + val day = resolve(3, DateTimeUnit.Day, rounded = true) + assertTrue( + day.getHour == 0, + day.getMinute == 0, + day.getSecond == 0, + day.getNano == 0, + day.toLocalDate == now.minusDays(3).toLocalDate + ) + }, + test("rounding to the start of a week lands on Monday at midnight") { + // now is Wed 2026-07-15; one week earlier is Wed 2026-07-08, whose week starts Mon 2026-07-06. + val week = resolve(1, DateTimeUnit.Week, rounded = true) + assertTrue( + week.getDayOfWeek == java.time.DayOfWeek.MONDAY, + week.getHour == 0 && week.getMinute == 0 && week.getSecond == 0 && week.getNano == 0, + !week.isAfter(now.minusWeeks(1)) + ) + }, + test("rounding to the start of a month lands on the 1st at midnight") { + val month = resolve(2, DateTimeUnit.Month, rounded = true) + assertTrue( + month.getDayOfMonth == 1, + month.getMonthValue == 5, // 2026-07-15 minus 2 months -> May + month.getYear == 2026, + month.getHour == 0 && month.getMinute == 0 && month.getSecond == 0 && month.getNano == 0 + ) + }, + test("rounding to the start of a year lands on Jan 1st at midnight") { + val year = resolve(1, DateTimeUnit.Year, rounded = true) + assertTrue( + year.getDayOfYear == 1, + year.getMonthValue == 1, + year.getYear == 2025, + year.getHour == 0 && year.getMinute == 0 && year.getSecond == 0 && year.getNano == 0 + ) + }, + test("a rounded result is never later than the unrounded one") { + val violations = allUnits.filter { unit => + resolve(1, unit, rounded = true).isAfter(resolve(1, unit, rounded = false)) + } + assertTrue(violations.isEmpty) ?? s"rounding moved these units forward in time: $violations" + }, + test("value = 0 with rounding gives the start of the current unit") { + val month = resolve(0, DateTimeUnit.Month, rounded = true) + assertTrue(month.getYear == 2026, month.getMonthValue == 7, month.getDayOfMonth == 1) + }, + test("a year subtracted from Feb 29 lands on a valid date") { + val leap = ZonedDateTime.of(2024, 2, 29, 10, 0, 0, 0, ZoneId.of("UTC")) + val got = resolveRelativeDateTime( + RelativeDateTime(value = 1, unit = DateTimeUnit.Year, isRoundedToUnitStart = false), + leap + ) + assertTrue(got.getYear == 2023, got.getMonthValue == 2, got.getDayOfMonth == 28) + }, + test("a month subtracted from the 31st clamps to the shorter month") { + val endOfMonth = ZonedDateTime.of(2026, 3, 31, 10, 0, 0, 0, ZoneId.of("UTC")) + val got = resolveRelativeDateTime( + RelativeDateTime(value = 1, unit = DateTimeUnit.Month, isRoundedToUnitStart = false), + endOfMonth + ) + assertTrue(got.getMonthValue == 2, got.getDayOfMonth == 28) + }, + test("rounding across a DST transition keeps midnight wall-clock time") { + // Europe/Berlin springs forward on 2026-03-29; rounding must not yield 01:00 or 23:00. + val berlin = ZonedDateTime.of(2026, 4, 10, 15, 30, 0, 0, ZoneId.of("Europe/Berlin")) + val got = resolveRelativeDateTime( + RelativeDateTime(value = 2, unit = DateTimeUnit.Week, isRoundedToUnitStart = true), + berlin + ) + assertTrue(got.getHour == 0, got.getMinute == 0, got.getDayOfWeek == java.time.DayOfWeek.MONDAY) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/latestNLiveCheckMain.scala b/server/src/test/scala/consumer/session_runner/latestNLiveCheckMain.scala new file mode 100644 index 000000000..397ece2b5 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/latestNLiveCheckMain.scala @@ -0,0 +1,134 @@ +package consumer.session_runner + +import org.apache.pulsar.client.admin.PulsarAdmin +import org.apache.pulsar.client.api.{PulsarClient, MessageId as PulsarMessageId} + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} +import scala.util.Try + +/** MANUAL live-broker verification of "latest n" under a CONCURRENT PRODUCER - the moving-anchor + * defect that frozen-log tests structurally cannot exercise. + * + * Deliberately a `main` and NOT a ZIO spec: `sbt test` must stay broker-free (CI runs the server + * suite without Pulsar). Run it by hand against the e2e stack (dekaf-e2e-pulsar on + * localhost:6650 / localhost:18080): + * + * sbt "Test/runMain consumer.session_runner.latestNLiveCheckMain" # paced producer + * sbt "Test/runMain consumer.session_runner.latestNLiveCheckMain outrun" # producer at full rate + * + * It creates a THROWAWAY topic (deleted afterwards), seeds an unbatched backlog, runs the REAL + * `entryFromLatest` walk while a producer keeps publishing, then reads from the resolved cut and + * checks, against the topic as measured AFTER the walk: + * + * - the delivered set is a CONTIGUOUS SUFFIX of the final log (no gaps, no double-delivery); + * - it contains the final last n; + * - n <= delivered <= n + (messages published while the walk ran); + * - it is NOT the whole backlog - the failure mode this walk used to have. + * + * In `outrun` mode the log may genuinely move faster than the walk for its whole bound; the only + * acceptable outcome then is `StartFromUnresolvableException` - a session refusing loudly - never + * a "successful" session showing the wrong set. + */ +object latestNLiveCheckMain: + private val n = 50L + private val seed = 400 + + def main(args: Array[String]): Unit = + val outrun = args.contains("outrun") + val adminUrl = sys.env.getOrElse("DEKAF_LIVE_CHECK_ADMIN_URL", "http://localhost:18080") + val brokerUrl = sys.env.getOrElse("DEKAF_LIVE_CHECK_BROKER_URL", "pulsar://localhost:6650") + val topicFqn = s"persistent://public/default/dekaf-latestn-live-${java.util.UUID.randomUUID().toString.take(8)}" + + val admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl).build() + val client = PulsarClient.builder().serviceUrl(brokerUrl).build() + + var failures = Vector.empty[String] + def check(ok: Boolean, what: => String): Unit = if !ok then failures :+= what + + try + admin.topics().createNonPartitionedTopic(topicFqn) + val producer = client.newProducer().topic(topicFqn).enableBatching(false).blockIfQueueFull(true).create() + (1 to seed).foreach(i => producer.send(s"seed-$i".getBytes("UTF-8"))) + + val producedDuringResolve = AtomicInteger(0) + val stop = AtomicBoolean(false) + val pump = new Thread( + (() => + var i = 0 + while !stop.get do + i += 1 + producer.send(s"live-$i".getBytes("UTF-8")) + producedDuringResolve.incrementAndGet() + if !outrun then Thread.sleep(8) + ): Runnable, + "latest-n-live-check-producer" + ) + pump.setDaemon(true) + pump.start() + + val lookups = AtomicInteger(0) + val startedAtMs = System.currentTimeMillis() + val walkOutcome = Try { + resolveLatestN( + n, + Vector(topicFqn), + topic => k => { lookups.incrementAndGet(); entryFromLatest(admin, topic)(k) }, + latestNEntryIsOlder + ) + } + val walkMs = System.currentTimeMillis() - startedAtMs + stop.set(true) + pump.join(10_000) + producer.flush() + producer.close() + + println(s"topic=$topicFqn mode=${if outrun then "outrun" else "paced"}") + println(s"walk: ${walkOutcome.fold(err => s"FAILED (${err.getClass.getSimpleName}: ${err.getMessage})", _ => "resolved")} " + + s"in ${walkMs}ms, ${lookups.get} lookups, producedDuringResolve=${producedDuringResolve.get}") + + walkOutcome match + case scala.util.Failure(err) => + // Refusing loudly is the CORRECT outcome when the log persistently outruns the + // walk; anything else that throws is a real failure. + check(err.isInstanceOf[StartFromUnresolvableException], s"unexpected walk failure: $err") + check(outrun, s"the walk refused under a paced producer - it should have kept up. $err") + case scala.util.Success(cut) => + def drainFrom(startAt: PulsarMessageId, inclusive: Boolean): Vector[String] = + val builder = client.newReader().topic(topicFqn).startMessageId(startAt) + val reader = (if inclusive then builder.startMessageIdInclusive() else builder).create() + val out = scala.collection.mutable.ArrayBuffer.empty[String] + while reader.hasMessageAvailable do + val message = reader.readNext(10, TimeUnit.SECONDS) + if message == null then throw new RuntimeException("reader timed out mid-drain") + out += new String(message.getData, "UTF-8") + reader.close() + out.toVector + + val all = drainFrom(PulsarMessageId.earliest, inclusive = false) + val delivered = cut(topicFqn) match + case LatestNSeek.Nothing => Vector.empty[String] + case LatestNSeek.Everything => all + case LatestNSeek.FromEntry(entryId, discard) => drainFrom(entryId, inclusive = true).drop(discard.toInt) + + val lastN = all.takeRight(n.toInt) + println(s"final log=${all.size} messages; delivered=${delivered.size} " + + s"[${delivered.headOption.getOrElse("-")} .. ${delivered.lastOption.getOrElse("-")}]") + + check(delivered.size >= n, s"delivered FEWER than n: ${delivered.size} < $n (cut=${cut(topicFqn)})") + check( + delivered.size <= n + producedDuringResolve.get, + s"delivered MORE than n + concurrent production: ${delivered.size} > $n + ${producedDuringResolve.get}" + ) + check(delivered == all.takeRight(delivered.size), "delivered set is NOT a contiguous suffix of the final log") + check(lastN.forall(delivered.contains), "delivered set is missing part of the final last n") + check(delivered.size < all.size, s"WHOLE BACKLOG delivered as 'latest $n' (${delivered.size} of ${all.size})") + finally + Try(client.close()) + Try(admin.topics().delete(topicFqn, true)) + Try(admin.close()) + + if failures.nonEmpty then + System.err.println(failures.mkString("LIVE CHECK FAILED:\n - ", "\n - ", "")) + sys.exit(1) + else println("LIVE CHECK PASSED") diff --git a/server/src/test/scala/consumer/session_runner/listenerGateAndBudgetTest.scala b/server/src/test/scala/consumer/session_runner/listenerGateAndBudgetTest.scala new file mode 100644 index 000000000..5f8d8d552 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/listenerGateAndBudgetTest.scala @@ -0,0 +1,302 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Modifier, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.jdk.CollectionConverters.* + +/** THE TWO THINGS THAT DECIDE WHETHER A DELIVERED MESSAGE IS KEPT, AND WHAT EACH OF THEM COSTS WHEN + * IT IS WRONG. + * + * `ConsumerListener.decide` is entered from one Pulsar listener thread per physical topic, for + * every message the broker hands over. It reads two pieces of shared state: + * + * - THE PAUSE GATE, written from gRPC threads. A stale read here delivers a message into a + * session the user has paused. + * - THE START-FROM BUDGET, which is the user's "skip the first n". Claiming one costs a message + * - the message is acknowledged into nothing and nobody ever sees it - so a claim that is not + * matched by a real acknowledgment means Pulsar redelivers a message whose budget is already + * spent, and the session shows a message the user asked to skip while reporting that exactly n + * were skipped. + * + * Everything here drives the real `received`/`decide`/`pause` paths with proxy consumers and + * hand-built messages. Only the broker is replaced. + */ +object listenerGateAndBudgetTest extends ZIOSpecDefault: + + private val p0 = "persistent://public/default/gate-partition-0" + + private def message(label: String, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(1_000L + entryId) + md.setPartitionKey(label) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"label":"$label"}""".getBytes("UTF-8")), Schema.BYTES, p0) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** A consumer that can lose its connection and can refuse an acknowledgment, both of which real + * brokers do. `acknowledged` records only the acknowledgments that actually SUCCEEDED - which + * is the whole question here. + */ + private final class RecordingConsumer(onPause: () => Unit = () => ()): + val connected = AtomicBoolean(true) + val acknowledgeFails = AtomicBoolean(false) + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + val paused = AtomicBoolean(false) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => p0 + case "getConsumerName" => "cs-gate-0" + case "isConnected" => java.lang.Boolean.valueOf(connected.get) + case "pause" => + paused.set(true) + onPause() + null + case "resume" => null + case "acknowledgeAsync" => + if acknowledgeFails.get then CompletableFuture.failedFuture(new RuntimeException("broker refused the acknowledgment")) + else + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(p0.hashCode) + case "toString" => "proxy-consumer(gate)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def openListener(): (ConsumerListener, ConcurrentLinkedQueue[String]) = + val delivered = ConcurrentLinkedQueue[String]() + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => { delivered.add(msg.getKey); () })) + listener.startAcceptingNewMessages() + (listener, delivered) + + private def targetRunner(consumerListener: ConsumerListener, consumer: Consumer[Array[Byte]]): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p0))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p0), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = Map(p0 -> consumer), + pauseArbiters = (Map(p0 -> consumer)).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private val budgetSuite = suite("a skip budget is spent by an ACKNOWLEDGED drop, and by nothing else")( + test("a drop the consumer could not acknowledge is handed back, and the budget survives it") { + // THE defect. `decide` claimed the budget and the acknowledgment helper then did + // NOTHING when the consumer was disconnected. Pulsar redelivered the message, the + // (already spent) budget let it through, and a session asked to skip one message showed + // it - while reporting that the skip had completed. + val (listener, delivered) = openListener() + listener.startFromDiscard = StartFromDiscard.shared(1) + val consumer = RecordingConsumer() + consumer.connected.set(false) + + listener.received(consumer.consumer, message("m1", 1L)) + val afterDisconnect = listener.startFromDiscard.remaining + + // The connection comes back and the broker redelivers what was never acknowledged. + consumer.connected.set(true) + listener.received(consumer.consumer, message("m1", 1L)) + + assertTrue( + afterDisconnect == 1L, + consumer.handedBack.asScala.toVector == Vector("m1"), + delivered.asScala.toVector.isEmpty, + consumer.acknowledged.asScala.toVector == Vector("m1"), + listener.startFromDiscard.remaining == 0L + ) ?? (s"budget after the disconnected delivery=$afterDisconnect handedBack=${consumer.handedBack.asScala.toVector} " + + s"delivered=${delivered.asScala.toVector} acknowledged=${consumer.acknowledged.asScala.toVector}") + }, + test("a drop whose ack FAILS stays decided: the unit stays spent on THAT message, its redelivery is paperwork") { + // The old contract REFUNDED the unit, which kept the count right and the set wrong: the + // refunded unit was spent on the NEXT message, and the redelivered original - the one + // the user asked to skip - was shown. Now the decision stands: budget spent once, on + // m1; m2 flows through untouched by that unit; m1's redelivery is acknowledged and + // shown to nobody. + val (listener, delivered) = openListener() + listener.startFromDiscard = StartFromDiscard.shared(1) + val consumer = RecordingConsumer() + consumer.acknowledgeFails.set(true) + + listener.received(consumer.consumer, message("m1", 1L)) + val afterFailedAck = listener.startFromDiscard.remaining + val retriesArmed = listener.awaitingAckRetryCount + + consumer.acknowledgeFails.set(false) + // The next NEW message arrives before m1's redelivery - with a refund, this one would + // have consumed the returned unit and m1 would later be shown. + listener.received(consumer.consumer, message("m2", 2L)) + listener.received(consumer.consumer, message("m1", 1L)) + + assertTrue( + afterFailedAck == 0L, + retriesArmed == 1, + delivered.asScala.toVector == Vector("m2"), + consumer.acknowledged.asScala.toVector == Vector("m2", "m1"), + listener.awaitingAckRetryCount == 0 + ) ?? (s"budget after failed ack=$afterFailedAck retriesArmed=$retriesArmed delivered=${delivered.asScala.toVector} " + + s"acknowledged=${consumer.acknowledged.asScala.toVector} retriesLeft=${listener.awaitingAckRetryCount}") + }, + test("a DELIVERED message whose ack fails is not shown twice when the broker redelivers it") { + // Same mechanism, other outcome: the delivery happened, only the paperwork failed. The + // redelivery must be acknowledged and NOT rendered again. + val (listener, delivered) = openListener() + val consumer = RecordingConsumer() + consumer.acknowledgeFails.set(true) + + listener.received(consumer.consumer, message("m1", 1L)) + consumer.acknowledgeFails.set(false) + listener.received(consumer.consumer, message("m1", 1L)) + + assertTrue( + delivered.asScala.toVector == Vector("m1"), + consumer.acknowledged.asScala.toVector == Vector("m1"), + listener.awaitingAckRetryCount == 0 + ) ?? (s"delivered=${delivered.asScala.toVector} acknowledged=${consumer.acknowledged.asScala.toVector}") + }, + test("exactly n UNIQUE messages are skipped when every acknowledgment lands") { + // The control: nothing above may cost the ordinary path its exactness. + val (listener, delivered) = openListener() + listener.startFromDiscard = StartFromDiscard.shared(3) + val consumer = RecordingConsumer() + + (1 to 5).foreach(i => listener.received(consumer.consumer, message(s"m$i", i.toLong))) + + assertTrue( + delivered.asScala.toVector == Vector("m4", "m5"), + consumer.acknowledged.asScala.toVector == Vector("m1", "m2", "m3", "m4", "m5"), + consumer.handedBack.asScala.toVector.isEmpty, + listener.startFromDiscard.remaining == 0L + ) ?? s"delivered=${delivered.asScala.toVector} acknowledged=${consumer.acknowledged.asScala.toVector}" + }, + test("a DELIVERED message on a disconnected consumer is handed back rather than shown unacknowledged") { + // Nothing may be decided about a message the consumer cannot answer for. Handing it + // back is the only outcome that neither loses it nor double-counts it. + val (listener, delivered) = openListener() + val consumer = RecordingConsumer() + consumer.connected.set(false) + + listener.received(consumer.consumer, message("m1", 1L)) + + assertTrue(delivered.asScala.toVector.isEmpty, consumer.handedBack.asScala.toVector == Vector("m1")) + } + ) + + private val gateSuite = suite("the pause gate")( + test("PAUSE CLOSES THE GATE BEFORE THE CONSUMERS STOP DELIVERING") { + // Order matters and it was the wrong way round: the consumers were paused first and the + // gate closed afterwards, so every callback the client had already buffered was + // delivered into a session the user had just paused. Closing the gate first makes the + // window empty by construction. + val (listener, _) = openListener() + val gateClosedWhenPaused = AtomicBoolean(false) + val consumer = RecordingConsumer(onPause = () => gateClosedWhenPaused.set(listener.decide(p0, canAcknowledge = true) == ConsumerListener.Action.Reject)) + val runner = targetRunner(listener, consumer.consumer) + + runner.pause() + + assertTrue( + consumer.paused.get, + gateClosedWhenPaused.get, + listener.decide(p0, canAcknowledge = true) == ConsumerListener.Action.Reject + ) ?? "the consumer was paused while the listener was still accepting messages" + }, + test("a paused listener rejects rather than consuming the skip budget") { + // A rejected message is coming back, so counting it as skipped would skip it twice. + val (listener, _) = openListener() + listener.startFromDiscard = StartFromDiscard.shared(2) + listener.stopAcceptingNewMessages() + + val action = listener.decide(p0, canAcknowledge = true) + + assertTrue(action == ConsumerListener.Action.Reject, listener.startFromDiscard.remaining == 2L) + }, + test("THE GATE CARRIES A HAPPENS-BEFORE - it is read by listener threads and written by gRPC threads") { + // A plain `var Boolean` has no memory-ordering guarantee at all, so a Pulsar listener + // thread was entitled to go on seeing "accepting" indefinitely after an RPC thread had + // paused the session. A data race cannot be observed reliably from a test, so the + // MECHANISM is pinned instead: reverting the field to a plain var fails this. + val field = classOf[ConsumerListener].getDeclaredFields.find(_.getName.toLowerCase.contains("acceptingnewmessages")) + val isSafe = field.exists(f => + classOf[java.util.concurrent.atomic.AtomicBoolean].isAssignableFrom(f.getType) || Modifier.isVolatile(f.getModifiers) + ) + assertTrue(field.isDefined, isSafe) ?? + s"the pause gate is ${field.map(f => s"${f.getType.getSimpleName} (volatile=${Modifier.isVolatile(f.getModifiers)})")}" + } + ) + + private val ackLoggingSuite = suite("a merge-path acknowledgment failure is LOGGED, as the scope note promises")( + test("a DELIVERED message whose acknowledgment fails leaves a warning naming the consumer") { + // `acknowledgeDrop`'s scope note says a failed merge-path acknowledgment "is logged" - + // but `acknowledge` (the Deliver/merge-Drop path) discarded the future outright, so an + // unacknowledged, soon-to-be-redelivered message left no trace at all. + val appender = new ch.qos.logback.core.read.ListAppender[ch.qos.logback.classic.spi.ILoggingEvent]() + appender.start() + // SLF4J hands a SubstituteLogger to callers that arrive while the backend is still + // initializing; under parallel suite execution the first fetch can land in that window + // and the WARN under test is replayed to the REAL logger later - without this appender. + // Re-fetch until the logback binding is in place (the same guard libraryScanTest uses). + var slf4jLogger = org.slf4j.LoggerFactory.getLogger(classOf[ConsumerListener].getName) + var attempts = 0 + while !slf4jLogger.isInstanceOf[ch.qos.logback.classic.Logger] && attempts < 500 do + Thread.sleep(2) + slf4jLogger = org.slf4j.LoggerFactory.getLogger(classOf[ConsumerListener].getName) + attempts += 1 + val logbackLogger = slf4jLogger.asInstanceOf[ch.qos.logback.classic.Logger] + logbackLogger.addAppender(appender) + try + val (listener, delivered) = openListener() + val consumer = RecordingConsumer() + consumer.acknowledgeFails.set(true) + + listener.received(consumer.consumer, message("m1", 1L)) + + val warned = appender.list.asScala.toVector + .filter(_.getLevel == ch.qos.logback.classic.Level.WARN) + .map(_.getFormattedMessage) + assertTrue( + delivered.asScala.toVector == Vector("m1"), + // The wording changed with the decision-stands contract; what the note promises + // is a WARN that names the consumer and says the ack failed. + warned.exists(m => m.contains("decision stands") && m.contains("acknowledgment failed") && m.contains("cs-gate-0")) + ) ?? s"delivered=${delivered.asScala.toVector} warned=$warned" + finally logbackLogger.detachAppender(appender) + } + ) + + def spec = suite(this.getClass.toString)(budgetSuite, gateSuite, ackLoggingSuite) diff --git a/server/src/test/scala/consumer/session_runner/liveDeliveryOrderSwitchTest.scala b/server/src/test/scala/consumer/session_runner/liveDeliveryOrderSwitchTest.scala new file mode 100644 index 000000000..cb646b3ba --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/liveDeliveryOrderSwitchTest.scala @@ -0,0 +1,353 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, SECONDS} +import scala.jdk.CollectionConverters.* + +/** THE ONE THAT MATTERS: a Guaranteed session holding messages behind a silent stream, switched to + * Best effort WHILE RUNNING, driven through the real listener and the real ordering layer. + * + * This is the whole promise of the operation, and each half of it is a separate way to lose data: + * + * - the held set is RELEASED, not dropped - those messages were received and never + * acknowledged, and they exist nowhere but in the merge; + * - each of them is delivered EXACTLY ONCE, in the new order - a release that also let the + * barrier deliver, or that re-offered anything, would double them; + * - NOTHING IS RE-READ FROM THE BROKER - no seek, no redelivery request, no negative + * acknowledgment. That is precisely what recreating the session could not promise: it would + * re-resolve the start-from against a log that has moved and hand back a different set. + * + * Only the broker is replaced, by proxy consumers and hand-built `MessageImpl`s; + * `ConsumerListener.received`, the guaranteed delivery pump and the ordering layer are the + * production ones. The merge's clock is injected so "the session has been stalled for a while" + * is a fact the test states rather than waits for. + */ +object liveDeliveryOrderSwitchTest extends ZIOSpecDefault: + + private val consumerName = "cs-live-order-0" + private val graceMs = 500L + + private def p(i: Int): String = s"persistent://public/default/live-order-partition-$i" + private def sid(i: Int): String = startFromStreamId(consumerName, p(i)) + + private final class MonoClock: + var monoMs: Long = 0L + def advance(ms: Long): Unit = monoMs += ms + + /** A consumer that records what the session did to it. `reReads` is the point: anything that + * would make the broker hand messages over again - a seek, a redelivery request, a negative + * acknowledgment - is recorded, and the switch must produce none of them. */ + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val reReads = ConcurrentLinkedQueue[String]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + // A FAR-FUTURE recorded end for the guaranteed resume's boundary + // re-capture: these tests pin the mid-replay stall and its relax, so no + // stream may read as finished and no hand-built message as past-end. + case "getLastMessageIds" => + java.util.List.of[org.apache.pulsar.client.api.MessageId](new MessageIdImpl(1L, 1_000_000L, -1)) + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + reReads.add(s"negativeAcknowledge/${args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey}") + null + case name if name.startsWith("seek") || name.startsWith("redeliver") => + reReads.add(name) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** A listener wired to a GUARANTEED ordering layer over `streamCount` streams, with the + * merge's clock in the test's hands. The `StartFromOrdering` constructor is package-visible + * for exactly this: the real plumbing, an injected clock. */ + private final class Fixture(streamCount: Int): + val clock = MonoClock() + val delivered = ConcurrentLinkedQueue[String]() + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector.tabulate(streamCount)(sid), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.monoMs, + policy = OrderingPolicy.GuaranteedOnly, + graceMs = graceMs + ) + val listener: ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => + delivered.add(msg.getKey) + () + )) + l.startAcceptingNewMessages() + l.startFromOrdering = new StartFromOrdering[HeldMessage](Some(merge), Map.empty) + l + val consumers: Vector[RecordingConsumer] = Vector.tabulate(streamCount)(i => RecordingConsumer(p(i))) + private var nextEntryId = Map.empty[Int, Long].withDefaultValue(0L) + + def deliver(partition: Int, key: String, publishTime: Long): Unit = + val entryId = nextEntryId(partition) + nextEntryId = nextEntryId.updated(partition, entryId + 1) + listener.received(consumers(partition).consumer, message(p(partition), key, publishTime, entryId)) + + def deliveredKeys: Vector[String] = delivered.asScala.toVector + def acknowledgedKeys: Vector[String] = consumers.flatMap(_.acknowledged.asScala.toVector) + def reReads: Vector[String] = consumers.flatMap(_.reReads.asScala.toVector) + + /** The runner `behindTheRpc` installed, for the properties that live on it rather than on + * the RPC's answer. */ + var installedRunner: Option[ConsumerSessionRunner] = None + + /** The same fixture behind the real RPC, so the whole path - `SetDeliveryOrder`, the + * lifecycle lock, the runner, the listener, the merge - can be driven at once. */ + def behindTheRpc(sessionName: String): consumer.ConsumerServiceImpl = + val consumersByTopic = Vector.tabulate(streamCount)(i => p(i) -> consumers(i).consumer).toMap + val target = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumersByTopic.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumersByTopic.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumersByTopic, + pauseArbiters = consumersByTopic.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = java.util.concurrent.atomic.AtomicLong(0)) + ) + val runner = ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty), + messageDeliveryOrder = MessageDeliveryOrder.Guaranteed + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + val sessions = new java.util.concurrent.ConcurrentHashMap[consumer.ConsumerSessionName, ConsumerSessionRunner]() + sessions.put(sessionName, runner) + installedRunner = Some(runner) + consumer.ConsumerServiceImpl(sessions) + + def spec = suite(this.getClass.toString)( + test("A LIVE SWITCH RELEASES THE HELD SET EXACTLY ONCE, IN BEST-EFFORT ORDER, WITHOUT TOUCHING THE BROKER") { + // Three partitions, the third of which never publishes anything - the ordinary shape of + // a Guaranteed stall. Arrival order is deliberately not publish-time order, so a + // release that merely flushed the queues would show up immediately. + val f = Fixture(streamCount = 3) + f.deliver(0, "a-100", 100L) + f.deliver(0, "a-300", 300L) + f.deliver(1, "b-200", 200L) + f.deliver(1, "b-400", 400L) + f.deliver(0, "a-500", 500L) + + val deliveredWhileStalled = f.deliveredKeys + val acknowledgedWhileStalled = f.acknowledgedKeys + val heldBehindTheSilentStream = f.merge.heldCount + + // The session has been stalled long enough for the user to notice and click. + f.clock.advance(graceMs + 1) + f.listener.relaxDeliveryOrderToBestEffort() + + // And the barrier really is gone: pumping it again must produce nothing. + f.listener.pumpGuaranteedDelivery() + + assertTrue( + deliveredWhileStalled.isEmpty, // Guaranteed held everything behind partition 2 + acknowledgedWhileStalled.isEmpty, // and acknowledged nothing, so nothing was consumable + heldBehindTheSilentStream == 5, + f.deliveredKeys == Vector("a-100", "b-200", "a-300", "b-400", "a-500"), + f.deliveredKeys.distinct == f.deliveredKeys, // exactly once + f.acknowledgedKeys.sorted == f.deliveredKeys.sorted, // each released message was acknowledged + f.reReads.isEmpty, // no seek, no redelivery request, no nack: nothing came from the broker again + f.merge.heldCount == 0, + !f.listener.startFromOrdering.isGuaranteedOrdering, + f.listener.startFromOrdering.isContinuousOrdering // still ordered, just no longer no-escape + ) ?? (s"deliveredWhileStalled=$deliveredWhileStalled held=$heldBehindTheSilentStream " + + s"delivered=${f.deliveredKeys} acked=${f.acknowledgedKeys} reReads=${f.reReads}") + }, + test("THROUGH THE REAL RPC: one SetDeliveryOrder call releases what the barrier was holding") { + // The whole feature end to end - `SetDeliveryOrder` -> lifecycle lock -> runner -> + // listener -> merge - on a session stalled behind a silent partition. No recreate, no + // re-read, no duplicate, and the session's LIVE order really moved. + val f = Fixture(streamCount = 2) + val service = f.behindTheRpc("cs-live-order-rpc") + f.deliver(0, "m-100", 100L) + f.deliver(0, "m-200", 200L) + val deliveredWhileStalled = f.deliveredKeys + f.clock.advance(graceMs + 1) + + val response = Await.result( + service.setDeliveryOrder(consumerPb.SetDeliveryOrderRequest( + consumerName = "cs-live-order-rpc", + messageDeliveryOrder = consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + )), + Duration(30, SECONDS) + ) + + assertTrue( + deliveredWhileStalled.isEmpty, + response.getStatus.code == com.google.rpc.code.Code.OK.value, + f.deliveredKeys == Vector("m-100", "m-200"), + f.acknowledgedKeys.sorted == Vector("m-100", "m-200"), + f.reReads.isEmpty, + f.merge.heldCount == 0, + !f.listener.startFromOrdering.isGuaranteedOrdering + ) ?? (s"status=${response.getStatus} delivered=${f.deliveredKeys} acked=${f.acknowledgedKeys} " + + s"reReads=${f.reReads} held=${f.merge.heldCount}") + }, + test("THE SWEEP CADENCE FOLLOWS THE POLICY, and a live relax re-arms it at the faster one") { + // Guaranteed has no residence for a sweep tick to expire and never gives a stream up, + // so it does not need the best-effort grace cadence - on a wide idle session that was + // four whole-set scans a second producing nothing. But the relax is exactly what turns + // the residence bound back on, and a session left on the slower cadence would then sit + // up to a second past its grace instead of a quarter of one. So the switch re-arms. + val f = Fixture(streamCount = 2) + val service = f.behindTheRpc("cs-live-order-cadence") + val runner = f.installedRunner.get + val observer = new io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + override def onNext(value: consumerPb.ResumeResponse): Unit = () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + + runner.resume(observer, isDebug = false) // arms the continuous sweep + val armedForGuaranteed = runner.armedSweepPeriodMs + + val response = Await.result( + service.setDeliveryOrder(consumerPb.SetDeliveryOrderRequest( + consumerName = "cs-live-order-cadence", + messageDeliveryOrder = consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + )), + Duration(30, SECONDS) + ) + val armedAfterRelax = runner.armedSweepPeriodMs + scala.util.Try(runner.stop()) + + assertTrue( + response.getStatus.code == com.google.rpc.code.Code.OK.value, + armedForGuaranteed.contains(guaranteedSweepPeriodMs), + armedForGuaranteed.contains(continuousSweepPeriodMs(guaranteed = true)), + armedAfterRelax.contains(mergeTopicsSweepPeriodMs), + runner.armedSweepPeriodMs.isEmpty // and stopping disarms it + ) ?? (s"armedForGuaranteed=$armedForGuaranteed armedAfterRelax=$armedAfterRelax " + + s"guaranteed=$guaranteedSweepPeriodMs bestEffort=$mergeTopicsSweepPeriodMs status=${response.getStatus}") + }, + test("delivery CONTINUES under the new rules for messages that arrive after the switch") { + val f = Fixture(streamCount = 2) + f.deliver(0, "a-100", 100L) // partition 1 is silent: the barrier holds this + f.clock.advance(graceMs + 1) + f.listener.relaxDeliveryOrderToBestEffort() + val releasedByTheSwitch = f.deliveredKeys + + // Arrivals after the switch are ordinary best-effort traffic: held for their own + // residence while the peer is quiet, then delivered by the sweep - never held forever. + f.deliver(0, "a-700", 700L) + val heldWhileFresh = f.deliveredKeys + f.clock.advance(graceMs + 1) + f.listener.sweepStartFromStall() + + assertTrue( + releasedByTheSwitch == Vector("a-100"), + heldWhileFresh == Vector("a-100"), + f.deliveredKeys == Vector("a-100", "a-700"), + f.acknowledgedKeys.sorted == Vector("a-100", "a-700"), + f.reReads.isEmpty, + f.merge.heldCount == 0 + ) ?? s"released=$releasedByTheSwitch delivered=${f.deliveredKeys} acked=${f.acknowledgedKeys}" + }, + test("switching twice releases nothing the second time - the operation is idempotent") { + val f = Fixture(streamCount = 2) + f.deliver(0, "a-100", 100L) + f.deliver(0, "a-200", 200L) + f.clock.advance(graceMs + 1) + f.listener.relaxDeliveryOrderToBestEffort() + val afterFirst = f.deliveredKeys + f.listener.relaxDeliveryOrderToBestEffort() + + assertTrue( + afterFirst == Vector("a-100", "a-200"), + f.deliveredKeys == afterFirst, + f.acknowledgedKeys.sorted == Vector("a-100", "a-200"), // one acknowledgment each, still + f.reReads.isEmpty + ) ?? s"afterFirst=$afterFirst delivered=${f.deliveredKeys} acked=${f.acknowledgedKeys}" + }, + test("a switch while the INTAKE GATE is shut still neither loses nor doubles the held set") { + // A pause closes the intake gate; the switch is a decision about what the session + // ALREADY holds, so it releases that set down the ordinary delivery path (in a live + // session, into the rate limiter, whose drain a pause has stopped). What must hold + // either way: nothing released twice, nothing lost, nothing re-read - and the gate + // itself is untouched, so a message arriving now is still handed straight back. + val f = Fixture(streamCount = 2) + f.deliver(0, "a-100", 100L) + f.deliver(0, "a-200", 200L) + f.listener.stopAcceptingNewMessages() + f.clock.advance(graceMs + 1) + f.listener.relaxDeliveryOrderToBestEffort() + + val afterSwitch = f.deliveredKeys + val reReadsAfterSwitch = f.reReads + f.deliver(1, "b-300", 300L) // the closed gate must refuse this + + assertTrue( + afterSwitch == Vector("a-100", "a-200"), + afterSwitch.distinct == afterSwitch, + f.acknowledgedKeys.sorted == Vector("a-100", "a-200"), + reReadsAfterSwitch.isEmpty, + f.merge.heldCount == 0, + f.deliveredKeys == afterSwitch, // the refused arrival reached nobody + f.reReads == Vector("negativeAcknowledge/b-300") // it was handed back, as a pause promises + ) ?? s"delivered=${f.deliveredKeys} acked=${f.acknowledgedKeys} reReads=${f.reReads}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/markerEntriesTest.scala b/server/src/test/scala/consumer/session_runner/markerEntriesTest.scala new file mode 100644 index 000000000..5f22627d2 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/markerEntriesTest.scala @@ -0,0 +1,182 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.{MessageId, Schema} +import org.apache.pulsar.client.impl.{BatchMessageIdImpl, MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.{MarkerType, MessageMetadata} +import zio.test.* + +import java.nio.ByteBuffer + +/** SERVER-ONLY MARKER ENTRIES - the third shape in which "one entry" is not "one message", after + * batching (many messages in one entry) and chunking (one message across many entries). + * + * A transaction commit or abort, and a replicated-subscription snapshot, are written into the + * topic's OWN managed ledger as ordinary entries. Established against Pulsar 3.2.1 by reading both + * sides rather than by guessing: + * + * - `TopicTransactionBuffer.commitTxn`/`abortTxn` build the record with + * `Markers.newTxnCommitMarker`/`newTxnAbortMarker` - which set `MessageMetadata.marker_type` - + * and append it with `ManagedLedger.asyncAddEntry`, so `getNumberOfEntries` counts it; + * - `PersistentTopicsBase.internalExamineMessageAsync` applies NO marker filtering: the entry + * goes straight to `generateResponseWithEntry`, which emits `X-Pulsar-marker-type` whenever the + * metadata has one, and the admin client's `TopicsImpl.getMessagesFromHttpResponse` parses that + * header back onto the `MessageMetadata` it hands to `MessageImpl` - the very object + * `getMessageBuilder` returns. So `examineMessage` DOES return markers and they ARE + * identifiable; + * - `AbstractBaseDispatcher.filterEntriesForConsumer` nulls and releases every entry for which + * `Markers.isServerOnlyMarker` holds, so no consumer is ever handed one. + * + * Counted by the broker, delivered to nobody: an entry-addressed count that treats a marker as a + * message holds one message too few for every marker it crosses. Hence SKIP rather than refuse - + * zero is the exact delivered count for a marker, so the walk stays correct by crossing it, and + * refusing would take a working mode away from every transactional topic. + * + * NOT reachable end to end in the e2e harness: transactions are disabled on the standalone broker + * the suite runs against, so no marker entry can be produced there. That is why the message-level + * classification is pinned here with hand-built metadata and the walk arithmetic with a plain + * lambda. + */ +object markerEntriesTest extends ZIOSpecDefault: + + private val topic = "persistent://public/default/markers" + + private def message(id: MessageId, configure: MessageMetadata => Unit): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + // publish_time is mandatory on MessageMetadata - reading it when unset throws. + md.setPublishTime(1_700_000_000_000L) + configure(md) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap("{}".getBytes("UTF-8")), Schema.BYTES, topic) + msg.setMessageId(id) + msg + + private def plainMessage: MessageImpl[Array[Byte]] = message(new MessageIdImpl(1L, 2L, -1), _ => ()) + + private def markerMessage(markerType: MarkerType): MessageImpl[Array[Byte]] = + message(new MessageIdImpl(1L, 2L, -1), _.setMarkerType(markerType.getValue)) + + private val classificationSuite = suite("recognising a marker entry in an examineMessage answer")( + test("a transaction commit marker is recognised and counts as ZERO messages") { + // The whole point: the entry exists, the broker counts it, and it holds nothing a + // consumer will ever see. + val marker = markerMessage(MarkerType.TXN_COMMIT) + assertTrue( + isServerOnlyMarkerEntry(marker), + logEntryOf(topic, marker).messagesInEntry == 0 + ) + }, + test("EVERY marker type Pulsar defines is recognised, not just the transactional ones") { + // Geo-replication writes REPLICATED_SUBSCRIPTION_* markers with no transaction in + // sight, so keying off the transaction ids instead of `marker_type` would miss them. + // Pulsar's own dispatcher filters on `hasMarkerType` alone, and so does this. + val unrecognised = MarkerType.values.toList.filterNot(t => isServerOnlyMarkerEntry(markerMessage(t))) + assertTrue(unrecognised.isEmpty) ?? s"marker types read as ordinary messages: ${unrecognised.mkString(", ")}" + }, + test("an ordinary unbatched message is not a marker and still counts as one message") { + val plain = plainMessage + assertTrue(!isServerOnlyMarkerEntry(plain), logEntryOf(topic, plain).messagesInEntry == 1) + }, + test("a batched entry keeps its own batch size - the marker rule does not touch the batch path") { + val batched = message(new BatchMessageIdImpl(1L, 2L, 0, 0, 50, null), _ => ()) + assertTrue(!isServerOnlyMarkerEntry(batched), logEntryOf(topic, batched).messagesInEntry == 50) + }, + test("a marker keeps its entry id and publish time - only its message count is zero") { + // The walk still has to be able to seek to it and to order it against other topics. + val marker = markerMessage(MarkerType.TXN_ABORT) + val entry = logEntryOf(topic, marker) + assertTrue( + entry.entryId == new MessageIdImpl(1L, 2L, -1), + entry.publishTime == 1_700_000_000_000L + ) + }, + test("a CHUNK piece is still REFUSED, with the explanation that names the mode and the remedy") { + // The opposite decision to the marker one, and the reason both live in `logEntryOf`: a + // chunk piece makes entry counts meaningless in BOTH directions, so there is no correct + // number to substitute the way zero is correct for a marker. + val chunk = message(new MessageIdImpl(1L, 2L, -1), _.setNumChunksFromMsg(3)) + val refusal = scala.util.Try(logEntryOf(topic, chunk)).failed.toOption + assertTrue( + refusal.exists(_.isInstanceOf[StartFromUnresolvableException]), + refusal.exists(_.getMessage.contains(topic)), + refusal.exists(_.getMessage.contains("CHUNKED")), + refusal.exists(_.getMessage.contains("Skip first n messages")) + ) ?? s"a chunk piece was not refused: $refusal" + } + ) + + /** The `topic#k` labels carry the walk's entry order: `k` counts back from the end, so a larger + * ordinal is strictly OLDER (the production comparator is `MessageIdImpl.compareTo`). */ + private def olderByOrdinal(a: String, b: String): Boolean = a.split("#").last.toInt > b.split("#").last.toInt + + /** Resolve `n` over logs described from their END: element 0 is each topic's LAST entry, given as + * (publish time, messages in that entry). A ZERO message count is exactly what + * [[entryFromLatest]] reports for a server-only marker. */ + private def walk(n: Long, logs: (String, Seq[(Long, Int)])*): Map[String, LatestNSeek[String]] = + val byTopic = logs.toMap + val lookup = (topicFqn: String) => + (k: Long) => + val entries = byTopic.getOrElse(topicFqn, Seq.empty) + Option.when(k >= 1 && k <= entries.size) { + val (publishTime, messages) = entries(k.toInt - 1) + LogEntry(s"$topicFqn#$k", publishTime, messages) + } + resolveLatestN(n, byTopic.keys.toVector.sorted, lookup, olderByOrdinal) + + private val marker = 0 + private val a = "persistent://public/default/a" + private val b = "persistent://public/default/b" + + private val walkSuite = suite("resolveLatestN across marker entries")( + test("a marker between messages costs the walk a STEP and not a message") { + // From the end: m5, MARKER, m4, m3, m2, m1. The last 3 MESSAGES are m3, m4, m5, so the + // anchor is the 4th entry back. Counting the marker as one message (the `max 1` floor + // this replaced) stopped on the 3rd entry instead and the session held two. + val cut = walk(3, a -> Seq(9L -> 1, 8L -> marker, 7L -> 1, 6L -> 1, 5L -> 1, 4L -> 1)) + assertTrue(cut(a) == LatestNSeek.FromEntry(s"$a#4", 0L)) + }, + test("consecutive markers are crossed without ending the walk or moving the count") { + // A burst of aborts is an ordinary shape on a topic under a failing transactional + // producer; a zero-count entry must not look like exhaustion. + val cut = walk(2, a -> Seq(9L -> marker, 8L -> marker, 7L -> marker, 6L -> 1, 5L -> 1, 4L -> 1)) + assertTrue(cut(a) == LatestNSeek.FromEntry(s"$a#5", 0L)) + }, + test("markers do not disturb the overshoot discard inside a batched stopping entry") { + // From the end: a 3-message batch, a MARKER, another 3-message batch. The last 4 + // messages need the second batch, of which 2 are older than the cut and are dropped + // from that topic's head. + val cut = walk(4, a -> Seq(9L -> 3, 8L -> marker, 7L -> 3, 6L -> 3)) + assertTrue(cut(a) == LatestNSeek.FromEntry(s"$a#3", 2L)) + }, + test("a log holding ONLY markers anchors at its oldest entry with nothing to discard") { + // It has no messages to show, and that is the honest answer: seeking to the oldest + // marker delivers everything published after it, and the broker filters the markers + // themselves out of the dispatch. + val cut = walk(3, a -> Seq(9L -> marker, 8L -> marker)) + assertTrue(cut(a) == LatestNSeek.FromEntry(s"$a#2", 0L)) + }, + test("a topic whose inspected TAIL is a marker anchors there with a ZERO head-drop") { + // B contributes nothing to the cut, so it is anchored at the tail the walk inspected and + // that entry's own messages are dropped from its head. A marker has none - dropping one + // anyway would swallow the first REAL message published after it, which is precisely the + // live traffic anchoring at the inspected tail exists to keep. + val cut = walk( + 2, + a -> Seq(100L -> 1, 90L -> 1, 80L -> 1), + b -> Seq(50L -> marker, 40L -> 1) + ) + assertTrue( + cut(a) == LatestNSeek.FromEntry(s"$a#2", 0L), + cut(b) == LatestNSeek.FromEntry(s"$b#1", 0L) + ) + }, + test("a non-contributing tail that IS a message still drops its own messages") { + // The control for the case above: `max 0` must not have turned the head-drop off. + val cut = walk( + 2, + a -> Seq(100L -> 1, 90L -> 1, 80L -> 1), + b -> Seq(50L -> 4, 40L -> 1) + ) + assertTrue(cut(b) == LatestNSeek.FromEntry(s"$b#1", 4L)) + } + ) + + def spec = suite(this.getClass.toString)(classificationSuite, walkSuite) diff --git a/server/src/test/scala/consumer/session_runner/mergeDeliveryFailureTest.scala b/server/src/test/scala/consumer/session_runner/mergeDeliveryFailureTest.scala new file mode 100644 index 000000000..8821c9e1e --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/mergeDeliveryFailureTest.scala @@ -0,0 +1,159 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.jdk.CollectionConverters.* + +/** WHAT HAPPENS WHEN THE CLIENT STREAM THROWS WHILE A RESOLVED BATCH IS BEING HANDED OUT. + * + * The global skip merge answers one `offer` with a BATCH of resolved messages - often several, and + * belonging to different topics than the one just offered. Each pair in that batch was already + * DEQUEUED from the merge, so it exists nowhere else. `StreamObserver.onNext` throws the instant the + * client's call has been cancelled, and that throw used to escape the batch loop: every pair after + * the failing one was neither delivered, acknowledged, nor handed back - and a NonDurable + * subscription has no ackTimeout, so the broker never redelivered them while the runner lived. + * + * The real `received` path is driven here with proxy consumers and a handler that throws exactly as + * a cancelled gRPC call does. Only the broker is replaced. + */ +object mergeDeliveryFailureTest extends ZIOSpecDefault: + + private val consumerName = "cs-merge-fail-0" + private def p(i: Int): String = s"persistent://public/default/merge-fail-partition-$i" + + /** A consumer on `topicFqn` that records which messages it acknowledged and which it handed back + * (negative-acknowledged). Both are the whole question here. `failFirstAck` makes the FIRST + * acknowledgment fail asynchronously - the disconnect race the retry set exists for - while + * every later one succeeds. */ + private final class RecordingConsumer(topicFqn: String, name: String = consumerName, failFirstAck: Boolean = false): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + private val failedOnce = java.util.concurrent.atomic.AtomicBoolean(false) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => name + case "isConnected" => java.lang.Boolean.TRUE + case "acknowledgeAsync" => + if failFirstAck && failedOnce.compareAndSet(false, true) then + CompletableFuture.failedFuture(new RuntimeException("broker went away mid-ack")) + else + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** A stream whose backlog ends on `lastEntryId`. */ + private def stream(topicFqn: String, lastEntryId: Long, name: String = consumerName): StartFromStream = + StartFromStream(startFromStreamId(name, topicFqn), EntryPosition(1L, lastEntryId, -1, 1)) + + /** A listener whose delivery handler throws (exactly as a cancelled gRPC `onNext` does) for any + * message whose key is in `throwOn`, and records the rest. */ + private def listenerThrowingOn(throwOn: Set[String], delivered: ConcurrentLinkedQueue[String]): ConsumerListener = + val handler = ConsumerSessionTargetMessageHandler(onNext = msg => + if throwOn.contains(msg.getKey) then throw new io.grpc.StatusRuntimeException(io.grpc.Status.CANCELLED) + delivered.add(msg.getKey) + () + ) + val l = ConsumerListener(handler) + l.startAcceptingNewMessages() + l + + def spec = suite(this.getClass.toString)( + test("a client-cancel throw mid-batch hands the rest back instead of losing them") { + // A skip of 1 over three streams. Offering c1 resolves the WHOLE batch at once: drop c1 + // (globally earliest), then deliver b1 and a1 in order. The client has cancelled, so the + // first delivery (b1) throws. a1 was already dequeued from the merge; without per-message + // containment it is neither delivered nor acknowledged nor handed back, and never comes + // back. With it, b1 is handed back for redelivery and a1 is still delivered. + val delivered = ConcurrentLinkedQueue[String]() + val listener = listenerThrowingOn(Set("b1"), delivered) + listener.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(p(0), 5), stream(p(1), 5), stream(p(2), 0))) + ) + val c0 = RecordingConsumer(p(0)) + val c1 = RecordingConsumer(p(1)) + val c2 = RecordingConsumer(p(2)) + + // a1 and b1 are held (their streams are still waited for and blind); c1 ends p2's backlog + // and unblocks the merge, resolving the batch. Pulsar catches a throw out of `received` + // per message, so the test does too. + scala.util.Try(listener.received(c0.consumer, message(p(0), "a1", 100L, 0L))) + scala.util.Try(listener.received(c1.consumer, message(p(1), "b1", 90L, 0L))) + scala.util.Try(listener.received(c2.consumer, message(p(2), "c1", 50L, 0L))) + + assertTrue( + c2.acknowledged.asScala.toVector == Vector("c1"), // dropped by the skip + c1.handedBack.asScala.toVector == Vector("b1"), // its delivery threw -> handed back + delivered.asScala.toVector == Vector("a1"), // the message AFTER the throw still got out + c0.acknowledged.asScala.toVector == Vector("a1") // and was acknowledged + ) ?? (s"acked(c2)=${c2.acknowledged.asScala.toVector} handedBack(c1)=${c1.handedBack.asScala.toVector} " + + s"delivered=${delivered.asScala.toVector} acked(c0)=${c0.acknowledged.asScala.toVector}") + }, + test("a Drop resolved by ANOTHER target's offer registers its failed ack where the redelivery arrives") { + // Two TARGETS (two listeners) share one session-wide ordering. Target B's offer + // resolves target A's held message as the drop; A's broker connection fumbles the ack. + // The failed-ack id must be remembered on A - the redelivery arrives THERE - or A + // re-decides it as a fresh message and, with the skip settled, DELIVERS the one + // message the user asked to skip. The old code remembered it on the listener that + // happened to process the batch: B. + val deliveredA = ConcurrentLinkedQueue[String]() + val deliveredB = ConcurrentLinkedQueue[String]() + val listenerA = listenerThrowingOn(Set.empty, deliveredA) + val listenerB = listenerThrowingOn(Set.empty, deliveredB) + val shared = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(p(0), 5, name = "csA"), stream(p(1), 0, name = "csB"))) + ) + listenerA.startFromOrdering = shared + listenerB.startFromOrdering = shared + val cA = RecordingConsumer(p(0), name = "csA", failFirstAck = true) + val cB = RecordingConsumer(p(1), name = "csB") + + // a1 (globally earliest) is held on A; b1 ends B's backlog and resolves the batch on + // B's thread: drop a1, deliver b1. a1's ack fails asynchronously - the decision stands. + scala.util.Try(listenerA.received(cA.consumer, message(p(0), "a1", 50L, 0L))) + scala.util.Try(listenerB.received(cB.consumer, message(p(1), "b1", 100L, 0L))) + // The broker redelivers the un-acked a1 to ITS listener: A. + scala.util.Try(listenerA.received(cA.consumer, message(p(0), "a1", 50L, 0L))) + + assertTrue( + deliveredA.asScala.toVector.isEmpty, // the skipped message never reaches the user + deliveredB.asScala.toVector == Vector("b1"), + cA.handedBack.asScala.toVector == Vector("a1"), // the failed ack handed it back once + cA.acknowledged.asScala.toVector == Vector("a1"), // the redelivery is finalized silently + listenerA.awaitingAckRetryCount == 0, // and the paperwork is closed on A... + listenerB.awaitingAckRetryCount == 0 // ...not parked forever on B + ) ?? (s"deliveredA=${deliveredA.asScala.toVector} deliveredB=${deliveredB.asScala.toVector} " + + s"handedBack(cA)=${cA.handedBack.asScala.toVector} acked(cA)=${cA.acknowledged.asScala.toVector} " + + s"retryA=${listenerA.awaitingAckRetryCount} retryB=${listenerB.awaitingAckRetryCount}") + } + ) diff --git a/server/src/test/scala/consumer/session_runner/mergeMessageDeliveryOrderingTest.scala b/server/src/test/scala/consumer/session_runner/mergeMessageDeliveryOrderingTest.scala new file mode 100644 index 000000000..afa771088 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/mergeMessageDeliveryOrderingTest.scala @@ -0,0 +1,736 @@ +package consumer.session_runner + +import zio.test.* + +/** Cross-topic continuous delivery ordering, pinned without a broker. + * + * The contract is BOUNDED LATENESS, never blocked delivery, and it rests on two rules: + * + * - ORDER-SAFE FAST PATH: a head is emitted at once when every silent stream has already + * offered something at or past its selected time. + * - BOUNDED RESIDENCE: otherwise it waits, but never past the grace measured on the merge's + * own MONOTONIC clock from its own arrival. The selected timestamp comes from outside the + * session, so it is never compared against any clock of ours - a future-dated head must cost + * at most the grace, exactly like everything else. + * + * Late arrivals - anything ordered before what already went out - are emitted immediately, out + * of order, and counted; a redelivery of a message whose delivery FAILED is one of them, never + * a duplicate to swallow. Time is injected and moves only when a test says so. + */ +object mergeMessageDeliveryOrderingTest extends ZIOSpecDefault: + + private val a = "cs@persistent://public/default/topic-a" + private val b = "cs@persistent://public/default/topic-b" + + private val graceMs = 500L + + /** Drain the guaranteed barrier: peek+commit until nothing is safe - the unit-level stand-in + * for the listener's delivery pump, with every send succeeding. */ + private def pumpAll(f: Fixture): Vector[String] = + val out = Vector.newBuilder[String] + var going = true + while going do + f.merge.peekGuaranteed() match + case Some(v) => out += v; f.merge.commitGuaranteed() + case None => going = false + out.result() + + private final class TestClock: + var monoMs: Long = 0L + def advance(ms: Long): Unit = monoMs += ms + + private final class Fixture( + streamIds: Vector[String], + budget: Long = 0, + withRecordedEnds: Boolean = false, + policyOverride: Option[OrderingPolicy] = None + ): + val clock = TestClock() + private var nextEntryId = Map.empty[String, Long].withDefaultValue(0L) + val merge = GlobalSkipMerge[String]( + streamIds = streamIds, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(budget), + nowMs = () => clock.monoMs, + graceMs = graceMs, + policy = policyOverride.getOrElse( + if withRecordedEnds then OrderingPolicy.ExactCutThenBestEffort else OrderingPolicy.BestEffortOnly + ) + ) + private val out = Vector.newBuilder[(String, StartFromOutcome)] + + def offer(streamId: String, publishTime: Long, value: String, atEnd: Boolean = false): Unit = + val entryId = nextEntryId(streamId) + nextEntryId = nextEntryId.updated(streamId, entryId + 1) + offerAt(streamId, publishTime, entryId, value, atEnd) + + /** Offer with an EXPLICIT entry id - how a broker redelivery is reproduced (same entry, + * same publish time, second arrival). `knownFailedRetry` is the listener's explicit + * lifecycle fact - only it licenses delivering a watermarked copy. */ + def offerAt(streamId: String, publishTime: Long, entryId: Long, value: String, atEnd: Boolean = false, knownFailedRetry: Boolean = false): Unit = + val key = MessageOrderKey(publishTime, streamId, ledgerId = 1L, entryId = entryId, batchIndex = -1) + out ++= merge.offer(streamId, key, atBacklogEnd = atEnd, payload = value, knownFailedRetry = knownFailedRetry) + + def sweep(): Unit = out ++= merge.sweepStalled() + + def noteResolved(resolved: (String, StartFromOutcome)): Unit = out += resolved + + def delivered: Vector[String] = out.result().collect { + case (v, StartFromOutcome.Deliver) => v + case (v, StartFromOutcome.DeliverOutOfOrder) => v + } + /** The subset the merge itself flagged as emitted below an already-emitted key. */ + def deliveredFlagged: Vector[String] = out.result().collect { case (v, StartFromOutcome.DeliverOutOfOrder) => v } + def dropped: Vector[String] = out.result().collect { case (v, StartFromOutcome.Drop) => v } + def requeued: Vector[String] = out.result().collect { case (v, StartFromOutcome.Requeue) => v } + + def spec = suite(this.getClass.toString)( + test("a two-stream history replay comes out in GLOBAL publish-time order, never arrival order") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") // b has never spoken: the startup hold keeps this in hand + val heldThroughStartup = f.delivered.isEmpty + f.offer(b, 50, "b-50") // both spoke: 50 emits; 100 waits - b has said nothing past it + val afterFirstExchange = f.delivered + f.offer(a, 200, "a-200") // a queues deeper; the merge still waits on b + val stillPinned = f.delivered + f.offer(b, 150, "b-150") // b moves past 100: 100 and 150 emit, 200 waits on b again + val afterCatchUp = f.delivered + f.offer(b, 250, "b-250") // b moves past 200: 200 emits, 250 now waits on a + val nearTheEnd = f.delivered + f.clock.advance(graceMs + 1) // a stays quiet: 250's own residence expires + f.sweep() + assertTrue( + heldThroughStartup, + afterFirstExchange == Vector("b-50"), + stillPinned == Vector("b-50"), + afterCatchUp == Vector("b-50", "a-100", "b-150"), + nearTheEnd == Vector("b-50", "a-100", "b-150", "a-200"), + f.delivered == Vector("b-50", "a-100", "b-150", "a-200", "b-250"), + f.merge.heldCount == 0, + f.merge.lateDeliveryCount == 0L // an ordered run has nothing to confess + ) ?? s"delivered=${f.delivered} late=${f.merge.lateDeliveryCount}" + }, + test("the startup hold releases after one grace even if a stream never speaks at all") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + val beforeGrace = f.delivered + f.clock.advance(graceMs + 1) + f.sweep() // the timer half: nothing else will ever call in + assertTrue(beforeGrace.isEmpty, f.delivered == Vector("a-100"), f.merge.heldCount == 0) + }, + test("a head nobody has spoken past waits its OWN residence out - that is the whole latency bill") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + f.offer(b, 90, "b-90") + f.clock.advance(graceMs + 1) + f.sweep() + val historyDone = f.delivered + f.offer(a, 300, "a-300") // b's newest word is 90: nothing proves 300 is safe + val heldFresh = f.delivered == historyDone + f.clock.advance(graceMs - 1) + f.sweep() + val heldJustUnderGrace = f.delivered == historyDone + f.clock.advance(2) + f.sweep() + assertTrue( + historyDone == Vector("b-90", "a-100"), + heldFresh, + heldJustUnderGrace, // the boundary is exact: one tick short still holds + f.delivered == Vector("b-90", "a-100", "a-300") + ) ?? s"delivered=${f.delivered}" + }, + test("a FUTURE-DATED head cannot wedge its stream - producer clocks are nobody's to trust") { + // Under a wall-clock release rule a head stamped ten minutes ahead sat pinned for ten + // minutes. Residence does not care what the producer's clock said. + val f = Fixture(Vector(a, b)) + f.offer(b, 50, "b-50") + f.offer(a, 50 + 10 * 60 * 1000, "a-future") + f.sweep() + val heldWhileFresh = f.delivered + f.clock.advance(graceMs + 1) + f.sweep() + assertTrue( + heldWhileFresh == Vector("b-50"), + f.delivered == Vector("b-50", "a-future"), + f.merge.heldCount == 0 + ) ?? s"delivered=${f.delivered}" + }, + test("timestamps running BACKWARD within one stream keep append order, and the dip is counted late") { + val f = Fixture(Vector(a, b)) + f.offer(b, 60, "b-60") + f.offer(a, 100, "a-100") // then a's producer clock steps back: + f.offer(a, 90, "a-90") + f.clock.advance(graceMs + 1) + f.sweep() + assertTrue( + // b-60 first (both spoke), then a IN APPEND ORDER - 100 before 90 - never resorted. + f.delivered == Vector("b-60", "a-100", "a-90"), + f.merge.lateDeliveryCount == 1L, // the dip is exactly one confessed disorder + // ...and the dip itself is the flagged row (2026-08-11): the marker pairs with the + // exact message the ledger counted, not with a session-level total alone. + f.deliveredFlagged == Vector("a-90") + ) ?? s"delivered=${f.delivered} late=${f.merge.lateDeliveryCount} flagged=${f.deliveredFlagged}" + }, + test("a stream released by the grace delivers LATE when it wakes - immediately, counted, nothing wedged") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + f.clock.advance(graceMs + 1) // b never spoke; the startup hold expires + f.sweep() + val emittedPastB = f.delivered + f.offer(b, 80, "b-late") // older than what already went out: best effort, worn openly + assertTrue( + emittedPastB == Vector("a-100"), + f.delivered == Vector("a-100", "b-late"), + f.merge.heldCount == 0, + f.merge.lateDeliveryCount == 1L, + f.deliveredFlagged == Vector("b-late") // the woken straggler carries the marker + ) ?? s"delivered=${f.delivered} late=${f.merge.lateDeliveryCount} flagged=${f.deliveredFlagged}" + }, + test("THE LIFECYCLE OF A COPY IS EXPLICIT: only a listener-confirmed failure licenses redelivery") { + // The two loss paths the old inferences opened, closed from both sides. A copy with + // NO recorded failure is handed back whatever the queue says - the original may be + // held, limiter-queued, mid-send, or acked-with-the-ack-in-flight. Only the + // listener's explicit "this delivery failed" turns the next copy into the retry. + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + f.offer(b, 50, "b-50") + f.clock.advance(graceMs + 1) + f.sweep() + val firstPass = f.delivered // both delivered once; nothing held + + // A copy WITHOUT a recorded failure: original left the merge, fate unknown - requeue. + f.offerAt(a, 100, entryId = 0, value = "a-100-copy-unknown") + // The listener then SAW the delivery fail; the next copy is the retry. + f.offerAt(a, 100, entryId = 0, value = "a-100-retry", knownFailedRetry = true) + + // And a copy whose ORIGINAL is still held is handed back too - never acknowledged. + f.offer(a, 300, "a-300") // held: b's newest word is 50, residence fresh + f.offerAt(a, 300, entryId = 1, value = "a-300-copy") + + assertTrue( + firstPass == Vector("b-50", "a-100"), + f.delivered == Vector("b-50", "a-100", "a-100-retry"), + f.requeued == Vector("a-100-copy-unknown", "a-300-copy"), + f.dropped.isEmpty, + f.merge.lateDeliveryCount >= 1L // the retry is late by definition + ) ?? s"delivered=${f.delivered} requeued=${f.requeued} dropped=${f.dropped}" + }, + test("skip-n WITH merge: the cut stays exact, and ordering simply refuses to stop afterwards") { + val f = Fixture(Vector(a, b), budget = 3, withRecordedEnds = true) + f.offer(a, 10, "a-10") + f.offer(b, 20, "b-20") + f.offer(a, 30, "a-30") + f.offer(b, 40, "b-40", atEnd = true) + f.offer(a, 50, "a-50", atEnd = true) + val atTheCut = (f.dropped, f.delivered) + // b-40 is out - the very offer of a-50 proved it order-safe (both streams have then + // spoken at or past 40) - while a-50 itself still waits: b's newest word is 40, so + // past the cut it is held like any other head, by its peers or its own residence. + // The exact-cut knowledge is spent; the continuous rules govern. + f.offer(a, 70, "a-70") + val afterA70 = f.delivered // a queues deeper; nothing new is provably safe + f.offer(b, 60, "b-60") // b speaks past 50: the exact path emits 50, then 60 + f.clock.advance(graceMs + 1) + f.sweep() // a-70's residence expires + assertTrue( + atTheCut == (Vector("a-10", "b-20", "a-30"), Vector("b-40")), + afterA70 == Vector("b-40"), + f.delivered == Vector("b-40", "a-50", "b-60", "a-70"), + !f.merge.isSettled, + f.merge.heldCount == 0 + ) ?? s"dropped=${f.dropped} delivered=${f.delivered}" + }, + test("the flow-control watermarks keep working after the cut - a hot stream nobody spoke past is paused") { + val clock = TestClock() + val merge = GlobalSkipMerge[String]( + streamIds = Vector(a, b), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.monoMs, + graceMs = graceMs, + policy = OrderingPolicy.BestEffortOnly, + pauseStreamAt = 5, + resumeStreamAt = 1 + ) + def offer(streamId: String, publishTime: Long, entryId: Long): Vector[(String, StartFromOutcome)] = + merge.offer(streamId, MessageOrderKey(publishTime, streamId, 1L, entryId, -1), atBacklogEnd = false, payload = s"$streamId-$publishTime") + offer(b, 50, 0) // b speaks once, then stays silent: nothing may pass 50 un-aged + (1 to 6).foreach(i => offer(a, 100 + i, i)) + val hotStreamPaused = merge.desiredPausedStreams.contains(a) + offer(b, 5_000, 1) // b speaks far past a's queue: everything drains + assertTrue( + hotStreamPaused, + merge.desiredPausedStreams.isEmpty, + merge.heldCount == 1 // only b's own final head remains + ) ?? s"desired=${merge.desiredPausedStreams} held=${merge.heldCount}" + }, + test("an ordering-only layer never settles and never counts skip progress") { + val f = Fixture(Vector(a, b)) + f.offer(a, 100, "a-100") + f.offer(b, 50, "b-50") + f.merge.settleIfDone() + assertTrue( + !f.merge.isSettled, + f.merge.isContinuousOrdering, + f.merge.progressDiscard.exists(_.remaining == 0) + ) + }, + test("a NON-PERSISTENT stream - no knowable positions - merges in order, nothing swallowed as a duplicate") { + // Non-persistent message ids carry no ledger/entry, so every one used to land at or + // below the (-1,-1) watermark its first message recorded: swallowed as a duplicate + // during a cut, or shunted down the late-retry path (instant, out of order) in + // continuous mode - which is exactly how a mixed-persistency e2e caught it. + val f = Fixture(Vector(a, b)) + // THE REAL SHAPE, measured against a broker: every non-persistent message arrives as + // ledger 0, entry 0 - a plausible-looking position that never advances. + val nonPersistentFqn = "non-persistent://public/default/topic-b" + def offerUnpositioned(publishTime: Long, value: String): Unit = + f.merge.offer(b, MessageOrderKey(publishTime, nonPersistentFqn, ledgerId = 0L, entryId = 0L, batchIndex = -1), atBacklogEnd = true, payload = value) match + case resolved => resolved.foreach(r => f.noteResolved(r)) + + f.offer(a, 100, "a-100") + offerUnpositioned(50, "b-50") + offerUnpositioned(150, "b-150") // the message the watermark used to damn as a copy + f.offer(a, 200, "a-200") + offerUnpositioned(250, "b-250") + f.clock.advance(graceMs + 1) + f.sweep() + + assertTrue( + f.dropped.isEmpty, // NOTHING is a duplicate on a stream that cannot have any + f.delivered == Vector("b-50", "a-100", "b-150", "a-200", "b-250"), + f.merge.lateDeliveryCount == 0L + ) ?? s"delivered=${f.delivered} dropped=${f.dropped} late=${f.merge.lateDeliveryCount}" + }, + test("EQUAL producer timestamps are never called order-safe - a tied head waits its residence out") { + // A silent stream that offered THROUGH time t may offer another AT t (producer + // timestamps are non-decreasing), and the documented tie-break could sort it first. + // The floor is therefore STRICT: a tied candidate is not provably safe, so it waits - + // and a tie is not a "late delivery" either, however the full key orders it. + val f = Fixture(Vector(a, b)) + f.offer(b, 100, "b-100") + f.offer(a, 100, "a-100") // both heads: the exact path emits a first (topic tie-break) + val afterExactPair = f.delivered + // b-100 remains: a is now the silent stream with lastOffered = 100, and 100 < 100 + // is false - the tie must not ride the fast path. + val heldOnTie = f.delivered == afterExactPair && f.merge.heldCount == 1 + f.clock.advance(graceMs + 1) + f.sweep() + assertTrue( + afterExactPair == Vector("a-100"), + heldOnTie, + f.delivered == Vector("a-100", "b-100"), + f.merge.lateDeliveryCount == 0L // a millisecond tie is a tie-break, not disorder + ) ?? s"delivered=${f.delivered} late=${f.merge.lateDeliveryCount}" + }, + test("an IRREVERSIBLY LATE head skips the wait - no window can un-late it") { + // Three streams; c speaks once and stays silent with a LOW floor, so nothing newer is + // ever provably safe. a-100 goes out by residence; then b-50 arrives - already older + // than what the user has seen. Holding it for another window would be pure latency + // with zero ordering value: it must go out immediately, counted. + val f = Fixture(Vector(a, b, "cs@persistent://public/default/topic-c")) + f.offer("cs@persistent://public/default/topic-c", 10, "c-10") + f.offer(a, 100, "a-100") + f.offer(b, 60, "b-60") // all three spoke: startup hold ends; 60 and 100 wait on c's floor + f.clock.advance(graceMs + 1) + f.sweep() // residence releases c-10, b-60, a-100 in order + val historyDone = f.delivered + f.offer(b, 50, "b-late") // older than a-100, floor still blocked by silent c + val emittedWithoutWaiting = f.delivered.lastOption.contains("b-late") + assertTrue( + historyDone == Vector("c-10", "b-60", "a-100"), + emittedWithoutWaiting, + f.merge.lateDeliveryCount == 1L + ) ?? s"delivered=${f.delivered} late=${f.merge.lateDeliveryCount}" + }, + test("A THOUSAND STREAMS, 99/1 SKEW: O(1) per message, bounded lateness, flow control engaged, zero disorder") { + // The wide-session pin the review asked for, in COUNTED operations rather than wall + // time (wall time flakes; operation counts do not). One hot stream pours 20,000 + // messages through a merge with 999 silent peers. If the emission gate were + // O(silent streams) per message - the shape the cached-min fast path exists to + // prevent - this run would read the clock tens of millions of times; O(1) keeps it + // within a few reads per message. + var clockReads = 0L + var mono = 0L + val streams = (0 until 1000).map(i => s"cs@persistent://public/default/wide-$i").toVector + val merge = GlobalSkipMerge[String]( + streamIds = streams, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => { clockReads += 1; mono }, + graceMs = graceMs, + policy = OrderingPolicy.BestEffortOnly + ) + val out = Vector.newBuilder[(String, StartFromOutcome)] + var entry = Map.empty[String, Long].withDefaultValue(0L) + def offer(id: String, t: Long, v: String): Unit = + val e = entry(id); entry = entry.updated(id, e + 1) + out ++= merge.offer(id, MessageOrderKey(t, id, 1L, e, -1), atBacklogEnd = false, payload = v) + + // Every peer speaks once (the startup hold ends with real floors everywhere)... + val seeds = streams.indices.map(i => f"seed-$i%04d").toVector + streams.zipWithIndex.foreach((id, i) => offer(id, 10L + i, seeds(i))) + // ...then the hot stream pours 20,000 newer messages while 999 peers stay silent. + val hot = streams.head + val hotValues = (0 until 20_000).map(i => f"hot-$i%05d").toVector + hotValues.zipWithIndex.foreach((v, i) => offer(hot, 5_000L + i, v)) + + val heldAtPeak = merge.heldCount + val hotPausedUnderPressure = merge.desiredPausedStreams.contains(hot) + + mono += graceMs + 1 // every residence expires; the sweep releases the world in order + out ++= merge.sweepStalled() + + val delivered = out.result().collect { case (v, StartFromOutcome.Deliver) => v } + assertTrue( + // Everything held except exactly TWO exact-path emissions: the last seed's offer + // makes every stream head-bearing once (silent set momentarily empty), releasing + // seed-0; the first hot offer re-heads stream 0 and releases seed-1 the same way. + // From then on stream 1 is the silent floor nobody has spoken past. + heldAtPeak == 21_000 - 2, + hotPausedUnderPressure, // the per-stream watermark asked for the hot source to pause + delivered.size == 21_000, + delivered == seeds ++ hotValues, // global publish-time order, end to end + merge.desiredPausedStreams.isEmpty, + merge.heldCount == 0, + merge.lateDeliveryCount == 0L, + clockReads < 21_000L * 10 // O(1) per message; the O(streams) shape would be ~21M + ) ?? s"held=$heldAtPeak delivered=${delivered.size} clockReads=$clockReads late=${merge.lateDeliveryCount}" + }, + test("GUARANTEED: a silent stream holds delivery FOREVER - no residence, no startup release, no give-up") { + val f = Fixture(Vector(a, b), policyOverride = Some(OrderingPolicy.GuaranteedOnly)) + f.offer(a, 100, "a-100") + f.clock.advance(graceMs * 100) // fifty seconds of silence: best-effort would have released long ago + f.sweep() + val heldThroughSilence = (f.delivered, f.merge.heldCount) + val waitingAfterWarning = f.merge.stalledStreamCount + f.offer(b, 50, "b-50") // b finally speaks: the exact rule emits b-50 - and ONLY it, + val pumped = pumpAll(f) // because delivering it makes b silent again and a-100 must + val stillHeld = f.merge.heldCount // wait for b's NEXT word. No compromises means this. + val waitingAfterProgress = f.merge.stalledStreamCount + f.offer(b, 150, "b-150") // b speaks past 100: a-100 is provably safe now + val pumpedAfter = pumpAll(f) + assertTrue( + heldThroughSilence == (Vector.empty, 1), + waitingAfterWarning == 1, + pumped == Vector("b-50"), + stillHeld == 1, + waitingAfterProgress == 0, + pumpedAfter == Vector("a-100"), // and b-150 now waits on silent a, in its turn + f.merge.heldCount == 1, + f.merge.lateDeliveryCount == 0L + ) ?? s"held=$heldThroughSilence pumped=$pumped after=$pumpedAfter" + }, + test("GUARANTEED: the hold is memory-BOUNDED - the ahead stream is paused at its watermark, the blind one never") { + // The no-escape wait must not be a memory hole. When one stream races ahead while + // another is headless, the ahead stream's backlog accumulates in the merge only up to + // the same flow-control watermarks every other mode uses; past them its CONSUMER is + // marked for pause until the barrier drains it. Delaying RECEIPT cannot violate the + // order - only early emission could - so the backpressure costs the guarantee nothing. + // The blind stream holds nothing and must never be paused: its next message is the + // only thing that can unblock the barrier. + val clock = TestClock() + val merge = GlobalSkipMerge[String]( + streamIds = Vector(a, b), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.monoMs, + graceMs = graceMs, + policy = OrderingPolicy.GuaranteedOnly, + pauseStreamAt = 5, + resumeStreamAt = 1 + ) + def offer(streamId: String, publishTime: Long, entryId: Long): Unit = + merge.offer(streamId, MessageOrderKey(publishTime, streamId, 1L, entryId, -1), atBacklogEnd = false, payload = s"m-$publishTime") + () + def drain(): Vector[String] = + val out = Vector.newBuilder[String] + var going = true + while going do + merge.peekGuaranteed() match + case Some(v) => out += v; merge.commitGuaranteed() + case None => going = false + out.result() + + // b is blind; a races 6 deep - past the per-stream watermark of 5. + (1 to 6).foreach(i => offer(a, 100L + i, i.toLong)) + val aheadPausedAtWatermark = merge.desiredPausedStreams + val heldAtPeak = merge.heldCount + + // b speaks far past a's queue: the barrier drains everything a held, in order, and + // the drain releases the pause marks. + offer(b, 5_000L, 0L) + val delivered = drain() + + assertTrue( + aheadPausedAtWatermark == Set(a), // the ahead stream is held still, the blind one runs + heldAtPeak == 6, + delivered == (1 to 6).map(i => s"m-${100 + i}").toVector, + merge.desiredPausedStreams.isEmpty, // the drain released the backpressure + merge.heldCount == 1 // b-5000 now waits on drained a, in its turn + ) ?? s"paused=$aheadPausedAtWatermark delivered=$delivered desired=${merge.desiredPausedStreams}" + }, + test("GUARANTEED: the barrier retries the SAME head in place - a failed send costs latency, never order") { + val f = Fixture(Vector(a, b), policyOverride = Some(OrderingPolicy.GuaranteedOnly)) + f.offer(a, 100, "a-100") + f.offer(b, 50, "b-50") + // First attempt at the safe head (b-50) fails: commit never happens. + val peeked1 = f.merge.peekGuaranteed() + f.merge.abortGuaranteed() + val heldAfterFailure = f.merge.heldCount + // The retry sees the SAME head; only success advances. + val peeked2 = f.merge.peekGuaranteed() + f.merge.commitGuaranteed() + // Delivering b-50 makes b silent again: a-100 is NOT provably safe and must wait - + // the same no-escape rule, seen through the barrier. + val peeked3 = f.merge.peekGuaranteed() + assertTrue( + peeked1.contains("b-50"), + heldAfterFailure == 2, // nothing left the merge on failure + peeked2.contains("b-50"), + peeked3.isEmpty, + f.merge.heldCount == 1, + f.merge.lateDeliveryCount == 0L // retried in place = never late + ) ?? s"p1=$peeked1 p2=$peeked2 p3=$peeked3" + }, + test("GUARANTEED: a copy of the in-flight head is requeued, never a second delivery") { + val f = Fixture(Vector(a, b), policyOverride = Some(OrderingPolicy.GuaranteedOnly)) + f.offer(a, 100, "a-100") + f.offer(b, 50, "b-50") + val peeked = f.merge.peekGuaranteed() // b-50 is in flight + f.offerAt(b, 50, entryId = 0, value = "b-50-copy") // unload redelivery meanwhile + f.merge.commitGuaranteed() + assertTrue( + peeked.contains("b-50"), + f.requeued == Vector("b-50-copy"), + f.delivered.isEmpty // barrier deliveries do not flow through offer resolutions + ) ?? s"requeued=${f.requeued}" + }, + test("GUARANTEED after a skip cut: the cut stays exact, then the REPLAY completes at the recorded ends") { + // FLIPPED PIN (owner decision 2026-08-09, the exact-replay redesign): this cell used + // to assert that the barrier re-armed EVERY stream after the cut and held the tail + // forever. Under the replay contract a stream that reached its recorded end is + // FINISHED, so once both streams have, the remaining heap drains in key order - the + // last message included - and the merge reports the chunk caught up. + val f = Fixture(Vector(a, b), budget = 3, withRecordedEnds = true, policyOverride = Some(OrderingPolicy.ExactCutThenGuaranteed)) + f.offer(a, 10, "a-10") + f.offer(b, 20, "b-20") + f.offer(a, 30, "a-30") + f.offer(b, 40, "b-40", atEnd = true) + val caughtUpMidReplay = f.merge.isReplayCaughtUp // a's end is still outstanding + f.offer(a, 50, "a-50", atEnd = true) + val cut = f.dropped + // Every recorded end is in hand: the barrier drains the whole heap - no held tail. + val pumped = pumpAll(f) + assertTrue( + cut == Vector("a-10", "b-20", "a-30"), + !caughtUpMidReplay, + pumped == Vector("b-40", "a-50"), + f.merge.heldCount == 0, + f.merge.isReplayCaughtUp, // the caller's cue to auto-pause and signal + !f.merge.isSettled // continuous ordering still never settles + ) ?? s"cut=$cut pumped=$pumped held=${f.merge.heldCount} caughtUp=${f.merge.isReplayCaughtUp}" + }, + test("GUARANTEED: EQUAL publish times resolve IMMEDIATELY by the documented tiebreak - topic, then log position") { + // Batching is the producer default, so tied timestamps are the production norm - and + // Guaranteed is the mode chosen precisely for exact order. A tie must resolve NOW + // (no residence wait exists in this mode, and none may be invented for ties) and + // DETERMINISTICALLY, by the documented total order: publish time, then topic FQN, + // then (ledger, entry, batch index). The stream ids are deliberately ordered AGAINST + // the topic order, so any fallback to heap/stream-id order goes red here. + def guaranteedMerge(streams: Vector[String]) = GlobalSkipMerge[String]( + streamIds = streams, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => 0L, + graceMs = graceMs, + policy = OrderingPolicy.GuaranteedOnly + ) + def drain(merge: GlobalSkipMerge[String]): Vector[String] = + val out = Vector.newBuilder[String] + var going = true + while going do + merge.peekGuaranteed() match + case Some(v) => out += v; merge.commitGuaranteed() + case None => going = false + out.result() + + // TOPIC tiebreak: three streams, every head at t=100, stream ids reverse-ordered + // against their topic FQNs. A second, later message per stream keeps each stream + // head-bearing while the tied heads drain, so the ties are seen to resolve without + // any clock movement or sweep - only the last commit leaves a headless stream. + val topicA = "persistent://public/default/tied-a" + val topicB = "persistent://public/default/tied-b" + val topicC = "persistent://public/default/tied-c" + val byTopic = guaranteedMerge(Vector("z-src", "y-src", "x-src")) + byTopic.offer("z-src", MessageOrderKey(100L, topicA, 1L, 0L, -1), atBacklogEnd = false, payload = "a-100") + byTopic.offer("z-src", MessageOrderKey(900L, topicA, 1L, 1L, -1), atBacklogEnd = false, payload = "a-900") + byTopic.offer("y-src", MessageOrderKey(100L, topicB, 1L, 0L, -1), atBacklogEnd = false, payload = "b-100") + byTopic.offer("y-src", MessageOrderKey(901L, topicB, 1L, 1L, -1), atBacklogEnd = false, payload = "b-901") + byTopic.offer("x-src", MessageOrderKey(100L, topicC, 1L, 0L, -1), atBacklogEnd = false, payload = "c-100") + byTopic.offer("x-src", MessageOrderKey(902L, topicC, 1L, 1L, -1), atBacklogEnd = false, payload = "c-902") + val topicTieOrder = drain(byTopic) + + // LOG-POSITION tiebreak: two streams over the SAME topic FQN (two targets on one + // topic) with heads tied on both time and topic - the entry id must decide, again + // against the stream-id order. + val tiedTopic = "persistent://public/default/tied-same" + val byEntry = guaranteedMerge(Vector("za", "yb")) + byEntry.offer("za", MessageOrderKey(100L, tiedTopic, 1L, 5L, -1), atBacklogEnd = false, payload = "a-e5") + byEntry.offer("za", MessageOrderKey(100L, tiedTopic, 1L, 8L, -1), atBacklogEnd = false, payload = "a-e8") + byEntry.offer("yb", MessageOrderKey(100L, tiedTopic, 1L, 3L, -1), atBacklogEnd = false, payload = "b-e3") + byEntry.offer("yb", MessageOrderKey(100L, tiedTopic, 1L, 9L, -1), atBacklogEnd = false, payload = "b-e9") + val entryTieOrder = drain(byEntry) + + assertTrue( + // a/b/c-100 drain at once in TOPIC order, then the exact rule continues with 900; + // z-src only then runs dry, which is what stops the barrier - not the tie. + topicTieOrder == Vector("a-100", "b-100", "c-100", "a-900"), + byTopic.heldCount == 2, // b-901 and c-902 wait on the drained z-src, per the promise + byTopic.lateDeliveryCount == 0L, // a tie-break is not disorder + entryTieOrder == Vector("b-e3", "a-e5", "a-e8"), + byEntry.heldCount == 1, // b-e9 waits on drained za + byEntry.lateDeliveryCount == 0L + ) ?? s"topicTieOrder=$topicTieOrder entryTieOrder=$entryTieOrder" + }, + test("GUARANTEED AT A THOUSAND STREAMS: selected-time merge order, no clock, and the by-design tail") { + // The width pin for the no-compromise mode, in counted operations like its + // best-effort sibling above. 1000 streams x 20 messages, offered stream by stream - + // arrival order maximally unlike emission order - then pumped through the + // deliver-then-advance barrier. Three facts at width: + // - emission is the selected-time order across a thousand append-ordered queues; + // - this mode has no residence, so the only clock bill is arrival stamping - O(1) + // per offer, nothing per emission; + // - on finite data the pump stops the instant the first stream drains dry, so of + // the final round exactly ONE message emits - the documented tail; one closing + // word per stream then releases everything except, again, the single message + // past the globally-last head. + var clockReads = 0L + val streams = (0 until 1000).map(i => f"cs@persistent://public/default/gwide-$i%04d").toVector + val merge = GlobalSkipMerge[String]( + streamIds = streams, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => { clockReads += 1; 0L }, + graceMs = graceMs, + policy = OrderingPolicy.GuaranteedOnly + ) + var entry = Map.empty[String, Long].withDefaultValue(0L) + val offerResolutions = Vector.newBuilder[(String, StartFromOutcome)] + def offer(id: String, t: Long, v: String): Unit = + val e = entry(id); entry = entry.updated(id, e + 1) + offerResolutions ++= merge.offer(id, MessageOrderKey(t, id, 1L, e, -1), atBacklogEnd = false, payload = v) + def drain(): Vector[String] = + val out = Vector.newBuilder[String] + var going = true + while going do + merge.peekGuaranteed() match + case Some(v) => out += v; merge.commitGuaranteed() + case None => going = false + out.result() + + // Message j of stream i is stamped j*10_000 + i: the global order interleaves ALL + // thousand streams every round, while each stream arrives as one 20-deep burst. + streams.zipWithIndex.foreach { (id, i) => + (0 until 20).foreach(j => offer(id, j * 10_000L + i, f"m-$j%02d-$i%04d")) + } + val drained = drain() + val heldAtFirstStop = merge.heldCount + val expectedPrefix = (0 until 19).flatMap(j => (0 until 1000).map(i => f"m-$j%02d-$i%04d")).toVector :+ "m-19-0000" + + streams.zipWithIndex.foreach((id, i) => offer(id, 1_000_000L + i, f"close-$i%04d")) + val drainedAfterClose = drain() + val expectedAfter = (1 until 1000).map(i => f"m-19-$i%04d").toVector :+ "close-0000" + + assertTrue( + drained.size == 19_001, + drained == expectedPrefix, + heldAtFirstStop == 999, // the rest of the final round, held on drained stream 0 + drainedAfterClose == expectedAfter, + merge.heldCount == 999, // the closing words of streams 1..999, in their turn + offerResolutions.result().isEmpty, // advance() emits NOTHING in this mode - only the barrier delivers + merge.lateDeliveryCount == 0L, // exactness at width: nothing to confess + clockReads < 21_000L * 10 // arrival stamps only; any per-emission clock would blow this + ) ?? s"drained=${drained.size} after=${drainedAfterClose.size} held=${merge.heldCount} clockReads=$clockReads late=${merge.lateDeliveryCount}" + }, + test("GUARANTEED AND IDLE AT WIDTH: a tick with nothing new to say costs NOTHING per stream") { + // Every continuous layer was given the BEST-EFFORT 250 ms sweep cadence, and each of + // those ticks materialized the whole waited-for-but-silent stream set. Guaranteed has + // no residence to expire and never gives a stream up, so at 2,000 idle streams that was + // four full-set scans a second, per session, forever - producing nothing whatsoever. + // The report path was no better: the waiting-stream COUNT the client is shown was + // another full materialization, once per delivered frame. + // + // Counted operations, not wall time (wall time flakes; operation counts do not), and + // the same instrument the wide-skew tests above use. + var mono = 0L + val streams = (0 until 2_000).map(i => f"cs@persistent://public/default/idle-$i%04d").toVector + val merge = GlobalSkipMerge[String]( + streamIds = streams, + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => mono, + graceMs = graceMs, + policy = OrderingPolicy.GuaranteedOnly + ) + // One stream speaks; the merge is now genuinely waiting on the other 1,999 forever. + merge.offer(streams.head, MessageOrderKey(100L, streams.head, 1L, 0L, -1), atBacklogEnd = false, payload = "m") + + mono += startFromMergeStallWarnMs + 1 + merge.sweepStalled() // the one tick with something to say: it discloses the stall + val costOfTheDisclosure = merge.silentStreamInspections + + // A hundred idle ticks and a hundred client frames afterwards, nothing changed. + (1 to 100).foreach { _ => + mono += mergeTopicsSweepPeriodMs + merge.sweepStalled() + merge.stalledStreamCount + } + + val hugeSet = (0 until 2_000).map(i => f"stream-$i%04d") + val logLine = stalledStreamsForLog(hugeSet) + + assertTrue( + merge.stallWarningCount == 1L, // surfaced once, as before + merge.stalledStreamCount == 1_999, // and the answer is unchanged + merge.heldCount == 1, // still holding: Guaranteed waits forever, by design + costOfTheDisclosure <= 2L * streams.size, // naming the streams costs one pass + merge.silentStreamInspections == costOfTheDisclosure, // the 100 idle ticks cost none + continuousSweepPeriodMs(guaranteed = true) > continuousSweepPeriodMs(guaranteed = false), + continuousSweepPeriodMs(guaranteed = false) == mergeTopicsSweepPeriodMs, + logLine.length < 1_000, // one line, not a 2,000-name wall + logLine.contains("stream-0000"), + logLine.contains(s"${2_000 - stallLogStreamNameCap} more") + ) ?? (s"disclosureCost=$costOfTheDisclosure afterwards=${merge.silentStreamInspections} " + + s"streams=${streams.size} warnings=${merge.stallWarningCount} stalled=${merge.stalledStreamCount} " + + s"guaranteedPeriod=${continuousSweepPeriodMs(guaranteed = true)} " + + s"bestEffortPeriod=${continuousSweepPeriodMs(guaranteed = false)} logLineLength=${logLine.length}") + }, + test("the plans wire the mode through: keepOrderingAfterCut on the skip, OrderOnly for everyone else") { + import consumer.session_config.MessageDeliveryOrder + val streams = Vector(StartFromStream(a, EntryPosition(1, 10, -1, 1)), StartFromStream(b, EntryPosition(1, 20, -1, 1))) + val skipKeep = StartFromOrdering.make[String](StartFromOrderingPlan.GlobalSkip(3, streams, MessageDeliveryOrder.BestEffort)) + val skipGuaranteed = StartFromOrdering.make[String](StartFromOrderingPlan.GlobalSkip(3, streams, MessageDeliveryOrder.Guaranteed)) + val skipPlain = StartFromOrdering.make[String](StartFromOrderingPlan.GlobalSkip(3, streams)) + val orderOnly = StartFromOrdering.make[String](StartFromOrderingPlan.Ordered( + Vector(StartFromStream(a, EntryPosition.empty), StartFromStream(b, EntryPosition.empty)), + MessageDeliveryOrder.BestEffort + )) + val guaranteedOnly = StartFromOrdering.make[String](StartFromOrderingPlan.Ordered( + Vector(StartFromStream(a, EntryPosition(1, 10, -1, 1)), StartFromStream(b, EntryPosition(1, 20, -1, 1))), + MessageDeliveryOrder.Guaranteed + )) + val passThrough = StartFromOrdering.make[String](StartFromOrderingPlan.PassThrough) + assertTrue( + skipKeep.isContinuousOrdering, + !skipKeep.isGuaranteedOrdering, + skipGuaranteed.isContinuousOrdering, + skipGuaranteed.isGuaranteedOrdering, + !skipPlain.isContinuousOrdering, + orderOnly.isContinuousOrdering, + !orderOnly.isGuaranteedOrdering, + guaranteedOnly.isGuaranteedOrdering, + !passThrough.isContinuousOrdering + ) + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/messageConvertersTest.scala b/server/src/test/scala/consumer/session_runner/messageConvertersTest.scala new file mode 100644 index 000000000..083f3597b --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/messageConvertersTest.scala @@ -0,0 +1,123 @@ +package consumer.session_runner + +import consumer.deserializer.deserializers.TreatBytesAsJson +import io.circe.parser.parse as parseJson +import org.apache.pulsar.client.api.Schema +import org.apache.pulsar.client.impl.MessageImpl +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.nio.ByteBuffer +import scala.jdk.CollectionConverters.* + +/** `serializeMessage` builds both the protobuf message sent to the browser and the JSON view that + * user filters/projections see as `message.*` in JS. + * + * Regression context: the key was built by splicing the raw string into a JSON literal + * (`parseJson(s""" "$key" """)`), so a key containing a quote, backslash or newline produced + * invalid JSON, parsed to Left and was SILENTLY DROPPED - key filters and key projections then + * quietly missed those messages. The value path had always used circe for exactly this reason. + */ +object messageConvertersTest extends ZIOSpecDefault: + + private val topicName = "persistent://public/default/topic-a" + private val deserializer = consumer.deserializer.Deserializer(deserializer = TreatBytesAsJson()) + + private def message( + key: String | Null = null, + payload: String = "null", + properties: Map[String, String] = Map.empty, + eventTime: Long = 0L, + publishTime: Long = 0L + ): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + // publish_time is mandatory on MessageMetadata - reading it when unset throws + // IllegalStateException, so every fixture message carries one. + md.setPublishTime(if publishTime > 0 then publishTime else 1_700_000_000_000L) + if key != null then md.setPartitionKey(key) + if eventTime > 0 then md.setEventTime(eventTime) + properties.foreach { (k, v) => md.addProperty().setKey(k).setValue(v) } + val msg = MessageImpl.create[Array[Byte]]( + md, + ByteBuffer.wrap(payload.getBytes("UTF-8")), + Schema.BYTES, + topicName + ) + // serializeMessage reads msg.getMessageId.toByteArray; MessageImpl.create leaves it null. + msg.setMessageId(new org.apache.pulsar.client.impl.MessageIdImpl(1L, 2L, -1)) + msg + + private def serialize(msg: MessageImpl[Array[Byte]]): ConsumerSessionMessage = + converters.serializeMessage(Map.empty, msg, deserializer) + + /** The key as the browser/JS side sees it, read back out of the JSON view. */ + private def keyInJsonView(msg: MessageImpl[Array[Byte]]): Option[String] = + parseJson(serialize(msg).messageAsJsonOmittingValue).toOption + .flatMap(_.hcursor.downField("key").as[String].toOption) + + def spec = suite(this.getClass.toString)( + test("a plain key survives serialization") { + assertTrue(keyInJsonView(message(key = "order-42")).contains("order-42")) + }, + test("a key containing a double quote survives") { + // Previously: the spliced literal became "a"b" -> invalid JSON -> key dropped entirely. + assertTrue(keyInJsonView(message(key = "a\"b")).contains("a\"b")) + }, + test("a key containing a backslash survives") { + assertTrue(keyInJsonView(message(key = "a\\b")).contains("a\\b")) + }, + test("a key containing a newline survives") { + assertTrue(keyInJsonView(message(key = "line1\nline2")).contains("line1\nline2")) + }, + test("a key containing a tab and a control character survives") { + assertTrue(keyInJsonView(message(key = "a\tbc")).contains("a\tbc")) + }, + test("a non-ASCII key survives") { + assertTrue(keyInJsonView(message(key = "世界-Grüße-🌍")).contains("世界-Grüße-🌍")) + }, + test("an absent key yields no key field rather than an empty string") { + val json = parseJson(serialize(message(key = null)).messageAsJsonOmittingValue).toOption.get + val keyField = json.hcursor.downField("key").as[String].toOption + assertTrue(keyField.isEmpty) + }, + test("an empty key is preserved as an empty string") { + assertTrue(keyInJsonView(message(key = "")).contains("")) + }, + test("the JSON view is always parseable, whatever the key contains") { + val nasty = List("\"", "\\", "\n", "\"}", "{\"a\":1}", "\u0000", "'", "世界") + val unparseable = nasty.filter { k => + parseJson(serialize(message(key = k)).messageAsJsonOmittingValue).isLeft + } + assertTrue(unparseable.isEmpty) ?? s"keys that produced invalid JSON: ${unparseable.map(k => s"[$k]").mkString(", ")}" + }, + test("the protobuf key matches the JSON view") { + // Both sides feed the UI, and they must agree on the actual key VALUE, not merely on + // whether one exists (the old `isDefined == isDefined` passed even if one side mangled a + // quote). The pb key is the circe-encoded string (`_.asJson.noSpaces`), so decode it + // before comparing - a key with a quote is exactly the case the encoding exists for. + val msg = message(key = "a\"b") + val serialized = serialize(msg) + val pbKeyDecoded = serialized.messagePb.key + .filter(_.nonEmpty) + .flatMap(raw => parseJson(raw).toOption) + .flatMap(_.asString) + assertTrue( + pbKeyDecoded.contains("a\"b"), + pbKeyDecoded == keyInJsonView(msg) + ) ?? s"pbKeyDecoded=$pbKeyDecoded jsonView=${keyInJsonView(msg)}" + }, + test("properties round-trip into the JSON view") { + val serialized = serialize(message(key = "k", properties = Map("a" -> "A", "b" -> "B"))) + val json = parseJson(serialized.messageAsJsonOmittingValue).toOption.get + val props = json.hcursor.downField("properties").as[Map[String, String]].toOption + assertTrue(props.contains(Map("a" -> "A", "b" -> "B"))) + }, + test("an eventTime of 0 is omitted from the JSON view") { + val json = parseJson(serialize(message(key = "k", eventTime = 0L)).messageAsJsonOmittingValue).toOption.get + assertTrue(json.hcursor.downField("eventTime").as[Long].toOption.isEmpty) + }, + test("a non-zero eventTime is retained") { + val json = parseJson(serialize(message(key = "k", eventTime = 1234L)).messageAsJsonOmittingValue).toOption.get + assertTrue(json.hcursor.downField("eventTime").as[Long].toOption.contains(1234L)) + } + ) diff --git a/server/src/test/scala/consumer/session_runner/messageIdStartFromTest.scala b/server/src/test/scala/consumer/session_runner/messageIdStartFromTest.scala new file mode 100644 index 000000000..07b8f1c53 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/messageIdStartFromTest.scala @@ -0,0 +1,143 @@ +package consumer.session_runner + +import org.apache.pulsar.client.api.PulsarClient +import org.apache.pulsar.client.impl.MessageIdImpl +import zio.test.* + +import java.util.concurrent.TimeUnit +import scala.util.Try + +/** Starting from a specific MESSAGE ID, across the physical topics one session covers. + * + * Regression context: a session's topic vector is built by concatenating every enabled target's + * resolved topics, and TWO TARGETS MAY SELECT THE SAME TOPIC - which is a supported configuration + * (each target has its own consumer, its own filters and its own colouring). The concatenation + * therefore named one physical topic twice, the duplicate pushed the vector past the + * single-topic fast path, and the multi-topic lookup read the SAME physical message once per name + * and threw "Multiple messages found for the same message id" on a perfectly valid session. + * + * [[resolveMessageIdAcrossTopics]] is pure - the broker sits behind a `String => Option[M]` lookup + * - so every shape is driven here with a plain lambda: no broker, and no mock. + */ +object messageIdStartFromTest extends ZIOSpecDefault: + + private val orders = "persistent://public/default/orders" + private val payments = "persistent://public/default/payments" + + /** A broker holding `messageOf`, plus a count of how many topics were actually asked. */ + private def broker(messageOf: Map[String, String]): (String => Option[String], () => Int) = + var reads = 0 + val lookup = (topicFqn: String) => + reads += 1 + messageOf.get(topicFqn) + (lookup, () => reads) + + private val messageId = new MessageIdImpl(7L, 3L, 0) + + /** Consumers as the seek planner sees them: just their topic. */ + private def consumersOn(topicFqns: String*): Vector[String] = topicFqns.toVector + + /** A real client aimed at a closed port, so an operational failure is a real one rather than a + * mock's idea of one. Two-second timeouts keep the suite quick. */ + private def withOfflineClient[A](f: PulsarClient => A): A = + val client = PulsarClient.builder.serviceUrl("pulsar://127.0.0.1:1").operationTimeout(2, TimeUnit.SECONDS).build + try f(client) + finally Try(client.close()) + + /** WHERE EACH CONSUMER STARTS once the message has been found. + * + * The whole session used to be seeked by the message's PUBLISH TIME as soon as it covered more + * than one topic - including the topic the id actually belongs to. "Start from this message" + * therefore became "start from this millisecond" on the very topic the user picked it from, and + * every earlier message sharing that millisecond (or that producer batch) came with it. + */ + private val seekSuite = suite("which position each topic is seeked to")( + test("THE TOPIC THAT OWNS THE ID IS SEEKED TO THE EXACT MESSAGE, not to its millisecond") { + val seeks = messageIdSeeks(consumersOn(orders, payments), identity, orders, messageId, 1_700_000_000_123L) + assertTrue(seeks.toMap.apply(orders) == MessageIdSeek.ById(messageId)) ?? s"seeks=$seeks" + }, + test("every OTHER topic is seeked by publish time - the only cross-topic position there is") { + val seeks = messageIdSeeks(consumersOn(orders, payments), identity, orders, messageId, 1_700_000_000_123L) + assertTrue(seeks.toMap.apply(payments) == MessageIdSeek.ByPublishTime(1_700_000_000_123L)) ?? s"seeks=$seeks" + }, + test("both consumers of a topic reached by two targets get the exact id") { + // Each enabled target has its own consumer on the topic, and both own the message. + val seeks = messageIdSeeks(Vector("a" -> orders, "b" -> orders, "c" -> payments), _._2, orders, messageId, 500L) + assertTrue( + seeks.count((_, seek) => seek == MessageIdSeek.ById(messageId)) == 2, + seeks.count((_, seek) => seek == MessageIdSeek.ByPublishTime(500L)) == 1 + ) ?? s"seeks=$seeks" + }, + test("a single-topic session is seeked by id and never by time") { + val seeks = messageIdSeeks(consumersOn(orders), identity, orders, messageId, 500L) + assertTrue(seeks.forall((_, seek) => seek == MessageIdSeek.ById(messageId))) + } + ) + + /** "NOT FOUND" IS A DIAGNOSIS OF THE USER'S INPUT, so it must not be what the server says when + * the fault is its own. Every operational failure used to collapse into the same `None` as a + * genuine absence. + */ + private val honestySuite = suite("a lookup that could not be made is not a lookup that found nothing")( + test("A MESSAGE ID THAT CANNOT BE PARSED is rejected as invalid, not reported as 'not found'") { + val result = withOfflineClient(client => Try(getMessageById(client, orders, Array[Byte](1, 2, 3)))) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + result.failed.toOption.exists(err => !err.getMessage.toLowerCase.contains("not found")) + ) ?? s"result=$result" + }, + test("A BROKER THAT CANNOT BE REACHED FAILS THE LOOKUP instead of answering 'not found'") { + // Reported as absence, this told the user their message id was wrong - and on a + // multi-topic session it let an unreachable topic be silently skipped, so the session + // was positioned from whichever topics happened to answer. + val realId = new MessageIdImpl(1L, 0L, -1).toByteArray + val result = withOfflineClient(client => Try(getMessageById(client, orders, realId))) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + result.failed.toOption.exists(_.getMessage.contains(orders)) + ) ?? s"result=$result" + } + ) + + private val dedupSuite = suite("looking the id up across a session's topics")( + test("TWO TARGETS ON THE SAME TOPIC resolve one message, not a duplicate of it") { + // THE regression. The same physical topic named twice is one message, and asking the + // broker for it twice is both wrong and wasteful. + val (lookup, reads) = broker(Map(orders -> "m1")) + val got = Try(resolveMessageIdAcrossTopics(Vector(orders, orders), lookup)) + assertTrue(got.toOption.flatten == Some("m1"), reads() == 1) ?? + s"the same topic reached by two targets must resolve once, got $got after ${reads()} lookups" + }, + test("three targets on one topic and one on another still resolve the single hit") { + val (lookup, reads) = broker(Map(orders -> "m1")) + val got = Try(resolveMessageIdAcrossTopics(Vector(orders, payments, orders, orders), lookup)) + assertTrue(got.toOption.flatten == Some("m1"), reads() == 2) ?? s"got $got after ${reads()} lookups" + }, + test("a message id that no topic holds resolves to nothing") { + val (lookup, _) = broker(Map.empty) + assertTrue(resolveMessageIdAcrossTopics(Vector(orders, payments), lookup).isEmpty) + }, + test("one topic holding it resolves to that message") { + val (lookup, _) = broker(Map(payments -> "m2")) + assertTrue(resolveMessageIdAcrossTopics(Vector(orders, payments), lookup) == Some("m2")) + }, + test("two GENUINELY DIFFERENT topics answering is still ambiguous and still refused") { + // The dedup must not paper over the real ambiguity it was added around: a message id is + // only unique within one topic, so two distinct topics answering means the session + // cannot know which message the user meant. + val (lookup, _) = broker(Map(orders -> "m1", payments -> "m2")) + val got = Try(resolveMessageIdAcrossTopics(Vector(orders, payments), lookup)) + assertTrue( + got.isFailure, + got.failed.toOption.exists(_.getMessage.contains("Multiple messages")) + ) ?? s"two different topics holding the id must be refused, got $got" + }, + test("no topics at all resolve to nothing without asking the broker") { + val (lookup, reads) = broker(Map(orders -> "m1")) + assertTrue(resolveMessageIdAcrossTopics(Vector.empty, lookup).isEmpty, reads() == 0) + } + ) + + def spec = suite(this.getClass.toString)(dedupSuite, seekSuite, honestySuite) diff --git a/server/src/test/scala/consumer/session_runner/nonPersistentTopicsTest.scala b/server/src/test/scala/consumer/session_runner/nonPersistentTopicsTest.scala new file mode 100644 index 000000000..4c016a0b0 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/nonPersistentTopicsTest.scala @@ -0,0 +1,130 @@ +package consumer.session_runner + +import _root_.consumer.start_from.* +import zio.test.* + +import java.time.Instant + +/** Start-from on NON-PERSISTENT topics. + * + * A non-persistent topic stores nothing: no backlog, no history, no entry to address. Every + * history-based position is therefore unsatisfiable on one, and the broker says so - `examineMessage` + * answers HTTP 405 ("Examine messages on a non-persistent topic is not allowed"). + * + * That 405 used to be swallowed: the admin call sits inside a `Try(...).toOption`, so the refusal + * became a `None` and the session quietly fell back to seeking earliest or latest. The user asked + * for a position in history and silently got a different one - the worst kind of failure, because + * the session looks like it worked. + * + * So the decision is made from the topic FQN, BEFORE any consumer is seeked, and it is a validation + * error rather than an incidental exception. Everything here is pure: the rule is a function of the + * mode and the resolved topic names. + */ +object nonPersistentTopicsTest extends ZIOSpecDefault: + + private val persistentTopic = "persistent://public/default/orders" + private val nonPersistentTopic = "non-persistent://public/default/telemetry" + private val nonPersistentPartition = "non-persistent://public/default/telemetry-partition-3" + + private val historyModes: Vector[ConsumerSessionStartFrom] = Vector( + EarliestMessage(), + NthMessageAfterEarliest(n = 10), + NthMessageBeforeLatest(n = 10), + MessageId(messageIdBytes = Array[Byte](8, 1)), + DateTime(dateTime = Instant.ofEpochSecond(1_700_000_000L)), + RelativeDateTime(value = 1, unit = DateTimeUnit.Hour, isRoundedToUnitStart = false), + ApproximateEntryPosition(fraction = 0.5), + ApproximatePublishTimePosition(fraction = 0.5) + ) + + private val detectionSuite = suite("detecting a topic with no history")( + test("a non-persistent topic is recognised by its scheme, without asking the broker") { + assertTrue( + isNonPersistentTopic(nonPersistentTopic), + isNonPersistentTopic(nonPersistentPartition) + ) + }, + test("a persistent topic is not mistaken for one") { + assertTrue( + !isNonPersistentTopic(persistentTopic), + !isNonPersistentTopic("persistent://public/default/orders-partition-0"), + // The substring appears inside the name, not as the scheme. + !isNonPersistentTopic("persistent://public/default/non-persistent-audit") + ) + }, + test("an unqualified name is treated as persistent rather than silently rejected") { + // Rejecting a session is the loud outcome, so it must never be triggered by a name shape + // this function did not expect. + assertTrue(!isNonPersistentTopic("public/default/orders"), !isNonPersistentTopic("")) + } + ) + + private val modeSuite = suite("which modes need a history")( + test("every mode except 'latest message' needs a retained history") { + val wrong = historyModes.filterNot(startFromNeedsRetainedHistory) + assertTrue(wrong.isEmpty) ?? s"these modes cannot work without a backlog but claim they can: $wrong" + }, + test("'earliest message' counts as needing a history, although a seek to it would 'work'") { + // On a non-persistent topic a seek to earliest silently behaves as "from now". Accepting + // it would mean answering a request for the start of the topic with the live tail. + assertTrue(startFromNeedsRetainedHistory(EarliestMessage())) + }, + test("'latest message' is the one position a non-persistent topic can honour") { + assertTrue(!startFromNeedsRetainedHistory(LatestMessage())) + } + ) + + private val rejectionSuite = suite("rejecting what cannot work")( + test("a history position over only non-persistent topics is rejected, naming the mode and the reason") { + val reasons = historyModes.map(mode => mode -> startFromRejectionReason(mode, Vector(nonPersistentTopic))) + val accepted = reasons.collect { case (mode, None) => mode } + val unclear = reasons.collect { + case (mode, Some(reason)) if !reason.contains("non-persistent") || !reason.contains(mode.getClass.getSimpleName) => mode -> reason + } + assertTrue(accepted.isEmpty, unclear.isEmpty) ?? + s"silently accepted: $accepted; rejected without a clear reason: $unclear" + }, + test("the reason names the topics that caused it") { + val reason = startFromRejectionReason(EarliestMessage(), Vector(nonPersistentTopic)) + assertTrue(reason.exists(_.contains(nonPersistentTopic))) + }, + test("'latest message' is accepted on non-persistent topics") { + assertTrue(startFromRejectionReason(LatestMessage(), Vector(nonPersistentTopic, nonPersistentPartition)).isEmpty) + }, + test("a mixed session is NOT rejected - one persistent topic is enough to have a history") { + // Failing the whole session because one of its topics is non-persistent would make a + // history position unusable on any session that happens to include a live topic. + val stillAccepted = historyModes.filter(mode => startFromRejectionReason(mode, Vector(nonPersistentTopic, persistentTopic)).isEmpty) + assertTrue(stillAccepted == historyModes) ?? + s"a mixed session must keep working; these were rejected: ${historyModes.diff(stillAccepted)}" + }, + test("a session over persistent topics only is never rejected") { + val rejected = historyModes.filter(mode => startFromRejectionReason(mode, Vector(persistentTopic)).isDefined) + assertTrue(rejected.isEmpty) + }, + test("a session that resolved to no topics at all is left to the emptiness check") { + // ConsumerSessionRunner.make already rejects that, with a message about the target - + // reporting it here as a non-persistent problem would be misleading. + assertTrue(startFromRejectionReason(EarliestMessage(), Vector.empty).isEmpty) + } + ) + + private val splitSuite = suite("what a mixed session does")( + test("the history position applies to the persistent topics only") { + val (history, liveOnly) = splitByRetainedHistory(Vector(persistentTopic, nonPersistentTopic, nonPersistentPartition), identity) + assertTrue(history == Vector(persistentTopic), liveOnly == Vector(nonPersistentTopic, nonPersistentPartition)) + }, + test("the split is over the physical topic of each consumer, not the session's selector") { + // A partitioned non-persistent topic resolves to one consumer per partition, and each + // has to be recognised on its own. + val consumers = Vector("a" -> persistentTopic, "b" -> nonPersistentPartition) + val (history, liveOnly) = splitByRetainedHistory(consumers, _._2) + assertTrue(history.map(_._1) == Vector("a"), liveOnly.map(_._1) == Vector("b")) + }, + test("a session with no non-persistent topic keeps every consumer on the history path") { + val (history, liveOnly) = splitByRetainedHistory(Vector(persistentTopic, persistentTopic), identity) + assertTrue(history.size == 2, liveOnly.isEmpty) + } + ) + + def spec = suite(this.getClass.toString)(detectionSuite, modeSuite, rejectionSuite, splitSuite) diff --git a/server/src/test/scala/consumer/session_runner/pauseLinearizabilityTest.scala b/server/src/test/scala/consumer/session_runner/pauseLinearizabilityTest.scala new file mode 100644 index 000000000..adf8bf838 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/pauseLinearizabilityTest.scala @@ -0,0 +1,250 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong} +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch} + +/** WHEN PAUSE RETURNS, NOTHING MAY STILL BE ON ITS WAY TO THE CLIENT. + * + * The limiter's drain loop reads the pause flag once per item and then calls the delivery, so a + * Pause landing between the read and the call was answered OK while that item went on to be sent + * and ACKNOWLEDGED - a delivery after the pause the user was told had taken effect. And the + * session's own `pause` paused every TARGET first and only then stopped the drainer, so on a wide + * session the queued backlog kept draining for the whole length of that (up to 2,000 consumer) + * walk. + * + * Both are pinned here with real threads and latches - no sleeps: the delivery is frozen INSIDE + * the send, and the question asked of Pause is simply whether it came back before that send did. + */ +object pauseLinearizabilityTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/cs-pause-linearizability" + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + /** Spin until `condition` holds. Bounded so a regression fails loudly instead of hanging. */ + private def spinUntil(what: String)(condition: => Boolean): Unit = + val deadline = java.lang.System.nanoTime() + 30_000_000_000L + while !condition && java.lang.System.nanoTime() < deadline do Thread.onSpinWait() + if !condition then throw new AssertionError(s"timed out waiting for $what") + + /** Wait until `t` has SETTLED: either parked (the pause is waiting for the delivery boundary) or + * finished (the pause returned without waiting - the defect). Both are reached promptly, so + * this answers quickly whichever way the code behaves. */ + private def awaitSettled(t: Thread): Thread.State = + spinUntil(s"thread ${t.getName} to settle") { + t.getState match + case Thread.State.WAITING | Thread.State.TIMED_WAITING | Thread.State.TERMINATED => true + case _ => false + } + t.getState + + /** A consumer that records, at the moment its permits are paused, whether the session's delivery + * pacer had ALREADY been stopped. */ + private final class OrderRecordingConsumer(observe: () => Boolean): + val drainerWasStoppedFirst = ConcurrentLinkedQueue[java.lang.Boolean]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => "cs-pause-linearizability-0" + case "isConnected" => java.lang.Boolean.TRUE + case "pause" => + drainerWasStoppedFirst.add(java.lang.Boolean.valueOf(observe())) + null + case "resume" => null + case "acknowledgeAsync" => CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def session(consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionRunner = + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.startAcceptingNewMessages() + val target = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumers.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumers.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = consumers, + pauseArbiters = consumers.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + ConsumerSessionRunner( + sessionName = "cs-pause-linearizability", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + def spec = suite("pause is linearizable against an in-flight delivery")( + test("P2.1: Pause must not return while a delivery it already admitted is still sending") { + // The window: the drain loop evaluates `!paused`, Pause sets the flag and answers the + // client, and only THEN does the already-admitted item run - so a message was sent and + // acknowledged after the user was told the session had stopped. + val pending = ConcurrentLinkedQueue[Runnable]() + val sent = ConcurrentLinkedQueue[Int]() + val deliveryStarted = AtomicBoolean(false) + val releaseDelivery = CountDownLatch(1) + + val core = DeliveryRateLimiterCore[Int](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (_, task) => { pending.add(task); () }, + // The send/ACK, frozen open: everything the browser and the broker learn about this + // message happens on the far side of the latch. + process = i => + deliveryStarted.set(true) + releaseDelivery.await() + sent.add(i) + () + , + holdPermits = () => true, + releasePermits = () => true + ) + core.setRate(100) + + (1 to 3).foreach(limiter.offer(_)) + val armedTick = pending.poll() + + val drainThread = worker("p21-drain")(armedTick.run()) + drainThread.start() + spinUntil("the drain to enter its first delivery")(deliveryStarted.get) + + val sentWhenPauseReturned = AtomicInteger(-1) + val pauseThread = worker("p21-pause") { + limiter.pauseDraining() + sentWhenPauseReturned.set(sent.size) + } + pauseThread.start() + // Either it parks at the delivery boundary (correct) or it runs straight through and + // terminates (the defect). Both settle at once, so nothing here is timing-sensitive. + val pauseState = awaitSettled(pauseThread) + + releaseDelivery.countDown() + drainThread.join(30_000) + pauseThread.join(30_000) + + assertTrue( + pauseState != Thread.State.TERMINATED, // Pause waited rather than answering over an open send + sentWhenPauseReturned.get == 1, // ...and the send it waited for had completed + sent.size == 1 // ...while the rest of the batch never left the queue + ) ?? s"pauseState=$pauseState sentWhenPauseReturned=${sentWhenPauseReturned.get} sent=${sent.size} queued=${core.queuedCount}" + }, + test("P2.1: a delivery may not START once Pause has returned") { + // The same window seen from the other side: an item the drain had NOT yet admitted must + // be refused outright, however late the drain thread gets around to it. + val pending = ConcurrentLinkedQueue[Runnable]() + val sent = ConcurrentLinkedQueue[Int]() + val core = DeliveryRateLimiterCore[Int](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (_, task) => { pending.add(task); () }, + process = i => { sent.add(i); () }, + holdPermits = () => true, + releasePermits = () => true + ) + core.setRate(100) + (1 to 3).foreach(limiter.offer(_)) + val armedTick = pending.poll() + + limiter.pauseDraining() + armedTick.run() + + assertTrue(sent.isEmpty, core.queuedCount == 3) + }, + test("P2.1: the delivery budget's own pause, made from INSIDE a delivery, must not wait for itself") { + // `sendPrepared` pauses the drain the moment the last budgeted message is sent - on the + // delivery thread, inside the delivery. A quiesce wait that did not recognise its own + // caller would deadlock the session exactly when its budget ran out. + val pending = ConcurrentLinkedQueue[Runnable]() + val sent = ConcurrentLinkedQueue[Int]() + val holder = scala.collection.mutable.ArrayBuffer[DeliveryRateLimiter[Int]]() + val core = DeliveryRateLimiterCore[Int](nowMs = () => 0L) + val limiter = DeliveryRateLimiter[Int]( + core = core, + schedule = (_, task) => { pending.add(task); () }, + process = i => + sent.add(i) + if i == 2 then holder.head.pauseDraining() + () + , + holdPermits = () => true, + releasePermits = () => true + ) + holder += limiter + core.setRate(100) + (1 to 5).foreach(limiter.offer(_)) + + val tick = pending.poll() + val tickThread = worker("p21-budget")(tick.run()) + tickThread.start() + tickThread.join(30_000) + + assertTrue( + !tickThread.isAlive, // it came back at all: the budget pause did not wait on itself + sent.size == 2, // and it stopped the line exactly at the message that spent the budget + core.queuedCount == 3 + ) ?? s"alive=${tickThread.isAlive} sent=${sent.size} queued=${core.queuedCount}" + }, + test("P2.1: the session's pause stops the DRAINER before it walks the targets") { + // `pause` paused every target first and only stopped the pacer in its `finally`, so on a + // 2,000-stream session the queued backlog went on being delivered and acknowledged for + // the whole length of that walk - after the user had asked for the session to stop. + val holder = scala.collection.mutable.ArrayBuffer[ConsumerSessionRunner]() + val probe = OrderRecordingConsumer(() => holder.head.deliveryRateLimiter.isDrainingPausedNow) + val runner = session(Map(topicFqn -> probe.consumer)) + holder += runner + + runner.pause() + + assertTrue( + probe.drainerWasStoppedFirst.size == 1, // vacuity guard: the consumer really was paused + probe.drainerWasStoppedFirst.peek == java.lang.Boolean.TRUE + ) ?? s"observations=${probe.drainerWasStoppedFirst.toArray.mkString(",")}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/replayBoundaryTest.scala b/server/src/test/scala/consumer/session_runner/replayBoundaryTest.scala new file mode 100644 index 000000000..7d0a707ba --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/replayBoundaryTest.scala @@ -0,0 +1,596 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, MessageId as PulsarMessageId, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** THE GUARANTEED EXACT-REPLAY CONTRACT, pinned at the pause boundary (owner decision 2026-08-09). + * + * Guaranteed is an exact replay: deliver everything recorded up to the moment Play was pressed + * (of what retention still holds), in strict key order, then AUTO-PAUSE with a caught-up signal. + * No live phase, no stall, no held tail, no 30 s give-up. Resume EXTENDS the boundary to now - + * one rule for manual pause and auto-pause - and the delta (itself immutable by then) replays + * exactly: the un-emitted remainder and the delta merge in one heap. A message that arrived past + * the boundary mid-replay is handed back (its consumer paused) and belongs to the NEXT chunk; it + * must not spend budget, must not enter the delivery memo, and must not be emitted. + * + * These tests were written BEFORE the implementation, per the owner's confirm-first rule, and the + * Guaranteed ones FAIL against the previous hold-forever code (which kept waiting on every stream + * forever and never signalled). The Best-effort test is a PIN - it passes before and after. + * + * Offline like the other lifecycle suites: proxy consumers, hand-built `MessageImpl`s, the real + * listener, ordering layer, runner and delivery pump. The recorded ends are the test's to move, + * which is how "messages were published while the session was paused" is stated rather than + * waited for. + */ +object replayBoundaryTest extends ZIOSpecDefault: + + private val consumerName = "cs-replay-0" + private def topic(i: Int): String = s"persistent://public/default/cs-replay-$i" + private def sid(i: Int): String = startFromStreamId(consumerName, topic(i)) + + /** A connected consumer whose RECORDED END the test controls. `lastIds` is what + * `getLastMessageIds` answers - moving it forward is the test's way of publishing while the + * session is not looking. Acks, nacks and pause/resume calls are recorded, and so is every + * BOUNDARY read of the last ids: the first Play must consume the boundaries decided at + * session build without asking the broker for ends at all, and only a read counter can pin + * that. The counter deliberately excludes reads made on the shared maintenance thread - + * the post-caught-up indicator refinement and the stall diagnostic legitimately read ends + * there, asynchronously, and they are not the boundary capture the pin is about. */ + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + val pauseCalls = ConcurrentLinkedQueue[String]() + val boundaryEndReads = AtomicLong(0) + /** Armed to make the next BOUNDARY `getLastMessageIds` fail the way an unreachable broker + * does (the maintenance thread's refinement reads stay unaffected - they are best-effort + * by design and must not be what the loud-failure pin observes). */ + @volatile var failLastIds: Boolean = false + @volatile var lastIds: java.util.List[PulsarMessageId] = java.util.Collections.emptyList() + + def recordedEnd(entryId: Long): Unit = + lastIds = java.util.List.of(new MessageIdImpl(1L, entryId, -1)) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "getLastMessageIds" => + if !Thread.currentThread.getName.startsWith("consumer-session-maintenance") then + boundaryEndReads.incrementAndGet() + if failLastIds then + throw new IllegalStateException(s"the broker would not answer getLastMessageIds for $topicFqn") + lastIds + case "pause" => + pauseCalls.add("pause") + null + case "resume" => + pauseCalls.add("resume") + null + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val frames = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + override def onNext(value: consumerPb.ResumeResponse): Unit = + frames.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + def received: Vector[consumerPb.ResumeResponse] = frames.asScala.toVector + def deliveredMessages: Vector[consumerPb.Message] = received.flatMap(_.messages) + /** Rows on screen: frames carrying a real message (a count-only placeholder has no id). */ + /** Rows on screen by partition key. The converter serializes the key as JSON, so the pb + * field carries `"a-100"` quotes and all - stripped here so assertions read plainly. */ + def deliveredKeys: Vector[String] = + deliveredMessages.flatMap(_.key).map(_.stripPrefix("\"").stripSuffix("\"")) + def caughtUpFrames: Vector[consumerPb.ConsumerStats] = + received.flatMap(_.consumerStats).filter(_.replayCaughtUp) + + /** A real Guaranteed session over proxy consumers: real listener, real ordering layer with + * RECORDED ENDS, real runner and delivery pump. The merge clock is injected. + * + * `endsAtPlay` is the boundary the session DECIDED at build time - what `handleStartFrom` + * armed the ordering layer with. `brokerEnds` is what the BROKER answers when asked for the + * last ids (`getLastMessageIds`, the resume re-capture's source); it defaults to the same + * values, which is the history-seek case. A LIVE-EDGE seek is the case where the two differ: + * the decided boundary is deliberately EMPTY (nothing to replay - the cursor sits at the + * live edge) while the broker still reports the backlog's real end behind that cursor. The + * old fixture conflated the two, so the re-capture path always read ends equal to the + * decided ones and its overwrite of a live-edge boundary was invisible (e2e CS-DM-R3B). + */ + private final class Fixture( + sessionName: String, + topicCount: Int, + endsAtPlay: Map[Int, Long], + guaranteed: Boolean = true, + brokerEnds: Option[Map[Int, Long]] = None + ): + var monoMs: Long = 0L + val pool: ConsumerSessionContextPool = ConsumerSessionContextPool() + val listener: ConsumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + + val consumers: Vector[RecordingConsumer] = Vector.tabulate(topicCount)(i => RecordingConsumer(topic(i))) + brokerEnds.getOrElse(endsAtPlay).foreach((i, entryId) => if entryId >= 0 then consumers(i).recordedEnd(entryId)) + + private def streamsAtPlay: Vector[StartFromStream] = + Vector.tabulate(topicCount) { i => + val end = endsAtPlay.get(i).filter(_ >= 0).map(e => EntryPosition(1L, e, -1, 1)).getOrElse(EntryPosition.empty) + StartFromStream(sid(i), end) + } + + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector.tabulate(topicCount)(sid), + drainedAtStart = streamsAtPlay.filter(_.lastAtStart == EntryPosition.empty).map(_.id).toSet, + discard = StartFromDiscard.shared(0), + nowMs = () => monoMs, + policy = if guaranteed then OrderingPolicy.GuaranteedOnly else OrderingPolicy.BestEffortOnly, + graceMs = 500L + ) + listener.startFromOrdering = new StartFromOrdering[HeldMessage]( + Some(merge), + streamsAtPlay.map(s => s.id -> s).toMap + ) + + private val consumersByTopic: Map[String, Consumer[Array[Byte]]] = + Vector.tabulate(topicCount)(i => topic(i) -> consumers(i).consumer).toMap + + val pauseArbiters: Map[String, ConsumerPauseArbiter] = + consumersByTopic.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)) + listener.pauseArbiters = pauseArbiters.map((fqn, arbiter) => startFromStreamId(consumerName, fqn) -> arbiter) + + val target: ConsumerSessionTargetRunner = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumersByTopic.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumersByTopic.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumersByTopic, + pauseArbiters = pauseArbiters, + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + val runner: ConsumerSessionRunner = ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty), + messageDeliveryOrder = if guaranteed then MessageDeliveryOrder.Guaranteed else MessageDeliveryOrder.BestEffort + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + def deliver(partition: Int, key: String, publishTime: Long, entryId: Long): Unit = + listener.received(consumers(partition).consumer, message(topic(partition), key, publishTime, entryId)) + + def acknowledged: Vector[String] = consumers.flatMap(_.acknowledged.asScala.toVector) + def handedBack: Vector[String] = consumers.flatMap(_.handedBack.asScala.toVector) + def close(): Unit = + Try(runner.stop()) + () + + def spec = suite(this.getClass.toString)( + test("PIN, Best effort: a burst published while paused is handed back and appears after resume") { + // The Best-effort side of the pause boundary, unchanged by the replay redesign: a + // paused session rejects what arrives (handed back for redelivery), and the burst is + // delivered - exactly once - when the redeliveries land after resume. This PASSED + // before the redesign and must keep passing; it is the control for the Guaranteed + // tests below. + // + // LISTENER TIER, deliberately: a runner-level best-effort session routes deliveries + // through the rate limiter's queue and drains them on the session's own timer thread, + // so the assertions would race real time. Down here the gate, the merge, the nack and + // the redelivery are the production ones and every step is synchronous - exactly the + // shape liveDeliveryOrderSwitchTest pins the relax path with. + var monoMs = 0L + val delivered = ConcurrentLinkedQueue[String]() + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => + delivered.add(msg.getKey) + () + )) + listener.startAcceptingNewMessages() + listener.startFromOrdering = new StartFromOrdering[HeldMessage]( + Some(GlobalSkipMerge[HeldMessage]( + streamIds = Vector(sid(0), sid(1)), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => monoMs, + policy = OrderingPolicy.BestEffortOnly, + graceMs = 500L + )), + Map.empty + ) + val consumers = Vector(RecordingConsumer(topic(0)), RecordingConsumer(topic(1))) + def deliver(partition: Int, key: String, publishTime: Long, entryId: Long): Unit = + listener.received(consumers(partition).consumer, message(topic(partition), key, publishTime, entryId)) + def deliveredKeys: Vector[String] = delivered.asScala.toVector + def handedBack: Vector[String] = consumers.flatMap(_.handedBack.asScala.toVector) + def acknowledged: Vector[String] = consumers.flatMap(_.acknowledged.asScala.toVector) + + deliver(0, "a-100", 100L, 0L) + deliver(1, "b-200", 200L, 0L) + monoMs += 501L + listener.sweepStartFromStall() // the timer half of the residence bound + val beforePause = deliveredKeys + + listener.stopAcceptingNewMessages() // the pause gate, exactly as target.pause closes it + deliver(0, "a-300", 300L, 1L) // the burst lands while paused + deliver(1, "b-400", 400L, 1L) + val rejectedWhilePaused = handedBack + + listener.startAcceptingNewMessages() // and resume re-opens it + deliver(0, "a-300", 300L, 1L) // the broker redelivers the burst + deliver(1, "b-400", 400L, 1L) + monoMs += 501L + listener.sweepStartFromStall() + + assertTrue( + beforePause == Vector("a-100", "b-200"), + rejectedWhilePaused.sorted == Vector("a-300", "b-400"), + deliveredKeys == Vector("a-100", "b-200", "a-300", "b-400"), + deliveredKeys.distinct == deliveredKeys, // exactly once each + acknowledged.sorted == Vector("a-100", "a-300", "b-200", "b-400") + ) ?? s"beforePause=$beforePause rejected=$rejectedWhilePaused delivered=$deliveredKeys acked=$acknowledged" + }, + test("GUARANTEED IS AN EXACT REPLAY: the recorded range delivers to the last message, in key order, then AUTO-PAUSES") { + // Recorded at play: stream 0 holds entries 0..1, stream 1 holds entry 0. The replay + // must deliver ALL THREE in key order and then pause itself with the caught-up signal. + // Against the previous hold-forever code this FAILS: after stream 1 delivered its only + // message the barrier kept waiting for its next word forever, so a-300 was a held tail + // and no signal ever came. + val f = Fixture("cs-replay-boundary", topicCount = 2, endsAtPlay = Map(0 -> 1L, 1 -> 0L)) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) + f.deliver(0, "a-300", 300L, 1L) // stream 0's recorded end + f.deliver(1, "b-200", 200L, 0L) // stream 1's recorded end + + assertTrue( + // Everything recorded is delivered - the last message included, because every + // end is known - in strict key order. + observer.deliveredKeys == Vector("a-100", "b-200", "a-300"), + f.acknowledged.sorted == Vector("a-100", "a-300", "b-200"), + // The runner-side auto-pause: intake closed, consumers held. + !f.listener.isAcceptingNewMessages, + f.pauseArbiters.values.forall(_.heldReasons.contains(PauseReason.Boundary)), + // And the caught-up signal went out on the stats channel. + observer.caughtUpFrames.nonEmpty, + f.merge.heldCount == 0 + ) ?? (s"delivered=${observer.deliveredKeys} acked=${f.acknowledged} " + + s"gateOpen=${f.listener.isAcceptingNewMessages} held=${f.merge.heldCount} " + + s"frames=${observer.received.size} caughtUp=${observer.caughtUpFrames.size}") + finally f.close() + }, + test("NO LIVE PHASE: a past-boundary arrival mid-replay is handed back, its consumer paused, and it is NOT emitted") { + // Stream 1 races ahead: it delivers its recorded end AND a message past it while + // stream 0 is still replaying. The past-end message belongs to the NEXT chunk - handed + // back, consumer paused via the arbiter, no budget, no memo, no emission - and its + // arrival is itself proof stream 1's recorded range is done. Against the previous + // code this FAILS: the past-end message entered the merge and was emitted. + val f = Fixture("cs-replay-next-chunk", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(1, "b-200", 200L, 0L) // stream 1's recorded end + f.deliver(1, "b-400", 400L, 1L) // PAST the boundary: next chunk's business + val pastEndDisposition = (f.handedBack.toVector, observer.deliveredKeys) + val boundaryHeldEarly = f.pauseArbiters(topic(1)).heldReasons.contains(PauseReason.Boundary) + f.deliver(0, "a-100", 100L, 0L) // stream 0 finishes the replay + + assertTrue( + pastEndDisposition == (Vector("b-400"), Vector.empty), + boundaryHeldEarly, // the racing consumer was paused the moment it crossed + observer.deliveredKeys == Vector("a-100", "b-200"), + !observer.deliveredKeys.contains("b-400"), + f.acknowledged.sorted == Vector("a-100", "b-200"), // b-400 was never decided + observer.caughtUpFrames.nonEmpty, // the replay still completed and signalled + // The signal knows newer messages exist: the handed-back entry was seen and + // counted (approximately - "~N", entries not messages). + observer.caughtUpFrames.exists(_.replayNewerEntriesApprox >= 1L), + observer.caughtUpFrames.forall(_.replayBoundaryAtMs > 0L), + f.merge.heldCount == 0 + ) ?? (s"pastEnd=$pastEndDisposition delivered=${observer.deliveredKeys} " + + s"acked=${f.acknowledged} caughtUp=${observer.caughtUpFrames.map(s => (s.replayNewerEntriesApprox, s.replayBoundaryAtMs))}") + finally f.close() + }, + test("RESUME EXTENDS THE BOUNDARY: the delta replays exactly, then the session auto-pauses again") { + // Chunk 1 replays and auto-pauses. New messages land while paused (the recorded end + // moves; the prefetched one is handed back at the gate). Resume re-captures every + // stream's end and the SAME session replays the delta - the handed-back message + // redelivers and now belongs to the chunk, exactly once. Against the previous code + // this FAILS twice over: no auto-pause ever happened, and the delta message was held + // behind the forever-silent peer stream instead of completing a chunk. + val f = Fixture("cs-replay-extend", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) + f.deliver(1, "b-200", 200L, 0L) + val chunkOne = observer.deliveredKeys + val chunkOneCaughtUp = observer.caughtUpFrames.nonEmpty + + // Published while paused: stream 1's log grows; the prefetched copy is refused at + // the closed gate and handed back. + f.consumers(1).recordedEnd(1L) + f.deliver(1, "b-400", 400L, 1L) + val refusedWhilePaused = f.handedBack.contains("b-400") + + // Resume: the boundary extends to now, the Boundary holds lift, and the broker's + // redelivery of b-400 is now IN the chunk. + val second = RecordingObserver() + f.runner.resume(second, isDebug = false) + val boundaryLifted = f.pauseArbiters.values.forall(!_.heldReasons.contains(PauseReason.Boundary)) + f.deliver(1, "b-400", 400L, 1L) + + assertTrue( + chunkOne == Vector("a-100", "b-200"), + chunkOneCaughtUp, + refusedWhilePaused, + boundaryLifted, + second.deliveredKeys == Vector("b-400"), // the delta, exactly - stream 0 has no delta + second.caughtUpFrames.nonEmpty, // and the second chunk announces its own boundary + f.acknowledged.sorted == Vector("a-100", "b-200", "b-400"), + f.acknowledged.size == 3, // exactly once each, across the seam + !f.listener.isAcceptingNewMessages, // auto-paused again + f.merge.heldCount == 0 + ) ?? (s"chunkOne=$chunkOne delta=${second.deliveredKeys} acked=${f.acknowledged} " + + s"boundaryLifted=$boundaryLifted caughtUp2=${second.caughtUpFrames.size} " + + s"gateOpen=${f.listener.isAcceptingNewMessages}") + finally f.close() + }, + test("THE SEAM IS LOUD ON THE WIRE: a skewed delta row carries the per-message flag, and the stats carry the counter") { + // The wire half of the seam contract: the merge-tier semantics live in + // replaySeamTest; here the flagged emission is driven through the real target + // pipeline and the real response path, so the row the browser renders carries + // Message.delivered_out_of_order and the frame's stats carry + // ConsumerStats.replay_seam_violations. + val f = Fixture("cs-replay-seam-wire", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) + f.deliver(1, "b-200", 200L, 0L) // chunk 1 completes: max emitted key is 200 + val cleanRows = observer.deliveredMessages.map(_.deliveredOutOfOrder) + + f.consumers(0).recordedEnd(1L) // stream 0 grew while paused + val second = RecordingObserver() + f.runner.resume(second, isDebug = false) + f.deliver(0, "a-150", 150L, 1L) // the delta undercuts the emitted maximum + + assertTrue( + cleanRows == Vector(false, false), + second.deliveredKeys == Vector("a-150"), // delivered - never silently dropped + second.deliveredMessages.map(_.deliveredOutOfOrder) == Vector(true), // and loudly flagged + second.received.flatMap(_.consumerStats).exists(_.replaySeamViolations == 1L), + second.caughtUpFrames.nonEmpty + ) ?? (s"cleanRows=$cleanRows delta=${second.deliveredKeys} " + + s"flags=${second.deliveredMessages.map(_.deliveredOutOfOrder)} " + + s"seams=${second.received.flatMap(_.consumerStats).map(_.replaySeamViolations)}") + finally f.close() + }, + test("the caught-up wire shape: boundary and indicator ride only with the marker, and a seam alone still surfaces stats") { + // The pure response builder's contract for the new fields, pinned like the progress + // shape above it: stats appear when the replay has something to say, and the + // boundary/indicator fields never ride without the caught-up marker. + val caughtUp = resumeResponse( + Seq.empty, + Vector.empty, + None, + replayCaughtUp = true, + replayBoundaryAtMs = 1234L, + replayNewerEntriesApprox = 7L, + replayExcludedTopics = Seq("persistent://a/b/late"), + replayExcludedTopicCount = 3 + ) + val seamOnly = resumeResponse(Seq.empty, Vector.empty, None, replaySeamViolations = 2L) + val notCaughtUp = resumeResponse(Seq.empty, Vector.empty, None, replayBoundaryAtMs = 1234L, replayNewerEntriesApprox = 7L) + assertTrue( + caughtUp.consumerStats.exists(stats => + stats.replayCaughtUp && stats.replayBoundaryAtMs == 1234L && stats.replayNewerEntriesApprox == 7L + && stats.replayExcludedTopics == Seq("persistent://a/b/late") && stats.replayExcludedTopicCount == 3 + ), + seamOnly.consumerStats.exists(_.replaySeamViolations == 2L), + notCaughtUp.consumerStats.isEmpty // no marker, no fields, no stats at all + ) ?? s"caughtUp=${caughtUp.consumerStats} seamOnly=${seamOnly.consumerStats} notCaughtUp=${notCaughtUp.consumerStats}" + }, + test("the newer-entries arithmetic is entry-honest: same-ledger exact, cross-ledger a lower bound, nothing from nothing") { + // The pure half of the "~N newer" indicator (the broker refinement task feeds it). + assertTrue( + replayNewerEntriesApproxOf(EntryPosition(1, 5, -1, 1), EntryPosition(1, 9, -1, 1)) == 4L, + replayNewerEntriesApproxOf(EntryPosition(1, 5, -1, 1), EntryPosition(1, 5, -1, 1)) == 0L, + replayNewerEntriesApproxOf(EntryPosition(1, 5, -1, 1), EntryPosition(2, 2, -1, 1)) == 3L, // newest ledger's entries, at least + replayNewerEntriesApproxOf(EntryPosition(1, 5, -1, 1), EntryPosition(2, -1, -1, 1)) == 1L, // rolled but unreadable: at least one + replayNewerEntriesApproxOf(EntryPosition.empty, EntryPosition(1, 2, -1, 1)) == 3L, // nothing recorded, three entries now + replayNewerEntriesApproxOf(EntryPosition(1, 5, -1, 1), EntryPosition.empty) == 0L, // nothing there at all + replayNewerEntriesApproxOf(EntryPosition.empty, EntryPosition.empty) == 0L + ) + }, + test("AN EMPTY REPLAY IS AN INSTANT CAUGHT-UP: nothing recorded means the signal fires on Play, not never") { + // Latest + Guaranteed, and the single non-persistent stream, both land here: every + // stream's boundary is empty, so there is nothing to replay and the session must say + // so immediately instead of sitting silent. Against the previous code this FAILS: a + // Guaranteed session with nothing to deliver simply never spoke. + // + // STRENGTHENED (2026-08-09, e2e CS-DM-R3B): the DECIDED boundary is empty while the + // BROKER still reports a real backlog end - the actual Latest x Guaranteed shape on a + // non-empty topic, where the seek put the cursor at the live edge PAST the backlog. + // The old fixture handed the resume path consumers whose getLastMessageIds answered + // empty too, so the first-play re-capture read the same ends and its OVERWRITE of the + // live-edge boundary was invisible: this pin passed while the real session armed + // `waiting` over the undeliverable backlog range and waited forever. The first Play + // must CONSUME the create-time boundary - instant caught-up, and ZERO broker end + // reads (the boundaryEndReads assertion is what makes the pin bite). + val f = Fixture( + "cs-replay-empty", + topicCount = 2, + endsAtPlay = Map(0 -> -1L, 1 -> -1L), + brokerEnds = Some(Map(0 -> 5L, 1 -> 3L)) + ) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + assertTrue( + observer.caughtUpFrames.nonEmpty, + observer.deliveredKeys.isEmpty, + !f.listener.isAcceptingNewMessages, + f.pauseArbiters.values.forall(_.heldReasons.contains(PauseReason.Boundary)), + // THE NO-RE-READ ASSERTION: the first Play consumes the boundaries decided at + // session build; the broker is not asked for ends at all. + f.consumers.forall(_.boundaryEndReads.get == 0L) + ) ?? (s"frames=${observer.received.size} caughtUp=${observer.caughtUpFrames.size} delivered=${observer.deliveredKeys} " + + s"endReads=${f.consumers.map(_.boundaryEndReads.get)}") + finally f.close() + }, + test("MIXED LIVE-EDGE AND HISTORY: the history stream's backlog replays, the live-edge stream contributes nothing, caught-up fires") { + // One target seeks stream 1 into its history (recorded end entry 1 - two messages) + // while stream 0 sits at the live edge over a non-empty backlog (decided boundary + // EMPTY, broker end entry 5). The replay is exactly stream 1's recorded range; the + // live-edge stream is never waited for and never read for ends. Against the + // re-capturing code this FAILS: the first Play overwrote stream 0's empty boundary + // with entry 5 - a range the cursor already sits past - so the barrier waited on it + // forever and no caught-up ever came. + val f = Fixture( + "cs-replay-mixed-live-edge", + topicCount = 2, + endsAtPlay = Map(0 -> -1L, 1 -> 1L), + brokerEnds = Some(Map(0 -> 5L, 1 -> 1L)) + ) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(1, "b-100", 100L, 0L) + f.deliver(1, "b-200", 200L, 1L) // stream 1's recorded end + assertTrue( + observer.deliveredKeys == Vector("b-100", "b-200"), + observer.caughtUpFrames.nonEmpty, + f.acknowledged.sorted == Vector("b-100", "b-200"), + !f.listener.isAcceptingNewMessages, + f.pauseArbiters.values.forall(_.heldReasons.contains(PauseReason.Boundary)), + f.consumers.forall(_.boundaryEndReads.get == 0L), + f.merge.heldCount == 0 + ) ?? (s"delivered=${observer.deliveredKeys} caughtUp=${observer.caughtUpFrames.size} " + + s"acked=${f.acknowledged} endReads=${f.consumers.map(_.boundaryEndReads.get)}") + finally f.close() + }, + test("A SECOND RESUME RE-CAPTURES: the live-edge stream's delta replays exactly, and only then is the broker asked for ends") { + // The other half of "create is the first Play": SUBSEQUENT resumes must re-capture - + // the cursor then genuinely sits at the previous chunk's boundary, so the freshly + // read ends describe a deliverable delta. Stream 0 starts at the live edge over a + // 6-entry backlog (instant caught-up, nothing shown - that is what Latest means); + // one message is recorded while auto-paused; the resume reads the new end and the + // delta replays, exactly once. Against the re-capturing code this FAILS at the first + // step already: no instant caught-up ever fires. + val f = Fixture( + "cs-replay-second-resume", + topicCount = 2, + endsAtPlay = Map(0 -> -1L, 1 -> -1L), + brokerEnds = Some(Map(0 -> 5L, 1 -> -1L)) + ) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + val instantCaughtUp = observer.caughtUpFrames.nonEmpty + val readsOnFirstPlay = f.consumers.map(_.boundaryEndReads.get).sum + + // Published while auto-paused: stream 0's log grows past the live edge. + f.consumers(0).recordedEnd(6L) + val second = RecordingObserver() + f.runner.resume(second, isDebug = false) + val readsOnSecondPlay = f.consumers.map(_.boundaryEndReads.get).sum + f.deliver(0, "a-600", 600L, 6L) // the delta - the one message past the old edge + + assertTrue( + instantCaughtUp, + readsOnFirstPlay == 0L, + readsOnSecondPlay > 0L, // the re-capture DID happen - later resumes extend + second.deliveredKeys == Vector("a-600"), + second.caughtUpFrames.nonEmpty, // and the extended chunk completes again + f.acknowledged == Vector("a-600"), + !f.listener.isAcceptingNewMessages, + f.merge.heldCount == 0 + ) ?? (s"instant=$instantCaughtUp reads=($readsOnFirstPlay, $readsOnSecondPlay) " + + s"delta=${second.deliveredKeys} caughtUp2=${second.caughtUpFrames.size} acked=${f.acknowledged}") + finally f.close() + }, + test("A FAILED RE-CAPTURE ON A LATER RESUME STILL FAILS THE RESUME LOUDLY") { + // The first Play consumes the create-time boundary and needs no broker; a LATER + // resume genuinely re-reads the ends, and a broker that will not answer must fail + // that resume with the topic named - never quietly arm a boundary it did not read. + val f = Fixture("cs-replay-recapture-fails", topicCount = 1, endsAtPlay = Map(0 -> 0L)) + try + val observer = RecordingObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) // the whole recorded range: instant chunk, auto-pause + val caughtUp = observer.caughtUpFrames.nonEmpty + + f.consumers(0).failLastIds = true + val second = Try(f.runner.resume(RecordingObserver(), isDebug = false)) + + assertTrue( + caughtUp, + second.isFailure, + second.failed.toOption.exists(_.getMessage.contains(topic(0))) + ) ?? s"caughtUp=$caughtUp second=$second" + finally f.close() + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/replayByteWatermarkTest.scala b/server/src/test/scala/consumer/session_runner/replayByteWatermarkTest.scala new file mode 100644 index 000000000..96ad68239 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/replayByteWatermarkTest.scala @@ -0,0 +1,439 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, MessageId as PulsarMessageId, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** THE BYTE WATERMARK UNDER THE REPLAY BARRIER - the server half of e2e CS-FC-5, made + * deterministic. + * + * The arrangement is CS-FC-5's, scaled to a test JVM: a GUARANTEED session with an unresolved + * counted cut (ExactCutThenGuaranteed, an unspendable skip budget), one HOT stream delivering fat + * payloads and one BLIND stream whose recorded range never arrives - so the barrier may not + * advance, nothing is ever emitted or discarded, and everything the hot stream delivers is HELD. + * Held bytes walk up to [[startFromMergePauseBytesAt]]'s injected stand-in, and the property + * under test is the LATCH: the crossing offer must leave the hot consumer paused (the arbiter's + * Merge reason) before it returns, the overshoot that still arrives is accepted but bounded by + * what the broker had in flight, and NOTHING that is not the watermark's own drain may lift the + * hold - not a second Play's User/Boundary release, not the delivery limiter's permit release, + * not a past-end arrival's nack, and never the give-up (which Guaranteed does not have). + * + * These pins were written while hunting CS-FC-5's intermittent unbounded admission (2 of 4 e2e + * runs let all 400 MiB through a 256 MiB cap). Every reconcile and its desired-set snapshot run + * under the session's ordering lock, desired can only shrink through dequeues the waiting barrier + * forbids, and the per-reason arbiter blocks cross-owner releases - each of those claims is + * pinned here deterministically, including the one real interleaving a latch can force (a second + * Play racing the crossing's hold). What stays e2e-only is the broker/client boundary: prefetch + * and reconnect redelivery can inflate what the BROKER dispatched, which no server-side latch + * decides. + */ +object replayByteWatermarkTest extends ZIOSpecDefault: + + private val consumerName = "cs-byte-watermark-0" + private def topic(i: Int): String = s"persistent://public/default/cs-byte-watermark-$i" + private def sid(i: Int): String = startFromStreamId(consumerName, topic(i)) + + /** A payload fat enough that the byte cap fires long before any count watermark could: the + * cap below is FOUR of these, against count watermarks of 1,000 per stream / 10,000 total. */ + private val payloadBytes: Int = 64 * 1024 + private val byteCap: Long = 4L * payloadBytes + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + private def awaitTrue(condition: => Boolean, boundMs: Long = 30_000L): Boolean = + val deadline = System.nanoTime() + boundMs * 1_000_000L + while !condition && System.nanoTime() < deadline do Thread.onSpinWait() + condition + + /** The blocked-thread barrier the backpressure race suite established: the racing thread's + * next step is entering a monitor this test holds shut, so BLOCKED is the only state it can + * reach, it always reaches it, and reaching it proves the interleaving. */ + private def awaitBlocked(t: Thread): Unit = + val deadline = System.nanoTime() + 30_000_000_000L + while t.getState != Thread.State.BLOCKED && System.nanoTime() < deadline do Thread.onSpinWait() + if t.getState != Thread.State.BLOCKED then + throw new AssertionError(s"the racing thread never queued on the lock; its state is ${t.getState}") + + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + val permitCalls = ConcurrentLinkedQueue[String]() + val isPaused = AtomicBoolean(false) + /** Armed to hold the caller INSIDE `consumer.pause()` - the crossing's hold mid-flight. */ + @volatile var pauseGate: Option[(CountDownLatch, CountDownLatch)] = None + @volatile var lastIds: java.util.List[PulsarMessageId] = java.util.Collections.emptyList() + + def recordedEnd(entryId: Long): Unit = + lastIds = java.util.List.of(new MessageIdImpl(1L, entryId, -1)) + + def resumeCalls: Int = permitCalls.asScala.count(_ == "resume") + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "getLastMessageIds" => lastIds + case "pause" => + pauseGate.foreach { (entered, release) => + entered.countDown() + release.await(60, TimeUnit.SECONDS) + () + } + permitCalls.add("pause") + isPaused.set(true) + null + case "resume" => + permitCalls.add("resume") + isPaused.set(false) + null + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val payload = Array.fill[Byte](payloadBytes)('x'.toByte) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(payload), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val frames = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + override def onNext(value: consumerPb.ResumeResponse): Unit = + frames.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + def received: Vector[consumerPb.ResumeResponse] = frames.asScala.toVector + def deliveredKeys: Vector[String] = + received.flatMap(_.messages).flatMap(_.key).map(_.stripPrefix("\"").stripSuffix("\"")) + def caughtUpFrames: Vector[consumerPb.ConsumerStats] = received.flatMap(_.consumerStats).filter(_.replayCaughtUp) + + /** CS-FC-5's shape over two proxy streams: an ExactCutThenGuaranteed merge with an unspendable + * budget, the injected byte cap, and recorded ends far past anything the test delivers - so + * the hot stream keeps queueing and the blind one keeps the barrier shut. */ + private final class Fixture(sessionName: String, hotEnd: Long = 1_000_000L, budget: Long = 10_000_000L): + var monoMs: Long = 0L + val pool: ConsumerSessionContextPool = ConsumerSessionContextPool() + val listener: ConsumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + + val hot = RecordingConsumer(topic(0)) + val blind = RecordingConsumer(topic(1)) + hot.recordedEnd(hotEnd) + blind.recordedEnd(1_000_000L) + + private val streams = Vector( + StartFromStream(sid(0), EntryPosition(1L, hotEnd, -1, 1)), + StartFromStream(sid(1), EntryPosition(1L, 1_000_000L, -1, 1)) + ) + + val discard: StartFromDiscard = StartFromDiscard.shared(budget) + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector(sid(0), sid(1)), + drainedAtStart = Set.empty, + discard = discard, + nowMs = () => monoMs, + pauseBytesAt = byteCap, + resumeBytesAt = byteCap / 4, + payloadBytesOf = held => Try(Option(held.message.getData).map(_.length.toLong).getOrElse(0L)).getOrElse(0L), + policy = OrderingPolicy.ExactCutThenGuaranteed, + graceMs = 500L + ) + listener.startFromOrdering = new StartFromOrdering[HeldMessage](Some(merge), streams.map(s => s.id -> s).toMap) + + private val consumersByTopic: Map[String, Consumer[Array[Byte]]] = + Map(topic(0) -> hot.consumer, topic(1) -> blind.consumer) + val pauseArbiters: Map[String, ConsumerPauseArbiter] = consumersByTopic.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)) + listener.pauseArbiters = pauseArbiters.map((fqn, arbiter) => startFromStreamId(consumerName, fqn) -> arbiter) + + val target: ConsumerSessionTargetRunner = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumersByTopic.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumersByTopic.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumersByTopic, + pauseArbiters = pauseArbiters, + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + val runner: ConsumerSessionRunner = ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty), + messageDeliveryOrder = MessageDeliveryOrder.Guaranteed + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + def hotArbiter: ConsumerPauseArbiter = pauseArbiters(topic(0)) + + def deliverHot(n: Int, fromEntry: Long = 0L): Unit = + (0 until n).foreach { i => + val entryId = fromEntry + i + listener.received(hot.consumer, message(topic(0), s"a-$entryId", 100L + entryId, entryId)) + } + + def close(): Unit = + Try(runner.stop()) + () + + def spec = suite(this.getClass.toString)( + test("THE BYTE CAP LATCHES ADMISSION SYNCHRONOUSLY: the crossing offer leaves the hot consumer paused before it returns") { + // Counts are unreachable (a handful of messages against 1,000 / 10,000 watermarks), + // so if anything pauses the stream it is the byte cap - CS-FC-5's discriminator. The + // hold must be applied INSIDE the crossing delivery (offer, mark, reconcile and the + // arbiter's Merge hold all run under the ordering lock before `received` returns), + // and later arrivals - the prefetch the pause cannot un-ask - are accepted and held + // without the hold moving. + // Every observation is SNAPSHOT before the assertion: assertTrue's smart diagnostics + // re-evaluate the expression tree after the finally, and close() legitimately holds + // the User reason on every arbiter on its way out. + val f = Fixture("cs-byte-latch") + try + f.listener.startAcceptingNewMessages() + f.deliverHot(3) + val pausedBeforeCap = f.hot.isPaused.get + f.deliverHot(1, fromEntry = 3L) // held bytes reach the cap exactly here + val pausedAtCrossing = f.hot.isPaused.get + val reasonsAtCrossing = f.hotArbiter.heldReasons + f.deliverHot(3, fromEntry = 4L) // the in-flight overshoot: accepted, still held + val pausedAfterOvershoot = f.hot.isPaused.get + val blindPaused = f.blind.isPaused.get + val held = (f.merge.heldCount, f.merge.heldBytesCount) + val remaining = f.discard.remaining + val decided = (f.hot.acknowledged.asScala.toVector, f.hot.handedBack.asScala.toVector) + assertTrue( + !pausedBeforeCap, + pausedAtCrossing, // the latch is synchronous with the crossing + reasonsAtCrossing.contains(PauseReason.Merge), + pausedAfterOvershoot, + held == (7, 7L * payloadBytes), // overshoot accepted - a watermark, not a wall + !blindPaused, // the stream that can unblock the barrier always runs + remaining == 10_000_000L, // nothing was claimed by the cut + decided == (Vector.empty, Vector.empty) // held, not decided + ) ?? (s"pausedBefore=$pausedBeforeCap pausedAt=$pausedAtCrossing reasons=$reasonsAtCrossing " + + s"held=$held remaining=$remaining blindPaused=$blindPaused decided=$decided") + finally f.close() + }, + test("NO GIVE-UP UNDER GUARANTEED: the blind stream outlives the best-effort window with the hold intact and the stall disclosed") { + // The deterministic version of CS-FC-5's 40-second fixpoint: hand the merge clock a + // whole give-up window and sweep. Guaranteed never abandons a stream, so nothing may + // drain, the hold may not lift, and no caught-up may be announced - the one thing the + // session says is the stall disclosure. + val f = Fixture("cs-byte-no-give-up") + try + f.listener.startAcceptingNewMessages() + f.deliverHot(4) + f.monoMs += startFromMergeStallWindowMs + 1_000L + f.listener.sweepStartFromStall() + f.listener.sweepStartFromStall() + val abandoned = f.listener.startFromOrdering.abandonedStreams + val warnings = f.listener.startFromOrdering.stallWarningCount + val pausedAfterWindow = f.hot.isPaused.get + val reasons = f.hotArbiter.heldReasons + val held = f.merge.heldCount + val remaining = f.discard.remaining + val acked = f.hot.acknowledged.asScala.toVector + assertTrue( + abandoned.isEmpty, + warnings >= 1L, // disclosed, not silent + pausedAfterWindow, + reasons.contains(PauseReason.Merge), + held == 4, + remaining == 10_000_000L, + acked.isEmpty + ) ?? s"abandoned=$abandoned warnings=$warnings paused=$pausedAfterWindow reasons=$reasons held=$held" + finally f.close() + }, + test("A SECOND PLAY IS NOT AN OWNER: releasing User and Boundary leaves the Merge hold standing") { + // The resume walk releases exactly the reasons it owns. With the byte cap's Merge + // hold standing, a second Play - generation bump, boundary re-capture, User and + // Boundary releases, gate re-open - must leave the hot consumer paused and the held + // set untouched. + val f = Fixture("cs-byte-second-play") + try + f.runner.resume(RecordingObserver(), isDebug = false) + f.deliverHot(4) + val resumesBeforeSecondPlay = f.hot.resumeCalls + f.runner.resume(RecordingObserver(), isDebug = false) + val pausedAfterSecondPlay = f.hot.isPaused.get + val reasons = f.hotArbiter.heldReasons + val resumesAfterSecondPlay = f.hot.resumeCalls + val held = f.merge.heldCount + val remaining = f.discard.remaining + assertTrue( + pausedAfterSecondPlay, + reasons.contains(PauseReason.Merge), + // The walk released User/Boundary it never held; with Merge standing the + // arbiter must not have resumed the consumer even once more. + resumesAfterSecondPlay == resumesBeforeSecondPlay, + held == 4, + remaining == 10_000_000L + ) ?? (s"paused=$pausedAfterSecondPlay reasons=$reasons " + + s"resumes=$resumesAfterSecondPlay (before=$resumesBeforeSecondPlay) held=$held") + finally f.close() + }, + test("THE LIMITER'S PERMIT RELEASE IS NOT AN OWNER: holding and releasing the Limiter reason leaves the Merge hold standing") { + // The delivery pacer's hooks land on the arbiter's Limiter reason - a full + // hold-then-release cycle, and the bare release its refusal rollback performs, must + // both leave a Merge-held consumer exactly where it was. + val f = Fixture("cs-byte-limiter-release") + try + f.listener.startAcceptingNewMessages() + f.deliverHot(4) + f.target.setPermitHold(true) + val heldDuringLimiter = f.hotArbiter.heldReasons + f.target.setPermitHold(false) // the ordinary release + val afterCycle = f.hotArbiter.heldReasons + f.target.setPermitHold(false) // the rollback's bare release, nothing held + val afterBareRelease = f.hotArbiter.heldReasons + val pausedAtEnd = f.hot.isPaused.get + val held = f.merge.heldCount + assertTrue( + heldDuringLimiter == Set(PauseReason.Merge, PauseReason.Limiter), + afterCycle == Set(PauseReason.Merge), + pausedAtEnd, + afterBareRelease == Set(PauseReason.Merge), + held == 4 + ) ?? (s"during=$heldDuringLimiter afterCycle=$afterCycle " + + s"afterBare=$afterBareRelease paused=$pausedAtEnd held=$held") + finally f.close() + }, + test("A PAST-END ARRIVAL'S NACK RELEASES NOTHING AND BYPASSES NO ACCOUNTING") { + // The hot stream's recorded end is small here, so a later entry is a NEXT-CHUNK + // arrival mid-hold: it is handed back and its consumer gains the Boundary reason - + // and that decision must neither lift the Merge hold, nor touch the held set, nor + // spend budget, nor acknowledge anything. + val f = Fixture("cs-byte-past-end", hotEnd = 10L) + try + f.listener.startAcceptingNewMessages() + f.deliverHot(4) + val heldBefore = (f.merge.heldCount, f.merge.heldBytesCount) + f.deliverHot(1, fromEntry = 11L) // strictly past entry 10: next chunk's business + val handedBack = f.hot.handedBack.asScala.toVector + val reasons = f.hotArbiter.heldReasons + val pausedAfterNack = f.hot.isPaused.get + val heldAfter = (f.merge.heldCount, f.merge.heldBytesCount) + val remaining = f.discard.remaining + val acked = f.hot.acknowledged.asScala.toVector + val caughtUp = f.runner.isReplayCaughtUpNow + assertTrue( + handedBack == Vector("a-11"), + reasons == Set(PauseReason.Merge, PauseReason.Boundary), + pausedAfterNack, + heldAfter == heldBefore, // no accounting bypass + remaining == 10_000_000L, + acked.isEmpty, + !caughtUp // the blind stream still gates the chunk + ) ?? s"handedBack=$handedBack reasons=$reasons held=$heldAfter before=$heldBefore caughtUp=$caughtUp" + finally f.close() + }, + test("LATCH-PINNED: a second Play racing the crossing's hold queues behind the ordering lock and cannot unlatch it") { + // The one interleaving a latch can force in this neighborhood: the byte crossing is + // MID-HOLD - blocked inside consumer.pause() with the ordering lock, the flow-control + // lock and the arbiter's monitor all held - when a second Play arrives. The play's + // boundary re-capture must queue on the ordering lock (BLOCKED is the only state its + // thread can reach), and once the hold completes, the play's User/Boundary releases + // run against a reason set that contains Merge - so the consumer stays paused and the + // held set stays intact. A regression that moved the hold outside the crossing's lock, + // or released it from the resume walk, fails this deterministically. + val f = Fixture("cs-byte-race-second-play") + try + f.runner.resume(RecordingObserver(), isDebug = false) + f.deliverHot(3) + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + f.hot.pauseGate = Some((entered, release)) + + val crossing = worker("byte-crossing") { + f.deliverHot(1, fromEntry = 3L) + } + crossing.start() + val holdInFlight = entered.await(30, TimeUnit.SECONDS) + f.hot.pauseGate = None + + val racing = worker("second-play") { + f.runner.resume(RecordingObserver(), isDebug = false) + } + racing.start() + awaitBlocked(racing) // queued on the ordering lock the crossing still holds + + val resumesBeforeRelease = f.hot.resumeCalls + release.countDown() + crossing.join(30_000) + racing.join(30_000) + + val pausedAfterBoth = f.hot.isPaused.get + val reasons = f.hotArbiter.heldReasons + val resumesAfterBoth = f.hot.resumeCalls + val held = f.merge.heldCount + val remaining = f.discard.remaining + assertTrue( + holdInFlight, + pausedAfterBoth, + reasons.contains(PauseReason.Merge), + resumesAfterBoth == resumesBeforeRelease, // the play lifted nothing it did not own + held == 4, + remaining == 10_000_000L + ) ?? (s"holdInFlight=$holdInFlight paused=$pausedAfterBoth " + + s"reasons=$reasons resumes=$resumesAfterBoth held=$held") + finally f.close() + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/replayRaceTest.scala b/server/src/test/scala/consumer/session_runner/replayRaceTest.scala new file mode 100644 index 000000000..40c1341e5 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/replayRaceTest.scala @@ -0,0 +1,386 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.{ConsumerSessionConfig, MessageDeliveryOrder} +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, MessageId as PulsarMessageId, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** THE REPLAY BOUNDARY'S LIFECYCLE RACES, pinned with latches - the exact neighborhood where this + * branch's worst bugs lived, given the same confirm-first discipline. + * + * - A RESUME RACING THE AUTO-PAUSE: the caught-up announcement runs under the ordering lock + * with a play-generation gate, and a Resume bumps the generation BEFORE its boundary + * extension takes that same lock - so a stale announcement can never pause the play the + * user just started, and the new play completes the chunk and announces itself. + * - A PAST-END NACK RACING THE BOUNDARY RE-CAPTURE: the next-chunk decision (nack + consumer + * hold) and the extension serialize on the ordering lock, so the handed-back message is + * released by the very resume that extended past it and is delivered exactly once. + * - THE RELAX RPC ARRIVING WHILE CAUGHT-UP-PAUSED (decided and pinned): it SUCCEEDS - the + * session becomes a normal Best-effort session that resumes live delivery - unless the + * user's own pause is also standing, which then keeps standing. + * + * Offline: proxy consumers, hand-built messages, the production listener / ordering layer / + * runner / pump. Interleavings are pinned by latches and a generation spin-wait on a monotonic + * counter - never by a sleep. + */ +object replayRaceTest extends ZIOSpecDefault: + + private val consumerName = "cs-replay-race-0" + private def topic(i: Int): String = s"persistent://public/default/cs-replay-race-$i" + private def sid(i: Int): String = startFromStreamId(consumerName, topic(i)) + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + /** Wait for a monotonic condition that another thread is guaranteed to make true - a latch + * over state rather than a sleep. Answers whether it happened inside the bound. */ + private def awaitTrue(condition: => Boolean, boundMs: Long = 30_000L): Boolean = + val deadline = System.nanoTime() + boundMs * 1_000_000L + while !condition && System.nanoTime() < deadline do Thread.onSpinWait() + condition + + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + @volatile var lastIds: java.util.List[PulsarMessageId] = java.util.Collections.emptyList() + /** Armed by a test to hold the ordering lock INSIDE a negative acknowledgment. */ + @volatile var nackGate: Option[(CountDownLatch, CountDownLatch)] = None + + def recordedEnd(entryId: Long): Unit = + lastIds = java.util.List.of(new MessageIdImpl(1L, entryId, -1)) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "getLastMessageIds" => lastIds + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + nackGate.foreach { (entered, release) => + entered.countDown() + release.await(60, TimeUnit.SECONDS) + () + } + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** Records frames; can be armed to BLOCK inside the write of a message-bearing frame - the + * shape of a delivery in flight across a lifecycle event, holding the send and ordering + * locks exactly as production does. */ + private final class GatedObserver(blockFirstMessageFrame: Boolean = false) extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val frames = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + private val armed = java.util.concurrent.atomic.AtomicBoolean(blockFirstMessageFrame) + override def onNext(value: consumerPb.ResumeResponse): Unit = + if value.messages.exists(_.key.isDefined) && armed.compareAndSet(true, false) then + entered.countDown() + release.await(60, TimeUnit.SECONDS) + () + frames.add(value) + () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + def received: Vector[consumerPb.ResumeResponse] = frames.asScala.toVector + def deliveredKeys: Vector[String] = + received.flatMap(_.messages).flatMap(_.key).map(_.stripPrefix("\"").stripSuffix("\"")) + def caughtUpFrames: Vector[consumerPb.ConsumerStats] = received.flatMap(_.consumerStats).filter(_.replayCaughtUp) + + private final class Fixture(sessionName: String, topicCount: Int, endsAtPlay: Map[Int, Long]): + var monoMs: Long = 0L + val pool: ConsumerSessionContextPool = ConsumerSessionContextPool() + val listener: ConsumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val consumers: Vector[RecordingConsumer] = Vector.tabulate(topicCount)(i => RecordingConsumer(topic(i))) + endsAtPlay.foreach((i, entryId) => if entryId >= 0 then consumers(i).recordedEnd(entryId)) + + private def streamsAtPlay: Vector[StartFromStream] = + Vector.tabulate(topicCount) { i => + val end = endsAtPlay.get(i).filter(_ >= 0).map(e => EntryPosition(1L, e, -1, 1)).getOrElse(EntryPosition.empty) + StartFromStream(sid(i), end) + } + + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector.tabulate(topicCount)(sid), + drainedAtStart = streamsAtPlay.filter(_.lastAtStart == EntryPosition.empty).map(_.id).toSet, + discard = StartFromDiscard.shared(0), + nowMs = () => monoMs, + policy = OrderingPolicy.GuaranteedOnly, + graceMs = 500L + ) + listener.startFromOrdering = new StartFromOrdering[HeldMessage](Some(merge), streamsAtPlay.map(s => s.id -> s).toMap) + + private val consumersByTopic: Map[String, Consumer[Array[Byte]]] = + Vector.tabulate(topicCount)(i => topic(i) -> consumers(i).consumer).toMap + val pauseArbiters: Map[String, ConsumerPauseArbiter] = consumersByTopic.map((fqn, c) => fqn -> ConsumerPauseArbiter(c)) + listener.pauseArbiters = pauseArbiters.map((fqn, arbiter) => startFromStreamId(consumerName, fqn) -> arbiter) + + val target: ConsumerSessionTargetRunner = ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumersByTopic.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumersByTopic.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumersByTopic, + pauseArbiters = pauseArbiters, + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + val runner: ConsumerSessionRunner = ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty), + messageDeliveryOrder = MessageDeliveryOrder.Guaranteed + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> target) + ) + + def deliver(partition: Int, key: String, publishTime: Long, entryId: Long): Unit = + listener.received(consumers(partition).consumer, message(topic(partition), key, publishTime, entryId)) + + def acknowledged: Vector[String] = consumers.flatMap(_.acknowledged.asScala.toVector) + def handedBack: Vector[String] = consumers.flatMap(_.handedBack.asScala.toVector) + def boundaryHeld: Boolean = pauseArbiters.values.exists(_.heldReasons.contains(PauseReason.Boundary)) + def close(): Unit = + Try(runner.stop()) + () + + private val awaitMs = 60_000L + + def spec = suite(this.getClass.toString)( + test("A RESUME RACING THE AUTO-PAUSE: the stale announcement is suppressed by the generation gate, and the new play announces") { + // The last replay message is IN FLIGHT - blocked inside the client write, holding the + // ordering and send locks - when the user presses Play again. The resume bumps the + // play generation first (it blocks on the send lock right after), so when the + // delivery completes and the barrier reports caught-up, the OLD play's announcement + // must see the foreign generation and do NOTHING: no pause of the play the user just + // started, no caught-up frame into the replaced stream. The new play's own resume + // pump then finds the chunk complete and announces it properly. + val f = Fixture("cs-race-resume-vs-autopause", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> -1L)) + try + val first = GatedObserver(blockFirstMessageFrame = true) + f.runner.resume(first, isDebug = false) + + val delivering = worker("pulsar-listener-race-a") { + f.deliver(0, "m-100", 100L, 0L) // the whole recorded range; its send blocks + } + delivering.start() + val sendInFlight = first.entered.await(30, TimeUnit.SECONDS) + + val second = GatedObserver() + val resuming = worker("grpc-resume-race-a") { + f.runner.resume(second, isDebug = false) + } + resuming.start() + // The generation bump is the resume's FIRST act, before it blocks on the send + // lock the in-flight delivery holds - the exact window the gate exists for. + val generationBumped = awaitTrue(f.runner.currentPlayGeneration == 2L) + + first.release.countDown() + delivering.join(awaitMs) + resuming.join(awaitMs) + + assertTrue( + sendInFlight, + generationBumped, + first.deliveredKeys == Vector("m-100"), // the in-flight delivery completed, once + first.caughtUpFrames.isEmpty, // THE GATE: the stale announcement wrote nothing + second.caughtUpFrames.nonEmpty, // the new play completed the chunk and said so + second.deliveredKeys.isEmpty, // and re-delivered nothing + f.acknowledged == Vector("m-100"), + !f.listener.isAcceptingNewMessages, // auto-paused under the NEW play + f.boundaryHeld, + f.runner.isReplayCaughtUpNow + ) ?? (s"first=${first.deliveredKeys}/${first.caughtUpFrames.size} " + + s"second=${second.deliveredKeys}/${second.caughtUpFrames.size} acked=${f.acknowledged} " + + s"gateOpen=${f.listener.isAcceptingNewMessages} boundaryHeld=${f.boundaryHeld}") + finally f.close() + }, + test("A PAST-END NACK RACING THE RE-CAPTURE: the extension waits for the decision, the resume releases it, the message lands once") { + // A past-boundary arrival is mid-decision - its consumer hold is placed and its + // negative acknowledgment is blocked, holding the ordering lock - while a Resume + // re-captures the (already grown) ends. The extension MUST queue behind the decision + // on the ordering lock; the resume that follows releases the Boundary hold it just + // obsoleted, and the broker's redelivery of the handed-back message arrives inside + // the new chunk: delivered exactly once, never lost, never doubled. + val f = Fixture("cs-race-nack-vs-extend", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val first = GatedObserver() + f.runner.resume(first, isDebug = false) + f.deliver(1, "b-200", 200L, 0L) // stream 1's recorded end, held for stream 0 + f.consumers(1).recordedEnd(1L) // the log grows: entry 1 exists now + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + f.consumers(1).nackGate = Some((entered, release)) + + val nacking = worker("pulsar-listener-race-b") { + f.deliver(1, "b-400", 400L, 1L) // past the CURRENT boundary: next chunk + } + nacking.start() + val decisionInFlight = entered.await(30, TimeUnit.SECONDS) + val boundaryHeldMidDecision = f.pauseArbiters(topic(1)).heldReasons.contains(PauseReason.Boundary) + + val second = GatedObserver() + val resuming = worker("grpc-resume-race-b") { + f.runner.resume(second, isDebug = false) // re-captures ends; extension queues on the lock + } + resuming.start() + val generationBumped = awaitTrue(f.runner.currentPlayGeneration == 2L) + + f.consumers(1).nackGate = None + release.countDown() + nacking.join(awaitMs) + resuming.join(awaitMs) + + val boundaryReleasedByResume = !f.boundaryHeld + f.deliver(1, "b-400", 400L, 1L) // the broker redelivers what was handed back + f.deliver(0, "a-100", 100L, 0L) // stream 0's recorded range arrives too + + assertTrue( + decisionInFlight, + generationBumped, + boundaryHeldMidDecision, // the racer was paused before it was handed back + boundaryReleasedByResume, // and the resume lifted exactly that hold + f.handedBack.count(_ == "b-400") == 1, // one nack, not a loop + second.deliveredKeys == Vector("a-100", "b-200", "b-400"), // the chunk, exactly, in key order + second.deliveredKeys.count(_ == "b-400") == 1, // the seam message landed ONCE + f.acknowledged.sorted == Vector("a-100", "b-200", "b-400"), + second.caughtUpFrames.nonEmpty // and the extended chunk completed + ) ?? (s"handedBack=${f.handedBack} delivered=${second.deliveredKeys} " + + s"acked=${f.acknowledged} boundaryHeldMid=$boundaryHeldMidDecision") + finally f.close() + }, + test("THE RELAX RPC WHILE CAUGHT-UP-PAUSED SUCCEEDS: the session becomes a live Best-effort session") { + // Decided and pinned (the brief's open question): SetDeliveryOrder's relax on a + // caught-up-paused session is the designed "continue live with Best effort" + // transition. It succeeds, the caught-up state clears, the Boundary holds lift, the + // intake re-opens, and new traffic flows under the best-effort rules. + val f = Fixture("cs-race-relax-caught-up", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val observer = GatedObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) + f.deliver(1, "b-200", 200L, 0L) + val caughtUpBeforeRelax = f.runner.isReplayCaughtUpNow && !f.listener.isAcceptingNewMessages && f.boundaryHeld + + f.runner.setDeliveryOrder(MessageDeliveryOrder.BestEffort) + + val liveAgain = f.listener.isAcceptingNewMessages && !f.boundaryHeld && !f.runner.isReplayCaughtUpNow + // New traffic - past the old boundary - is ordinary best-effort delivery now: + // held for its own residence, released by the sweep, delivered through the + // limiter's drain. + f.deliver(1, "b-400", 400L, 1L) + val acceptedByTheMerge = f.merge.heldCount == 1 + f.monoMs += 501L + f.listener.sweepStartFromStall() + val delivered = awaitTrue(observer.deliveredKeys.contains("b-400")) + + assertTrue( + caughtUpBeforeRelax, + f.runner.deliveryOrder == MessageDeliveryOrder.BestEffort, + !f.listener.startFromOrdering.isGuaranteedOrdering, + f.listener.startFromOrdering.isContinuousOrdering, + liveAgain, + acceptedByTheMerge, // no next-chunk interception under best effort + delivered, + f.handedBack.isEmpty, // nothing was refused on the way + observer.deliveredKeys == Vector("a-100", "b-200", "b-400") + ) ?? (s"caughtUpBefore=$caughtUpBeforeRelax liveAgain=$liveAgain " + + s"delivered=${observer.deliveredKeys} handedBack=${f.handedBack}") + finally f.close() + }, + test("THE RELAX NEVER OVERRIDES THE USER'S OWN PAUSE: their pause stands, only the boundary state dissolves") { + // The user paused a caught-up session on top of its auto-pause, then relaxed the + // order. The Boundary holds lift and the caught-up state clears - but the gate stays + // shut and the User hold stands: relaxing a session must never un-pause one the user + // explicitly stopped. Their next Resume goes live under Best effort. + val f = Fixture("cs-race-relax-user-paused", topicCount = 2, endsAtPlay = Map(0 -> 0L, 1 -> 0L)) + try + val observer = GatedObserver() + f.runner.resume(observer, isDebug = false) + f.deliver(0, "a-100", 100L, 0L) + f.deliver(1, "b-200", 200L, 0L) + f.runner.pause() // the user's own pause, on top of the auto-pause + + f.runner.setDeliveryOrder(MessageDeliveryOrder.BestEffort) + + val stillPausedForTheUser = !f.listener.isAcceptingNewMessages + val userHoldStands = f.pauseArbiters.values.forall(_.heldReasons.contains(PauseReason.User)) + val boundaryDissolved = !f.boundaryHeld && !f.runner.isReplayCaughtUpNow + + val second = GatedObserver() + f.runner.resume(second, isDebug = false) // and the user's resume goes live + f.deliver(1, "b-400", 400L, 1L) + f.monoMs += 501L + f.listener.sweepStartFromStall() + val delivered = awaitTrue(second.deliveredKeys.contains("b-400")) + + assertTrue( + stillPausedForTheUser, + userHoldStands, + boundaryDissolved, + f.runner.deliveryOrder == MessageDeliveryOrder.BestEffort, + delivered + ) ?? (s"pausedForUser=$stillPausedForTheUser userHold=$userHoldStands " + + s"boundaryDissolved=$boundaryDissolved delivered=${second.deliveredKeys}") + finally f.close() + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/replaySeamTest.scala b/server/src/test/scala/consumer/session_runner/replaySeamTest.scala new file mode 100644 index 000000000..b1618bf24 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/replaySeamTest.scala @@ -0,0 +1,196 @@ +package consumer.session_runner + +import org.apache.pulsar.client.impl.MessageIdImpl +import zio.test.* + +/** THE RESUME SEAM, pinned with hand clocks at the merge tier. + * + * Within one replay chunk the guaranteed barrier makes cross-stream disorder impossible; across + * a boundary extension it cannot - a delta message may carry an order key LOWER than something + * an earlier chunk already emitted (producer clock skew, or an inversion the source log itself + * stores). The contract (owner decision 2026-08-09): such a message is delivered LOUDLY FLAGGED + * - never silently late, never dropped - and counted on a session-level ledger. The comparison + * is against EMITTED KEYS OF THE SAME KIND, never against wall clock: under the event-time key, + * ordinary old event times are not violations. + * + * Also here: the counted start-from budget across the seam - a past-boundary arrival spends + * nothing, and its post-extension redelivery is delivered exactly once, never re-dropped. + */ +object replaySeamTest extends ZIOSpecDefault: + + private val topicA = "persistent://public/default/seam-a" + private val topicB = "persistent://public/default/seam-b" + private val consumerName = "cs-seam-0" + private def sid(topicFqn: String): String = startFromStreamId(consumerName, topicFqn) + + /** A guaranteed ordering layer over two streams with RECORDED ENDS, plus the pump loop's + * unit-tier stand-in: peek, read the seam flag, commit - capturing what a client would see. */ + private final class Fixture(endA: EntryPosition, endB: EntryPosition, budget: Long = 0, monoStart: Long = 0L): + var monoMs: Long = monoStart + private val streams = Vector(StartFromStream(sid(topicA), endA), StartFromStream(sid(topicB), endB)) + val merge = GlobalSkipMerge[String]( + streamIds = streams.map(_.id), + drainedAtStart = streams.filter(_.lastAtStart == EntryPosition.empty).map(_.id).toSet, + discard = StartFromDiscard.shared(budget), + nowMs = () => monoMs, + policy = if budget > 0 then OrderingPolicy.ExactCutThenGuaranteed else OrderingPolicy.GuaranteedOnly + ) + val ordering = new StartFromOrdering[String](Some(merge), streams.map(s => s.id -> s).toMap) + + private val resolutions = Vector.newBuilder[(String, StartFromOutcome)] + + def offer(topicFqn: String, orderTime: Long, entryId: Long, value: String): Unit = + resolutions ++= ordering.offer(consumerName, topicFqn, orderTime, new MessageIdImpl(1L, entryId, -1), value) + + def dropped: Vector[String] = resolutions.result().collect { case (v, StartFromOutcome.Drop) => v } + def nextChunk: Vector[String] = resolutions.result().collect { case (v, StartFromOutcome.NextChunk) => v } + + /** Drain the barrier, capturing each emission WITH the seam flag the pump would stamp. */ + def pumpAll(): Vector[(String, Boolean)] = + val out = Vector.newBuilder[(String, Boolean)] + var going = true + while going do + ordering.peekGuaranteed() match + case Some(value) => + out += (value -> ordering.peekIsSeamViolation) + ordering.commitGuaranteed() + case None => going = false + out.result() + + def extend(endA: EntryPosition, endB: EntryPosition): Unit = + ordering.extendReplayBoundary(Vector(StartFromStream(sid(topicA), endA), StartFromStream(sid(topicB), endB))) + + def spec = suite(this.getClass.toString)( + test("A DELTA KEY BELOW THE EMITTED MAXIMUM IS DELIVERED, FLAGGED AND COUNTED - never silently late, never dropped") { + val f = Fixture(endA = EntryPosition(1, 0, -1, 1), endB = EntryPosition(1, 0, -1, 1)) + f.offer(topicA, 1000L, entryId = 0, "a-1000") // stream A's whole recorded range + f.offer(topicB, 2000L, entryId = 0, "b-2000") // stream B's whole recorded range + val chunkOne = f.pumpAll() + val caughtUpAfterChunkOne = f.ordering.isReplayCaughtUp + + // Resume extends the boundary; the delta on A carries a key BELOW b-2000, which is + // already on screen - the producer-clock-skew seam. + f.extend(endA = EntryPosition(1, 1, -1, 1), endB = EntryPosition(1, 0, -1, 1)) + f.offer(topicA, 1500L, entryId = 1, "a-1500") + val delta = f.pumpAll() + + assertTrue( + chunkOne == Vector("a-1000" -> false, "b-2000" -> false), + caughtUpAfterChunkOne, + delta == Vector("a-1500" -> true), // delivered, loudly flagged + f.ordering.replaySeamViolationCount == 1L, // and counted for the banner + f.merge.lateDeliveryCount == 1L, // the lateness ledger agrees + f.dropped.isEmpty, + f.ordering.isReplayCaughtUp // the delta chunk completed too + ) ?? s"chunkOne=$chunkOne delta=$delta seams=${f.ordering.replaySeamViolationCount}" + }, + test("AN EQUAL KEY ACROSS THE SEAM IS A TIE-BREAK, NOT A VIOLATION - and a newer one is plain delivery") { + val f = Fixture(endA = EntryPosition(1, 0, -1, 1), endB = EntryPosition(1, 0, -1, 1)) + f.offer(topicA, 1000L, entryId = 0, "a-1000") + f.offer(topicB, 2000L, entryId = 0, "b-2000") + f.pumpAll() + + f.extend(endA = EntryPosition(1, 2, -1, 1), endB = EntryPosition(1, 0, -1, 1)) + f.offer(topicA, 2000L, entryId = 1, "a-2000") // ties the emitted maximum + f.offer(topicA, 2500L, entryId = 2, "a-2500") // and moves past it + val delta = f.pumpAll() + + assertTrue( + delta == Vector("a-2000" -> false, "a-2500" -> false), + f.ordering.replaySeamViolationCount == 0L, + f.merge.lateDeliveryCount == 0L + ) ?? s"delta=$delta seams=${f.ordering.replaySeamViolationCount} late=${f.merge.lateDeliveryCount}" + }, + test("THE COMPARISON IS AGAINST EMITTED KEYS, NEVER WALL CLOCK: ancient event times are not violations") { + // The order key is whatever the session selected - under the event-time key it can + // legitimately be decades behind this process's clock. The fixture's monotonic clock + // sits far ahead of every key to prove no wall-clock comparison sneaks in: a delta + // key ABOVE the emitted maximum is clean however old it is in absolute terms, and + // only a key BELOW the emitted maximum - of the same kind - is flagged. + val f = Fixture( + endA = EntryPosition(1, 0, -1, 1), + endB = EntryPosition(1, 0, -1, 1), + monoStart = 1_700_000_000_000L // "now" dwarfs every event time below + ) + f.offer(topicA, 5000L, entryId = 0, "a-5000") // event times near the epoch + f.offer(topicB, 6000L, entryId = 0, "b-6000") + f.pumpAll() + + f.extend(endA = EntryPosition(1, 2, -1, 1), endB = EntryPosition(1, 0, -1, 1)) + f.offer(topicA, 7000L, entryId = 1, "a-7000") // older than the wall clock by an era: clean + f.offer(topicA, 5500L, entryId = 2, "a-5500") // below the emitted maximum: the seam + val delta = f.pumpAll() + + assertTrue( + delta == Vector("a-7000" -> false, "a-5500" -> true), + f.ordering.replaySeamViolationCount == 1L + ) ?? s"delta=$delta seams=${f.ordering.replaySeamViolationCount}" + }, + test("THE COUNTED BUDGET NEVER CROSSES THE BOUNDARY: a past-end arrival spends nothing and is delivered once, later") { + // Skip-first-2 under Guaranteed. The recorded history is a@10, a@30 (stream A) and + // b@20 (stream B, entry 0 only). A past-boundary arrival lands MID-CUT with budget + // still unspent - it must not consume a unit, must not be dropped, must not enter + // the duplicate watermark; after the boundary extends, its redelivery is an ordinary + // in-chunk message: DELIVERED, with the budget long since spent on the recorded two. + val f = Fixture(endA = EntryPosition(1, 2, -1, 1), endB = EntryPosition(1, 0, -1, 1), budget = 2) + def remaining: Long = f.ordering.progressDiscard.map(_.remaining).getOrElse(-1L) + + f.offer(topicA, 10L, entryId = 0, "a-10") // globally first: dropped once B speaks + f.offer(topicB, 20L, entryId = 0, "b-20") // B's recorded end; a-10 drops (budget 2 -> 1) + val remainingMidCut = remaining + f.offer(topicB, 40L, entryId = 1, "b-40") // STRICTLY past B's end: next chunk's business + val remainingAfterPastEnd = remaining + f.offer(topicA, 30L, entryId = 2, "a-30") // A's recorded end; b-20 (globally second) drops + val cutState = (f.dropped, f.nextChunk, remaining) + val chunkOne = f.pumpAll() // the replay remainder: a-30 alone + val caughtUpAfterChunkOne = f.ordering.isReplayCaughtUp + + // Resume: the boundary extends over the handed-back entry, and the broker redelivers. + f.extend(endA = EntryPosition(1, 2, -1, 1), endB = EntryPosition(1, 1, -1, 1)) + f.offer(topicB, 40L, entryId = 1, "b-40-redelivered") + val delta = f.pumpAll() + + assertTrue( + remainingMidCut == 1L, + remainingAfterPastEnd == 1L, // the past-end arrival spent NOTHING + cutState == (Vector("a-10", "b-20"), Vector("b-40"), 0L), // the cut is the GLOBAL first two + chunkOne == Vector("a-30" -> false), + caughtUpAfterChunkOne, + delta == Vector("b-40-redelivered" -> false), // delivered exactly once, not re-dropped + f.dropped == Vector("a-10", "b-20") // and the drop set never grew past the budget + ) ?? (s"remaining=($remainingMidCut, $remainingAfterPastEnd) cut=$cutState chunkOne=$chunkOne " + + s"delta=$delta dropped=${f.dropped} nextChunk=${f.nextChunk}") + }, + test("a NON-guaranteed counted cut is untouched by the boundary machinery - past-end interception is guaranteed-only") { + // The control: the same shape under ExactCutThenBestEffort must keep the old + // behavior - recorded ends release the wait but nothing is ever handed back as + // next-chunk, because best effort has no replay boundary. Hand clock, so the + // residence releases are the test's to trigger. + var monoMs = 0L + val streams = Vector( + StartFromStream(sid(topicA), EntryPosition(1, 0, -1, 1)), + StartFromStream(sid(topicB), EntryPosition(1, 0, -1, 1)) + ) + val merge = GlobalSkipMerge[String]( + streamIds = streams.map(_.id), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(1), + nowMs = () => monoMs, + policy = OrderingPolicy.ExactCutThenBestEffort + ) + val ordering = new StartFromOrdering[String](Some(merge), streams.map(s => s.id -> s).toMap) + val out = Vector.newBuilder[(String, StartFromOutcome)] + out ++= ordering.offer(consumerName, topicA, 10L, new MessageIdImpl(1L, 0L, -1), "a-10") + out ++= ordering.offer(consumerName, topicB, 20L, new MessageIdImpl(1L, 0L, -1), "b-20") + out ++= ordering.offer(consumerName, topicB, 40L, new MessageIdImpl(1L, 1L, -1), "b-40") + monoMs += mergeTopicsGraceMs + 1 // every residence expires; best effort releases the rest + out ++= ordering.sweepStalled() + val outcomes = out.result() + assertTrue( + outcomes.collect { case (v, StartFromOutcome.Drop) => v } == Vector("a-10"), + outcomes.collect { case (v, StartFromOutcome.NextChunk) => v }.isEmpty, // no boundary, no next chunk + outcomes.collect { case (v, StartFromOutcome.Deliver) => v } == Vector("b-20", "b-40"), + merge.replaySeamViolationCount == 0L + ) ?? s"outcomes=$outcomes" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/sessionContextConcurrencyTest.scala b/server/src/test/scala/consumer/session_runner/sessionContextConcurrencyTest.scala new file mode 100644 index 000000000..a20299bd1 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/sessionContextConcurrencyTest.scala @@ -0,0 +1,429 @@ +package consumer.session_runner + +import java.util.concurrent.atomic.AtomicLong + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.TreatBytesAsJson +import _root_.consumer.message_filter.basic_message_filter.targets.{BasicMessageFilterTarget, BasicMessageFilterValueTarget} +import _root_.consumer.message_filter.{JsMessageFilter, MessageFilter, MessageFilterChain, MessageFilterChainMode} +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.value_projections.ValueProjectionList +import io.circe.parser.parse as parseJson +import org.apache.pulsar.client.api.{Consumer, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch} +import scala.jdk.CollectionConverters.* + +/** One consumer session owns ONE GraalVM JS context, and every partition of every target is + * delivered on its own Pulsar listener thread. + * + * A GraalVM context may MIGRATE between threads but may not be entered by two at once - the loser + * gets "Multi threaded access requested by thread ... but is not allowed for language(s) js". And + * the context is entered per DELIVERED MESSAGE whether or not the user configured any JS, because + * `setCurrentMessage` is itself a JS call. + * + * Worse than the exception is the state hand-off it hides: `setCurrentMessage` writes the message + * under test into a GLOBAL JS variable, and the filter chain, the coloring rules and the value + * projections all read it back out afterwards. Two threads interleaving there evaluate one + * message's filter against another message's contents - a silently wrong retained set, with no + * exception anywhere. + * + * Everything here runs offline. `ConsumerSessionTargetRunner.resume` installs the REAL production + * message handler and `ConsumerListener.received` is the REAL delivery path; only the broker is + * replaced, by a proxy consumer and hand-built `MessageImpl`s. + */ +object sessionContextConcurrencyTest extends ZIOSpecDefault: + + private val consumerName = "cs-ctx-race-0" + + private def partitionFqn(i: Int): String = s"persistent://public/default/cs-ctx-race-partition-$i" + + /** Retains the even `n`s and nothing else - the oracle for the state hand-off. + * + * The filter reads `message.value` out of the shared context, so if another thread overwrote + * the current message in between, THIS message is judged by THAT message's payload: an odd `n` + * gets retained, or an even one dropped. Both show up as a wrong retained set. + */ + private val evenOnly: MessageFilterChain = + MessageFilterChain( + isEnabled = true, + isNegated = false, + mode = MessageFilterChainMode.All, + filters = Vector( + MessageFilter( + isEnabled = true, + isNegated = false, + targetField = BasicMessageFilterTarget(target = BasicMessageFilterValueTarget()), + filter = JsMessageFilter(jsCode = "v => v.n % 2 === 0") + ) + ) + ) + + private def message(topicFqn: String, n: Int, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + // publish_time is mandatory on MessageMetadata - reading it when unset throws. + md.setPublishTime(1_700_000_000_000L + n) + md.setPartitionKey(n.toString) + val msg = MessageImpl.create[Array[Byte]]( + md, + ByteBuffer.wrap(s"""{"n":$n}""".getBytes("UTF-8")), + Schema.BYTES, + topicFqn + ) + // serializeMessage reads msg.getMessageId.toByteArray; MessageImpl.create leaves it null. + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** The handful of things `ConsumerListener.received` asks a consumer. + * + * CONNECTED, and the acknowledgment is answered rather than short-circuited. This used to say + * `isConnected = false` purely to skip the acknowledge without a broker, which stopped working + * - and rightly so: a message the consumer cannot answer for is now handed straight back + * instead of being decided, so a disconnected fixture delivers nothing at all and this suite + * would have measured an empty session. A real delivering consumer is connected. + */ + private def consumerOn(topicFqn: String): Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "acknowledgeAsync" => java.util.concurrent.CompletableFuture.completedFuture(null) + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def targetRunner( + filterChain: MessageFilterChain, + topicFqns: Vector[String], + pool: ConsumerSessionContextPool, + listener: ConsumerListener + ): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = TreatBytesAsJson()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = topicFqns)), + messageFilterChain = filterChain, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = topicFqns, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = listener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + /** What one concurrent delivery run produced. */ + private final case class Outcome( + retained: Vector[Int], + dropped: Int, + jsErrors: Vector[String], + escaped: Vector[String], + consoleResults: Vector[String], + finished: Boolean, + // Production counters, read after the run. Both are incremented from the message handler + // BEFORE the context lease, i.e. concurrently from every partition thread. + processedByTarget: Long, + processedBySession: Long + ) + + private def nOf(msg: ConsumerSessionMessage): Option[Int] = + msg.messageValueAsJson.toOption + .flatMap(json => parseJson(json).toOption) + .flatMap(_.hcursor.downField("n").as[Int].toOption) + + /** `partitions` listener threads deliver `perPartition` messages each into ONE listener, exactly + * as Pulsar does for a partitioned topic: one listener per target, one thread per partition. + * + * `consoleRounds > 0` additionally drives `ConsumerServiceImpl.runCode`'s path from a further + * thread - the browser console evaluates in the SAME session context, off a gRPC thread. + */ + private def deliverConcurrently( + filterChain: MessageFilterChain, + partitions: Int, + perPartition: Int, + consoleRounds: Int = 0 + ): Outcome = + val topicFqns = Vector.tabulate(partitions)(partitionFqn) + val pool = ConsumerSessionContextPool() + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + val runner = targetRunner(filterChain, topicFqns, pool, listener) + + // A REAL session, so `incrementNumMessageProcessed` below is the production method rather + // than a stand-in - that counter is stamped onto every pb.Message the browser receives. + val session = ConsumerSessionRunner( + sessionName = "cs-ctx-race", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> runner) + ) + + val retained = new ConcurrentLinkedQueue[Int]() + val dropped = new AtomicInteger(0) + val jsErrors = new ConcurrentLinkedQueue[String]() + val escaped = new ConcurrentLinkedQueue[String]() + val consoleResults = new ConcurrentLinkedQueue[String]() + + runner.resume( + onNext = (msg, _, _, errors, _) => + errors.foreach(jsErrors.add) + msg match + case Some(m) => nOf(m).foreach(retained.add) + case None => dropped.incrementAndGet() + , + isDebug = true, + incrementNumMessageProcessed = () => session.incrementNumMessageProcessed(), + onStartFromDiscardProgress = () => (), + admitDelivery = _ => DeliveryAdmission.Prepare + ) + + val start = new CountDownLatch(1) + + def worker(name: String)(body: => Unit): Thread = + val runnable: Runnable = () => + start.await() + try body + catch + case err: Throwable => + escaped.add(s"${err.getClass.getSimpleName}: ${err.getMessage}") + () + val t = new Thread(runnable, name) + t.setDaemon(true) + t + + val deliverers = topicFqns.zipWithIndex.map { (topicFqn, p) => + worker(s"pulsar-listener-$p") { + val consumer = consumerOn(topicFqn) + var i = 0 + while i < perPartition do + // Caught PER MESSAGE, as Pulsar does: an exception out of `received` is logged by + // the consumer's listener executor and the next message is delivered anyway. So + // the run continues past a collision and the retained set stays assertable. + try + // Globally unique `n`, so odd and even are spread across every partition. + listener.received(consumer, message(topicFqn, i * partitions + p, i.toLong)) + catch + case err: Throwable => + escaped.add(s"${err.getClass.getSimpleName}: ${err.getMessage}") + () + i += 1 + } + } + + val consoles = Option.when(consoleRounds > 0) { + worker("grpc-run-code") { + var i = 0 + while i < consoleRounds do + // Exactly what ConsumerServiceImpl.runCode does with the session's pool. + consoleResults.add(pool.withContext(0)(_.runCode("1 + 1"))) + i += 1 + } + }.toVector + + val workers = deliverers ++ consoles + workers.foreach(_.start()) + start.countDown() + workers.foreach(_.join(120_000)) + + Outcome( + retained = retained.asScala.toVector, + dropped = dropped.get, + jsErrors = jsErrors.asScala.toVector, + escaped = escaped.asScala.toVector, + consoleResults = consoleResults.asScala.toVector, + finished = workers.forall(!_.isAlive), + processedByTarget = runner.stats.messageProcessed.get, + processedBySession = session.numMessageProcessed + ) + + def spec = suite(this.getClass.toString)( + test("concurrent partition listeners never collide in the session's JS context") { + val partitions = 3 + val perPartition = 200 + val total = partitions * perPartition + val outcome = deliverConcurrently(evenOnly, partitions, perPartition) + val expected = (0 until total).filter(_ % 2 == 0).toVector + + assertTrue( + outcome.finished, + outcome.escaped.isEmpty, + outcome.jsErrors.isEmpty, + outcome.retained.sorted == expected, + outcome.dropped == total - expected.size + ) ?? (s"escaped=${outcome.escaped.take(3)} jsErrors=${outcome.jsErrors.take(3)} " + + s"retained=${outcome.retained.size}/${expected.size} dropped=${outcome.dropped}/${total - expected.size} " + + s"wronglyRetained=${outcome.retained.filter(_ % 2 != 0).take(5)}") + }, + test("a session with NO user JS enters the context per message too") { + // `setCurrentMessage` is a JS call, so the context is entered before any `isEnabled` + // check - a session that configured no filter at all races just the same. + val partitions = 4 + val perPartition = 150 + val total = partitions * perPartition + val outcome = deliverConcurrently(MessageFilterChain.empty, partitions, perPartition) + + assertTrue( + outcome.finished, + outcome.escaped.isEmpty, + outcome.retained.sorted == (0 until total).toVector, + outcome.dropped == 0 + ) ?? s"escaped=${outcome.escaped.take(3)} retained=${outcome.retained.size}/$total dropped=${outcome.dropped}" + }, + test("the browser console shares the session context with the listener threads") { + // `ConsumerServiceImpl.runCode` evaluates in `getContext(0)` off a gRPC thread while the + // listeners are inside the same context. `runCode` swallows what it catches, so a + // collision here is not an exception - it is an "[ERROR] ..." handed to the user as the + // answer to their expression. + val partitions = 2 + val perPartition = 200 + val outcome = deliverConcurrently(evenOnly, partitions, perPartition, consoleRounds = 200) + val badResults = outcome.consoleResults.filter(_ != "2") + + assertTrue( + outcome.finished, + outcome.escaped.isEmpty, + outcome.consoleResults.size == 200, + badResults.isEmpty + ) ?? s"escaped=${outcome.escaped.take(3)} console=${badResults.take(3)} (${badResults.size} of ${outcome.consoleResults.size})" + }, + test("the session's processed counter survives concurrent increment") { + // The end-to-end test below asserts the counters are RIGHT, but it cannot prove they are + // ATOMIC: the increments sit just before the JS lease, and the lock downstream throttles + // arrivals, so threads almost never collide in that window. It passes either way. + // + // This one contends on the counter directly and does discriminate: with a plain + // read-modify-write it loses updates well before the assertion. + val session = ConsumerSessionRunner( + sessionName = "cs-counter-race", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map.empty + ) + + val threads = 8 + val perThread = 50000 + val start = new CountDownLatch(1) + val workers = Vector.tabulate(threads) { i => + val r: Runnable = () => + start.await() + var k = 0 + while k < perThread do + session.incrementNumMessageProcessed() + k += 1 + val t = new Thread(r, s"pulsar-listener-counter-$i") + t.start() + t + } + start.countDown() + workers.foreach(_.join(60000)) + + val expected = (threads * perThread).toLong + assertTrue(session.numMessageProcessed == expected) ?? + s"expected $expected, counted ${session.numMessageProcessed} (lost ${expected - session.numMessageProcessed})" + }, + test("every delivered message is counted, on every partition thread") { + // Scope, stated honestly: this asserts the counters are RIGHT end to end - reached + // through the real handler, on one listener thread per partition, and counting EVERY + // delivered message. It does NOT pin their atomicity, and it must not be read as doing + // so: making the increments a plain read-modify-write leaves this test GREEN, because + // the JS lease immediately downstream throttles arrivals so the threads rarely collide + // in the pre-lease window. The test above contends on the counter directly and is what + // actually catches lost updates. + // + // What this one does cover that the other cannot: counting is independent of filtering. + // `evenOnly` drops half the messages, and both counters must still see ALL of them. + val partitions = 4 + val perPartition = 250 + val total = partitions * perPartition + val outcome = deliverConcurrently(evenOnly, partitions, perPartition) + + assertTrue( + outcome.finished, + outcome.processedByTarget == total.toLong, + outcome.processedBySession == total.toLong + ) ?? (s"expected $total; target counted ${outcome.processedByTarget} " + + s"(lost ${total - outcome.processedByTarget}), session counted ${outcome.processedBySession} " + + s"(lost ${total - outcome.processedBySession})") + }, + test("a lease holds the current message for the WHOLE message, not for one JS call") { + // The state hand-off on its own, without waiting for a collision to happen to land. + // `setCurrentMessage` and every read of it are SEPARATE entries into the context, so + // excluding per call is not enough: between this thread's write and its read, another + // partition can legally enter and overwrite `globalThis.__dekaf_currentMessage`. That + // outcome throws nothing - it silently judges one message by another's contents. + val pool = ConsumerSessionContextPool() + val interloperReady = new CountDownLatch(1) + val readBack = new java.util.concurrent.atomic.AtomicReference("") + + // BOTH threads take context 0 EXPLICITLY, not `withNextContext`. The pool is one context + // today (poolSize pinned to 1), so `withNextContext` happens to hand both the same one - + // but nothing pins that pin, and a future pool-size bump would silently give the two + // threads DIFFERENT contexts, so they would never contend and this test would pass + // vacuously. Pinning both to key 0 keeps the collision real whatever the pool size. + val interloper = new Thread( + { () => + interloperReady.countDown() + pool.withContext(0)(_.setCurrentMessage("""{"key":"B"}""", Right("""{"n":2}"""))) + }: Runnable, + "other-partition" + ) + interloper.setDaemon(true) + + pool.withContext(0) { sessionContext => + sessionContext.setCurrentMessage("""{"key":"A"}""", Right("""{"n":1}""")) + interloper.start() + interloperReady.await() + // Not a readiness wait - the opposite. It WIDENS the window deliberately, so that + // "nothing got in" is a claim about the lease rather than about how fast the two + // threads happened to run. + Thread.sleep(200) + readBack.set(sessionContext.runCode(s"$CurrentMessageVarName.key + '/' + $CurrentMessageVarName.value.n")) + } + interloper.join(30_000) + + assertTrue(readBack.get.contains("A/1"), !readBack.get.contains("B")) ?? + s"the message read back mid-lease was not the one this thread set: ${readBack.get}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/sessionOutputSerializationTest.scala b/server/src/test/scala/consumer/session_runner/sessionOutputSerializationTest.scala new file mode 100644 index 000000000..14b78e44e --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/sessionOutputSerializationTest.scala @@ -0,0 +1,551 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong, AtomicReference} +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters.* + +/** THE SESSION'S OUTPUT PATH IS SHARED AND ITS INPUTS ARE NOT. + * + * Pulsar delivers each physical topic on its own listener thread. Everything downstream of that is + * shared by the whole session: one global ordering layer that decides the delivery ORDER, one + * gRPC `StreamObserver` that every response leaves through, and one start-from budget. + * + * Three things were unserialized, and each loses something different: + * + * - the ordering layer's lock was released before the messages it resolved were processed, so a + * later message could overtake an earlier one and stateful filters and projections saw them in + * a different order than the merge decided; + * - progress pushes called `StreamObserver.onNext` straight from listener threads, and + * `StreamObserver` is not thread-safe; + * - the discard budget was decremented before the message was acknowledged, so a progress push + * that threw (a cancelled stream, a concurrently entered observer) spent the budget on a + * message that was never acknowledged - its redelivery was then DELIVERED instead of skipped, + * and the session showed a message the user asked to skip. + * + * Everything here runs offline: `ConsumerListener.received` is the real delivery path, driven from + * real threads with a proxy consumer and hand-built messages. Only the broker is replaced. + */ +object sessionOutputSerializationTest extends ZIOSpecDefault: + + private val consumerName = "cs-serialized-0" + private val p0 = "persistent://public/default/cs-serialized-partition-0" + private val p1 = "persistent://public/default/cs-serialized-partition-1" + + /** A delivered message labelled by its partition key, which is what the assertions read back. */ + private def message(topicFqn: String, label: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(label) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"label":"$label"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + /** A connected consumer that records what was acknowledged and what was handed back. */ + private final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def stream(topicFqn: String, lastEntryId: Long): StartFromStream = + StartFromStream(startFromStreamId(consumerName, topicFqn), EntryPosition(1L, lastEntryId, -1, 1)) + + private def worker(name: String)(body: => Unit): Thread = + val t = new Thread((() => body): Runnable, name) + t.setDaemon(true) + t + + private def targetRunner(consumerListener: ConsumerListener): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p0))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p0), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(consumerListener: ConsumerListener): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-serialized", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> targetRunner(consumerListener)) + ) + + /** A `StreamObserver` exactly as unforgiving as the real one: it is not thread-safe, and it + * says so. `received` is a plain `var List` so a lost update shows up as a missing response, + * and `overlaps` counts every time two threads were inside `onNext` at once. + */ + private final class UnsafeObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val inside = AtomicInteger(0) + val overlaps = AtomicInteger(0) + var received: List[consumerPb.ResumeResponse] = Nil + + override def onNext(value: consumerPb.ResumeResponse): Unit = + if inside.incrementAndGet() != 1 then overlaps.incrementAndGet() + val current = received + Thread.`yield`() + received = value :: current + inside.decrementAndGet() + () + + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + + private val observerSuite = suite("every response leaves through one serialized sender")( + test("concurrent listener threads never enter the response observer at once") { + // Progress pushes made this concrete: one listener thread per physical topic, each + // calling `onNext` directly on the session's single observer while a normal response + // could be on its way out from another. + val runner = session(ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ()))) + val observer = UnsafeObserver() + runner.resume(observer, isDebug = false) + val threads = 8 + val perThread = 400 + val start = CountDownLatch(1) + + val workers = Vector.tabulate(threads) { i => + worker(s"pulsar-listener-$i") { + start.await() + var k = 0 + while k < perThread do + runner.sendResponse(observer, Seq(consumerPb.Message(numMessageProcessed = k.toLong)), Vector.empty) + k += 1 + } + } + workers.foreach(_.start()) + start.countDown() + workers.foreach(_.join(120_000)) + + assertTrue( + observer.overlaps.get == 0, + observer.received.size == threads * perThread + ) ?? s"${observer.overlaps.get} concurrent entries; ${observer.received.size} of ${threads * perThread} responses survived" + }, + test("concurrent progress pushes never enter the response observer at once") { + // The same thing through the production path: every partition thread claiming the + // shared discard, each of which may push a progress frame. + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.startAcceptingNewMessages() + listener.startFromDiscard = StartFromDiscard.shared(80_000) + val runner = session(listener) + val observer = UnsafeObserver() + runner.resume(observer, isDebug = false) + + val threads = 8 + val start = CountDownLatch(1) + val workers = Vector.tabulate(threads) { i => + worker(s"pulsar-listener-progress-$i") { + start.await() + var k = 0 + while k < 10_000 do + listener.decide(p0, canAcknowledge = true) + k += 1 + } + } + workers.foreach(_.start()) + start.countDown() + workers.foreach(_.join(120_000)) + + assertTrue( + observer.overlaps.get == 0, + listener.startFromDiscard.remaining == 0L, + observer.received.nonEmpty + ) ?? s"${observer.overlaps.get} concurrent entries across ${observer.received.size} progress frames" + } + ) + + private val budgetSuite = suite("a failing progress push must not cost a skipped message")( + test("a progress observer that throws still leaves exactly n messages skipped and acknowledged") { + // The client's stream can be cancelled at any moment, and `StreamObserver.onNext` then + // throws. The budget was already decremented by the time it did, and the message was + // NOT acknowledged - so the broker redelivered it, the (now spent) budget let it + // through, and the session showed a message the user had asked to skip. + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.startAcceptingNewMessages() + listener.startFromDiscard = StartFromDiscard.shared(3) + listener.onStartFromDiscardProgress = () => throw new IllegalStateException("call already cancelled") + + val delivered = ConcurrentLinkedQueue[String]() + listener.targetMessageHandler.onNext = msg => delivered.add(msg.getKey) + + val consumer = RecordingConsumer(p0) + val escaped = ConcurrentLinkedQueue[String]() + (1 to 5).foreach { i => + try listener.received(consumer.consumer, message(p0, s"m$i", 100L + i, i.toLong)) + catch case err: Throwable => escaped.add(s"${err.getClass.getSimpleName}: ${err.getMessage}") + } + + assertTrue( + escaped.asScala.toVector.isEmpty, + consumer.acknowledged.asScala.toVector == Vector("m1", "m2", "m3", "m4", "m5"), + delivered.asScala.toVector == Vector("m4", "m5"), + listener.startFromDiscard.remaining == 0L + ) ?? (s"escaped=${escaped.asScala.toVector} acknowledged=${consumer.acknowledged.asScala.toVector} " + + s"delivered=${delivered.asScala.toVector}") + } + ) + + /** Records the ORDER of everything the client sees, and whether the two kinds of event ever + * overlapped. `onNext` is deliberately slow so "nothing overlapped" is a claim about the lock + * rather than about how fast two threads happened to run. + */ + private final class SequencedObserver(writeDelayMs: Long = 0) extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + val enteredNext = CountDownLatch(1) + private val insideNext = AtomicInteger(0) + val completed = AtomicBoolean(false) + val completedWhileWriting = AtomicBoolean(false) + val nextAfterCompleted = AtomicInteger(0) + private val frames = ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + + def received: Vector[consumerPb.ResumeResponse] = frames.asScala.toVector + + override def onNext(value: consumerPb.ResumeResponse): Unit = + if completed.get then nextAfterCompleted.incrementAndGet() + insideNext.incrementAndGet() + enteredNext.countDown() + if writeDelayMs > 0 then Thread.sleep(writeDelayMs) + frames.add(value) + insideNext.decrementAndGet() + () + + override def onError(t: Throwable): Unit = () + + override def onCompleted(): Unit = + if insideNext.get > 0 then completedWhileWriting.set(true) + completed.set(true) + + /** A listener whose progress snapshot can be interleaved on purpose. The FIRST caller is held + * inside the snapshot and comes away with an INCOMPLETE reading; every caller after it gets the + * COMPLETE one. That is precisely the window `sendResponse` used to leave open - the response, + * including its progress counters, was built BEFORE the send lock was taken. + */ + private final class GatedProgressListener(gate: CountDownLatch, reachedSnapshot: CountDownLatch) + extends ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())): + private val callers = AtomicInteger(0) + private val stale = StartFromDiscard.shared(2) + private val fresh = StartFromDiscard.shared(2) + fresh.claim(p0) + fresh.claim(p0) + + override def progressDiscard: StartFromDiscard = + if callers.incrementAndGet() == 1 then + reachedSnapshot.countDown() + gate.await(60, TimeUnit.SECONDS) + stale + else fresh + + private val terminalSuite = suite("progress never goes backwards, and nothing follows the end of the stream")( + test("AN OLDER PROGRESS FRAME CANNOT OVERTAKE A NEWER COMPLETE ONE") { + // The response was built - and its start-from counters read - before `sendLock` was + // taken, so two listener threads could snapshot in one order and send in the other. The + // client clears its progress panel when it sees `complete`, then a stale incomplete + // frame arriving behind it reopened a "skipping..." panel that never went away. + val gate = CountDownLatch(1) + val reached = CountDownLatch(1) + val runner = session(GatedProgressListener(gate, reached)) + val observer = SequencedObserver() + runner.resume(observer, isDebug = false) + + val older = worker("pulsar-listener-stale")(runner.sendResponse(observer, Seq.empty, Vector.empty)) + older.start() + reached.await(60, TimeUnit.SECONDS) + + val newer = worker("pulsar-listener-fresh")(runner.sendResponse(observer, Seq.empty, Vector.empty)) + newer.start() + // Under the fix the newer thread CANNOT get in: the older one holds the send lock while + // it is held at the snapshot. Under the defect it sails past and lands first. + newer.join(2_000) + + gate.countDown() + older.join(60_000) + newer.join(60_000) + + val progress = observer.received.flatMap(_.consumerStats).flatMap(_.startFromProgress) + val firstComplete = progress.indexWhere(_.complete) + assertTrue( + progress.size == 2, + firstComplete >= 0, + progress.drop(firstComplete).forall(_.complete) + ) ?? s"progress frames in the order the client saw them: ${progress.map(p => s"${p.messagesSkipped}/${p.messagesToSkip} complete=${p.complete}")}" + }, + test("NO RESPONSE REACHES THE CLIENT AFTER THE STREAM HAS BEEN COMPLETED, and the send SAYS SO") { + // `stop` called `onCompleted` with no terminal gate at all, so any listener thread still + // in flight - or any later push - called `onNext` on a finished stream. Real gRPC throws + // there; the browser sees a stream that ended and then spoke again. + // + // Not writing is only half the contract. The gate used to swallow the send and return + // NORMALLY, and its caller reads a normal return as "the client has it" and + // acknowledges - so a delivery in flight across the end of the stream was consumed off + // the broker and shown to nobody. A send carrying a message must FAIL. + val runner = session(ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ()))) + val observer = SequencedObserver() + runner.resume(observer, isDebug = false) + + runner.stop() + val refused = scala.util.Try(runner.sendResponse(observer, Seq(consumerPb.Message(numMessageProcessed = 1L)), Vector.empty)).isFailure + + assertTrue(observer.completed.get, refused, observer.nextAfterCompleted.get == 0) ?? + s"refused=$refused; ${observer.nextAfterCompleted.get} responses were written after the stream was completed" + }, + test("THE STREAM IS NOT COMPLETED WHILE A RESPONSE IS STILL BEING WRITTEN") { + // `onCompleted` was called outside the send lock, so it could interleave with an + // `onNext` from a listener thread - and `StreamObserver` is not thread-safe. + val runner = session(ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ()))) + val observer = SequencedObserver(writeDelayMs = 400) + runner.resume(observer, isDebug = false) + + val writer = worker("pulsar-listener-writing")(runner.sendResponse(observer, Seq(consumerPb.Message()), Vector.empty)) + writer.start() + observer.enteredNext.await(60, TimeUnit.SECONDS) + + runner.stop() + writer.join(60_000) + + assertTrue(observer.completed.get, !observer.completedWhileWriting.get) ?? + "the response stream was completed while a listener thread was inside onNext" + } + ) + + private val orderSuite = suite("resolved messages are processed in the order the merge decided")( + test("a later message cannot overtake the one the merge released first") { + // The merge holds p0/100 until p1 speaks. p1's message resolves BOTH - dropping p1/50 + // and releasing p0/100 - and the releasing thread then has to process p0/100. Meanwhile + // p0's own thread offers 200, which the merge (its budget now spent) passes straight + // through. + // + // Releasing the merge's lock before processing let p0/200 be handled first: the + // session's stateful filters, projections and accumulated state then saw the two + // messages in the opposite order to the one the merge had just decided on. + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + listener.startAcceptingNewMessages() + listener.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(p0, 9), stream(p1, 0))) + ) + + val processed = ConcurrentLinkedQueue[String]() + val firstEntered = CountDownLatch(1) + listener.targetMessageHandler.onNext = msg => + if msg.getKey == "a1" then + firstEntered.countDown() + // WIDENS the window on purpose: "nothing overtook it" must be a claim about + // the lock, not about how fast the two threads happened to run. + Thread.sleep(300) + processed.add(msg.getKey) + + val consumerP0 = RecordingConsumer(p0) + val consumerP1 = RecordingConsumer(p1) + val escaped = ConcurrentLinkedQueue[String]() + + // p0's first message is held by the merge and returns nothing. + listener.received(consumerP0.consumer, message(p0, "a1", 100L, 0L)) + + val releaser = worker("pulsar-listener-p1") { + try listener.received(consumerP1.consumer, message(p1, "b1", 50L, 0L)) + catch case err: Throwable => escaped.add(s"p1: ${err.getMessage}") + } + val overtaker = worker("pulsar-listener-p0") { + firstEntered.await(30, TimeUnit.SECONDS) + try listener.received(consumerP0.consumer, message(p0, "a2", 200L, 1L)) + catch case err: Throwable => escaped.add(s"p0: ${err.getMessage}") + } + + releaser.start() + overtaker.start() + releaser.join(60_000) + overtaker.join(60_000) + + assertTrue( + escaped.asScala.toVector.isEmpty, + processed.asScala.toVector == Vector("a1", "a2"), + consumerP1.acknowledged.asScala.toVector == Vector("b1") + ) ?? s"escaped=${escaped.asScala.toVector} processed=${processed.asScala.toVector}" + }, + test("the ordering lock is NOT held across the client write - a blocked client cannot park the merge") { + // The limiter's own file states the invariant: offers under the session's ordering + // lock are an ENQUEUE, "so a Pulsar listener thread is never parked and the global + // merge is never starved into its silent-stream give-up". The default unlimited path + // (rate 0) used to break exactly that: `offer` answered processNow and ran the whole + // delivery - JS lease, gRPC write - inline on the offering thread, ordering lock + // held. One backpressured client then parked every listener thread AND the sweep. + // + // Wired like production: an Ordered(BestEffort) layer - the DEFAULT session shape - + // and a limiter at rate 0 whose drain runs on its own timer thread. The client write + // BLOCKS on a latch; the merge must remain fully usable meanwhile, and the delivery + // order must still be exactly the merge's decision order. + val delivered = ConcurrentLinkedQueue[String]() + val writeEntered = CountDownLatch(1) + val writeRelease = CountDownLatch(1) + val allDelivered = CountDownLatch(3) + val listener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => + if msg.getKey == "b1" then + writeEntered.countDown() + writeRelease.await(60, TimeUnit.SECONDS) + delivered.add(msg.getKey) + allDelivered.countDown() + () + )) + listener.startAcceptingNewMessages() + listener.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.Ordered( + // Best effort carries EMPTY recorded ends - the continuous rules need none. + Vector( + StartFromStream(startFromStreamId(consumerName, p0), EntryPosition.empty), + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition.empty) + ), + _root_.consumer.session_config.MessageDeliveryOrder.BestEffort + ) + ) + val drainExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(runnable => { + val t = Thread(runnable, "delivery-rate-limit-lock-test") + t.setDaemon(true) + t + }) + val limiter = DeliveryRateLimiter[HeldMessage]( + core = DeliveryRateLimiterCore[HeldMessage](nowMs = () => java.lang.System.nanoTime() / 1_000_000L), + schedule = (delayMs, task) => { drainExecutor.schedule(task, delayMs, TimeUnit.MILLISECONDS); () }, + process = held => held.listener.deliverNow(held), + holdPermits = () => true, + releasePermits = () => true + ) + listener.deliveryRateLimiter = Some(limiter) + + val consumerP0 = RecordingConsumer(p0) + val consumerP1 = RecordingConsumer(p1) + + // a1 is held (p1 has not spoken); b1's offer resolves b1 itself for delivery. + listener.received(consumerP0.consumer, message(p0, "a1", 100L, 0L)) + val offeringThread = worker("pulsar-listener-p1")( + listener.received(consumerP1.consumer, message(p1, "b1", 50L, 0L)) + ) + offeringThread.start() + // The write is genuinely in flight (and blocked)... + val writeInFlight = writeEntered.await(30, TimeUnit.SECONDS) + // ...and the OFFERING thread has already returned: the write does not ride it. + offeringThread.join(10_000) + val offeringThreadFree = !offeringThread.isAlive + // The merge stays fully usable while the client is wedged - another partition's + // offer needs the ordering lock and must not park behind the write. + val laterOffer = worker("pulsar-listener-p0")( + listener.received(consumerP0.consumer, message(p0, "a2", 200L, 1L)) + ) + laterOffer.start() + laterOffer.join(10_000) + val mergeUsableDuringWrite = !laterOffer.isAlive + + writeRelease.countDown() + // p1 speaking past both held messages releases them; the queue's FIFO order must be + // exactly the merge's decision order - b1, a1, a2 - with b2 still held. + listener.received(consumerP1.consumer, message(p1, "b2", 300L, 1L)) + val everythingArrived = allDelivered.await(30, TimeUnit.SECONDS) + + assertTrue( + writeInFlight, + offeringThreadFree, + mergeUsableDuringWrite, + everythingArrived, + delivered.asScala.toVector == Vector("b1", "a1", "a2"), + consumerP1.acknowledged.asScala.toVector == Vector("b1"), + consumerP0.acknowledged.asScala.toVector == Vector("a1", "a2") + ) ?? (s"writeInFlight=$writeInFlight offeringThreadFree=$offeringThreadFree " + + s"mergeUsableDuringWrite=$mergeUsableDuringWrite delivered=${delivered.asScala.toVector}") + }, + test("a pass-through session is NOT serialized - the ordinary path pays nothing") { + // The lock exists to preserve an order the merge decided. A session with no merge + // decided no order, so taking a session-wide lock per message there would serialize + // every partition's deserialization for nothing. + // + // DETERMINISTIC, not statistical: thread A parks INSIDE `inOrder` until thread B has + // ALSO entered it. Lock-free, B walks straight in and releases A - overlap proven. + // Under any session-wide lock B can never enter while A is inside, the await times + // out, and the test fails - it cannot pass by a lucky scheduling and cannot flake on + // a single-core runner the way "count how often four spinning threads overlapped" + // could. + val ordering = StartFromOrdering.passThrough[String] + val firstInside = CountDownLatch(1) + val secondEntered = CountDownLatch(1) + val overlapped = AtomicBoolean(false) + + val first = worker("pass-through-first") { + ordering.inOrder { + firstInside.countDown() + overlapped.set(secondEntered.await(30, TimeUnit.SECONDS)) + } + } + val second = worker("pass-through-second") { + firstInside.await(30, TimeUnit.SECONDS) + ordering.inOrder { + secondEntered.countDown() + } + } + first.start() + second.start() + first.join(60_000) + second.join(60_000) + + assertTrue(overlapped.get) ?? + "a pass-through ordering layer took a session-wide lock it has no order to protect" + } + ) + + def spec = suite(this.getClass.toString)(observerSuite, budgetSuite, terminalSuite, orderSuite) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/sessionResourceSafetyTest.scala b/server/src/test/scala/consumer/session_runner/sessionResourceSafetyTest.scala new file mode 100644 index 000000000..3bf566c7b --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/sessionResourceSafetyTest.scala @@ -0,0 +1,430 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import java.util.concurrent.ConcurrentLinkedQueue +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** A CONSUMER SESSION OWNS BROKER RESOURCES, and every path that stops owning them has to release + * them. + * + * A session holds one Pulsar consumer per physical topic (each with a live subscription and its own + * listener thread) and one GraalVM engine with its JS contexts. Four paths used to drop the handle + * without releasing anything: + * + * - a target subscribing to several topics built them in a plain `map`, so a failure on the third + * topic left the first two subscribed and unreachable - the partly-built runner was never + * returned, so nothing could close them; + * - the same one level up: a session builds one runner per enabled target; + * - creating a session under a name that already existed simply overwrote the old runner, whose + * consumers went on consuming for the life of the process; + * - stopping swallowed unsubscribe failures and never closed the consumers, the Graal contexts or + * the client's response stream at all. + * + * The broker sits behind plain functions and proxy consumers, so all of it runs offline. + */ +object sessionResourceSafetyTest extends ZIOSpecDefault: + + private val topicA = "persistent://public/default/res-a" + private val topicB = "persistent://public/default/res-b" + + private val buildSuite = suite("building a set of resources, all or nothing")( + test("a failure part-way through RELEASES everything already built") { + // THE leak. Three subscriptions, the third refused: without this the first two stayed + // subscribed with nothing holding a handle to them. + val released = ConcurrentLinkedQueue[String]() + val result = Try(buildAllOrRelease[String, String]( + inputs = Vector("a", "b", "boom"), + build = input => if input == "boom" then throw new RuntimeException("broker refused") else s"consumer-$input", + release = released.add(_) + )) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.getMessage.contains("broker refused")), + released.asScala.toVector == Vector("consumer-a", "consumer-b") + ) ?? s"result=$result released=${released.asScala.toVector}" + }, + test("the ORIGINAL failure propagates, not one thrown while cleaning up") { + // Cleaning up is second-chance work: a close that fails on the way out must not replace + // the cause of the failure with a consequence of it. + val released = ConcurrentLinkedQueue[String]() + val result = Try(buildAllOrRelease[String, String]( + inputs = Vector("a", "b", "boom"), + build = input => if input == "boom" then throw new RuntimeException("broker refused") else s"consumer-$input", + release = resource => + released.add(resource) + throw new IllegalStateException("close also failed") + )) + assertTrue( + result.failed.toOption.exists(_.getMessage.contains("broker refused")), + released.asScala.toVector == Vector("consumer-a", "consumer-b") + ) ?? s"result=$result released=${released.asScala.toVector}" + }, + test("nothing is released when everything builds") { + val released = ConcurrentLinkedQueue[String]() + val built = buildAllOrRelease[String, String](Vector("a", "b"), input => s"consumer-$input", released.add(_)) + assertTrue(built == Vector("consumer-a", "consumer-b"), released.asScala.toVector.isEmpty) + }, + test("the very first failing resource releases nothing and still fails") { + val released = ConcurrentLinkedQueue[String]() + val result = Try(buildAllOrRelease[String, String](Vector("boom"), _ => throw new RuntimeException("no"), released.add(_))) + assertTrue(result.isFailure, released.asScala.toVector.isEmpty) + } + ) + + /** A consumer that records what was done to it, and can refuse to unsubscribe or to close. */ + private final class RecordingConsumer(topicFqn: String, unsubscribeFails: Boolean = false, closeFails: Boolean = false): + val unsubscribed = AtomicBoolean(false) + val closed = AtomicBoolean(false) + val paused = AtomicBoolean(false) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "unsubscribe" => + unsubscribed.set(true) + if unsubscribeFails then throw new RuntimeException(s"cannot unsubscribe from $topicFqn") + null + case "close" => + closed.set(true) + if closeFails then throw new RuntimeException(s"cannot close the consumer for $topicFqn") + null + case "pause" => paused.set(true); null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private def targetRunner(pool: ConsumerSessionContextPool, consumers: Map[String, Consumer[Array[Byte]]]): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = consumers.keys.toVector)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = consumers.keys.toVector, + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = consumers, + pauseArbiters = (consumers).map((fqn, c) => fqn -> ConsumerPauseArbiter(c)), + consumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session( + sessionName: String, + pool: ConsumerSessionContextPool, + consumers: Map[String, Consumer[Array[Byte]]] + ): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = sessionName, + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = pool, + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> targetRunner(pool, consumers)) + ) + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + val completed = AtomicBoolean(false) + override def onNext(value: consumerPb.ResumeResponse): Unit = () + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = completed.set(true) + + private val stopSuite = suite("stopping a session releases what it holds")( + test("a consumer is CLOSED and not merely unsubscribed") { + // Unsubscribing deletes the subscription; the consumer object, its connection and its + // listener thread are only released by closing it. Stopping did the first and not the + // second, so every session ever stopped leaked its consumers. + val pool = ConsumerSessionContextPool() + val a = RecordingConsumer(topicA) + val runner = session("cs-stop", pool, Map(topicA -> a.consumer)) + + runner.stop() + + assertTrue(a.unsubscribed.get, a.closed.get) ?? + s"unsubscribed=${a.unsubscribed.get} closed=${a.closed.get}" + }, + test("a consumer that refuses to unsubscribe is still CLOSED, and the failure is reported") { + // Reported, because deleting the session used to answer OK while the subscription it + // failed to delete stayed on the broker; closed, because a failed unsubscribe must not + // strand the consumer as well. + val pool = ConsumerSessionContextPool() + val a = RecordingConsumer(topicA, unsubscribeFails = true) + val b = RecordingConsumer(topicB) + val runner = session("cs-stop-fail", pool, Map(topicA -> a.consumer, topicB -> b.consumer)) + + val result = Try(runner.stop()) + + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.getMessage.contains(topicA)), + a.closed.get, + // The other consumer must not be stranded by its neighbour's failure. + b.unsubscribed.get, + b.closed.get + ) ?? s"result=$result aClosed=${a.closed.get} bUnsubscribed=${b.unsubscribed.get} bClosed=${b.closed.get}" + }, + test("the session's GraalVM contexts are closed") { + // One engine and one JS context per session, held for the session's whole life. Nothing + // closed them, so every session ever created leaked a Graal context. + val pool = ConsumerSessionContextPool() + val runner = session("cs-stop-graal", pool, Map(topicA -> RecordingConsumer(topicA).consumer)) + val stillUsable = Try(pool.getContext(0).context.eval("js", "1 + 1")).isSuccess + + runner.stop() + + assertTrue(stillUsable, Try(pool.getContext(0).context.eval("js", "1 + 1")).isFailure) ?? + "the session's JS context was still open after the session was stopped" + }, + test("A CONSUMER THAT REFUSES TO CLOSE IS REPORTED, not silently left running") { + // The close was wrapped in a bare `Try` whose result was discarded, so a consumer still + // connected and still holding its listener thread was invisible: `deleteConsumer` + // answered OK with the consumer very much alive. + val pool = ConsumerSessionContextPool() + val stubborn = RecordingConsumer(topicA, closeFails = true) + val runner = session("cs-stop-close-fail", pool, Map(topicA -> stubborn.consumer)) + + val result = Try(runner.stop()) + + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.getMessage.contains(topicA)), + result.failed.toOption.exists(_.getMessage.toLowerCase.contains("close")), + // Unsubscribing still happened - the failure is the close, and only the close. + stubborn.unsubscribed.get + ) ?? s"result=$result" + }, + test("a consumer that refuses BOTH reports both failures") { + val pool = ConsumerSessionContextPool() + val stubborn = RecordingConsumer(topicA, unsubscribeFails = true, closeFails = true) + val runner = session("cs-stop-both-fail", pool, Map(topicA -> stubborn.consumer)) + + val message = Try(runner.stop()).failed.toOption.map(_.getMessage).getOrElse("") + + assertTrue(message.toLowerCase.contains("unsubscribe"), message.toLowerCase.contains("close")) ?? s"message=$message" + }, + test("the client's response stream is completed") { + // The stored observer was left open: the browser kept a stream to a session that no + // longer exists and was never told it had ended. + val pool = ConsumerSessionContextPool() + val runner = session("cs-stop-observer", pool, Map(topicA -> RecordingConsumer(topicA).consumer)) + val observer = RecordingObserver() + runner.resume(observer, isDebug = false) + + runner.stop() + + assertTrue(observer.completed.get) + } + ) + + private val replaceSuite = suite("creating a session over one that already exists")( + test("the session it REPLACES is stopped, not abandoned") { + // Creating twice under one name (the browser re-creating on a config change) used to + // overwrite the entry. The old runner's consumers stayed subscribed and delivering, with + // nothing left holding a handle to them. + val sessions = new java.util.concurrent.ConcurrentHashMap[String, ConsumerSessionRunner]() + val oldConsumer = RecordingConsumer(topicA) + val oldPool = ConsumerSessionContextPool() + val newConsumer = RecordingConsumer(topicA) + val replacement = session("cs-dup", ConsumerSessionContextPool(), Map(topicA -> newConsumer.consumer)) + + storeConsumerSession(sessions, "cs-dup", session("cs-dup", oldPool, Map(topicA -> oldConsumer.consumer))) + storeConsumerSession(sessions, "cs-dup", replacement) + + assertTrue( + oldConsumer.unsubscribed.get, + oldConsumer.closed.get, + !newConsumer.closed.get, + sessions.get("cs-dup") eq replacement + ) ?? s"oldClosed=${oldConsumer.closed.get} newClosed=${newConsumer.closed.get}" + }, + test("a first create under a fresh name stops nothing") { + val sessions = new java.util.concurrent.ConcurrentHashMap[String, ConsumerSessionRunner]() + val consumer = RecordingConsumer(topicA) + val runner = session("cs-fresh", ConsumerSessionContextPool(), Map(topicA -> consumer.consumer)) + + storeConsumerSession(sessions, "cs-fresh", runner) + + assertTrue(!consumer.closed.get, sessions.get("cs-fresh") eq runner) + }, + test("a replaced session that fails to stop is still replaced") { + // Otherwise one undeletable subscription would make the name permanently unusable. + val sessions = new java.util.concurrent.ConcurrentHashMap[String, ConsumerSessionRunner]() + val stubborn = RecordingConsumer(topicA, unsubscribeFails = true) + val replacement = session("cs-dup-fail", ConsumerSessionContextPool(), Map(topicB -> RecordingConsumer(topicB).consumer)) + + storeConsumerSession(sessions, "cs-dup-fail", session("cs-dup-fail", ConsumerSessionContextPool(), Map(topicA -> stubborn.consumer))) + val result = Try(storeConsumerSession(sessions, "cs-dup-fail", replacement)) + + assertTrue(result.isSuccess, stubborn.closed.get, sessions.get("cs-dup-fail") eq replacement) + } + ) + + /** A pool that refuses to give its contexts up. Subclassed rather than mocked, so the real + * aggregation path in `ConsumerSessionRunner.stop` is the thing under test. */ + private final class RefusingPool extends ConsumerSessionContextPool(isDebug = false): + override def close(): Vector[String] = Vector("JS context 0: still executing on another thread") + + /** Real clients aimed at a closed port, so a construction failure is a real broker failure + * rather than a mock's idea of one. */ + private def withOfflineClients[A](f: (org.apache.pulsar.client.api.PulsarClient, org.apache.pulsar.client.admin.PulsarAdmin) => A): A = + val client = org.apache.pulsar.client.api.PulsarClient.builder + .serviceUrl("pulsar://127.0.0.1:1") + .operationTimeout(2, java.util.concurrent.TimeUnit.SECONDS) + .build + val admin = org.apache.pulsar.client.admin.PulsarAdmin.builder + .serviceHttpUrl("http://127.0.0.1:1") + .connectionTimeout(2, java.util.concurrent.TimeUnit.SECONDS) + .readTimeout(2, java.util.concurrent.TimeUnit.SECONDS) + .requestTimeout(2, java.util.concurrent.TimeUnit.SECONDS) + .build + try f(client, admin) + finally + Try(client.close()) + Try(admin.close()) + + private val oneEnabledTarget = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector(ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicA))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + )), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def poolIsClosed(pool: ConsumerSessionContextPool): Boolean = + Try(pool.getContext(0).context.eval("js", "1 + 1")).isFailure + + private val constructionSuite = suite("a session that cannot be built releases what it had already taken")( + test("A TARGET THAT FAILS TO BUILD CLOSES THE SESSION'S GRAAL POOL") { + // The pool is the FIRST thing a session takes and was created outside the all-or-nothing + // guard, so a target that failed to resolve its topics released the targets built before + // it and left a whole GraalVM engine open with nothing holding a handle to it. The + // browser retries a failed create, so this leaked an engine per attempt. + val pool = ConsumerSessionContextPool() + val usableBefore = !poolIsClosed(pool) + + val result = withOfflineClients((client, admin) => + Try(ConsumerSessionRunner.make( + pulsarClient = client, + adminClient = admin, + sessionName = "cs-build-fail", + sessionConfig = oneEnabledTarget, + sessionContextPool = pool + )) + ) + + assertTrue(usableBefore, result.isFailure, poolIsClosed(pool)) ?? + s"result=$result poolClosed=${poolIsClosed(pool)}" + }, + test("a session with no enabled targets closes the pool as well") { + val pool = ConsumerSessionContextPool() + val result = withOfflineClients((client, admin) => + Try(ConsumerSessionRunner.make( + pulsarClient = client, + adminClient = admin, + sessionName = "cs-no-targets", + sessionConfig = oneEnabledTarget.copy(targets = Vector.empty), + sessionContextPool = pool + )) + ) + assertTrue(result.isFailure, poolIsClosed(pool)) + } + ) + + private val aggregationSuite = suite("stopping reports everything it could not release")( + test("A GRAAL POOL THAT WILL NOT CLOSE IS REPORTED, not swallowed twice over") { + // Swallowed once inside `close` and then discarded again by the caller, so a JS context + // holding its heap for the life of the process was reported to the client as released. + val runner = session("cs-stop-pool-fail", RefusingPool(), Map(topicA -> RecordingConsumer(topicA).consumer)) + + val result = Try(runner.stop()) + + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.getMessage.contains("JS context 0")) + ) ?? s"result=$result" + }, + test("an UNEXPECTED throw out of a target's stop is a failure, not an empty result") { + // `Try(target.stop()).getOrElse(Vector.empty)` read "this target blew up" as "this + // target released everything cleanly". + val pool = ConsumerSessionContextPool() + val exploding = new ConsumerSessionTargetRunner( + targetIndex = 7, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(topicA))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(topicA), + schemasByTopic = Map.empty, + sessionContextPool = pool, + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())), + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ): + override def stop(quarantine: CleanupQuarantine): Vector[String] = + throw new IllegalStateException("the target could not be released at all") + + val runner = session("cs-stop-target-throws", pool, Map.empty).copy(targets = Map(7 -> exploding)) + + val result = Try(runner.stop()) + + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.getMessage.contains("could not be released at all")), + result.failed.toOption.exists(_.getMessage.contains("target 7")) + ) ?? s"result=$result" + } + ) + + def spec = + suite(this.getClass.toString)(buildSuite, stopSuite, replaceSuite, constructionSuite, aggregationSuite) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/startFromBrokerFailureTest.scala b/server/src/test/scala/consumer/session_runner/startFromBrokerFailureTest.scala new file mode 100644 index 000000000..42940352f --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromBrokerFailureTest.scala @@ -0,0 +1,260 @@ +package consumer.session_runner + +import org.apache.pulsar.client.admin.PulsarAdminException +import org.apache.pulsar.client.api.Consumer +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.TimeoutException +import scala.util.Try + +/** A BROKER THAT COULD NOT ANSWER IS NOT AN EMPTY TOPIC. + * + * Every start-from position that has to be looked up went through `Try(...).toOption`, so a + * timeout, a 401, a 404 or a broker restarting mid-request produced exactly the same `None` as + * "this log holds nothing there" - and every caller reads that `None` as an answer: + * + * - `resolveLatestN` reads it as "the log holds fewer than n messages" and seeks to EARLIEST, so + * a transient 500 turned "the latest 5 messages" into the entire backlog; + * - the approximate-entry seek reads it as "that entry is gone" and falls back to EARLIEST; + * - the approximate-time span reads it as "this partition holds nothing" and computes the cutoff + * from only the partitions that happened to answer; + * - the global merge reads a failed `getLastMessageIds` as "this stream is already drained" and + * stops waiting for it, so a whole partition can be left out of a global skip or latest. + * + * In every case the session was created successfully and started somewhere the user did not ask + * for. The two states are kept apart by [[isEmptyLogAnswer]], whose classification is MEASURED - + * see its scaladoc for the exact status codes and reasons this Pulsar answers with. + */ +object startFromBrokerFailureTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/failing" + private val otherTopicFqn = "persistent://public/default/failing-other" + + /** The measured 412 an EMPTY topic answers `examinemessage` with. */ + private def emptyTopicError: Throwable = + PulsarAdminException.PreconditionFailedException( + new RuntimeException("Could not examine messages due to the total message is zero"), + "Could not examine messages due to the total message is zero", + 412 + ) + + /** The measured 500 a walk that ran PAST THE START of the log answers with. */ + private val pastStartReason = + "\n --- An unexpected error occurred in the server ---\n\nMessage: Incorrect parameter input error code: -14\n\n" + + "Stacktrace:\n\norg.apache.bookkeeper.mledger.ManagedLedgerException: Incorrect parameter input error code: -14" + + private def pastStartOfLogError: Throwable = + PulsarAdminException.ServerSideErrorException(new RuntimeException("past the start"), pastStartReason, pastStartReason, 500) + + /** A 500 that means the broker is unwell, not that the log ran out. */ + private val serverErrorReason = "\n --- An unexpected error occurred in the server ---\n\nMessage: Failed to get managed ledger" + + private def serverError: Throwable = + PulsarAdminException.ServerSideErrorException(new RuntimeException("boom"), serverErrorReason, serverErrorReason, 500) + + /** MEASURED: a broker that cannot be reached at all reports the SAME statusCode 500 as a walk + * that ran off the start of the log, with a null `httpError`. It is the one shape that makes + * classifying on the status code alone unsafe. */ + private def unreachableBrokerError: Throwable = + PulsarAdminException( + new java.util.concurrent.CompletionException(new RuntimeException("retries exhausted")), + "java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: " + + "Could not complete the operation. Number of retries has been exhausted. Failed reason: connection refused", + 500 + ) + + private val classificationSuite = suite("telling an empty log from a broker that could not answer")( + test("the measured EMPTY-TOPIC answer is an answer") { + assertTrue(isEmptyLogAnswer(emptyTopicError)) + }, + test("the measured PAST-THE-START answer is an answer") { + assertTrue(isEmptyLogAnswer(pastStartOfLogError)) + }, + test("an answer buried in a cause chain is still recognised") { + // The admin client wraps, and so does everything between here and it. + val wrapped = new RuntimeException("Failed to resolve start position", new RuntimeException("wrapper", pastStartOfLogError)) + assertTrue(isEmptyLogAnswer(wrapped)) + }, + test("an unrelated server error is NOT an empty log") { + // THE defect: this used to be indistinguishable from an empty topic. + assertTrue(!isEmptyLogAnswer(serverError)) + }, + test("an UNREACHABLE BROKER is not an empty log, although it reports the same 500") { + // MEASURED, and the reason the status code alone cannot decide this: a broker that was + // never reached answers with statusCode 500 exactly as a walk past the start of the log + // does. Classifying 500 as "nothing there" would have turned every connection failure + // into a silent seek to earliest. + assertTrue(!isEmptyLogAnswer(unreachableBrokerError)) + }, + test("a timeout, an authorization failure and a missing topic are NOT empty logs") { + val timeout = new TimeoutException("Request timed out after 30000 ms") + val notAuthorized = PulsarAdminException.NotAuthorizedException(new RuntimeException("no"), "Don't have permission", 401) + val notFound = PulsarAdminException.NotFoundException(new RuntimeException("no"), "Topic not found", 404) + val misclassified = Vector[Throwable](timeout, notAuthorized, notFound).filter(isEmptyLogAnswer) + assertTrue(misclassified.isEmpty) ?? s"read as an empty log: ${misclassified.map(_.getMessage)}" + }, + test("an exception carrying no message at all is NOT an empty log") { + assertTrue(!isEmptyLogAnswer(new RuntimeException())) + }, + test("a cause chain that loops terminates instead of hanging") { + // Bounded on purpose: this runs on the session-creation path, and the JVM permits a + // cycle of length two even though it refuses direct self-causation. + val first = new RuntimeException("round") + val second = new RuntimeException("and round") + first.initCause(second) + second.initCause(first) + assertTrue(!isEmptyLogAnswer(first)) + } + ) + + private val lookupSuite = suite("asking the broker one question")( + test("an answer comes back as an answer") { + assertTrue(brokerAnswer("examining an entry", topicFqn)("entry-1") == Some("entry-1")) + }, + test("'there is nothing there' comes back as None, for both of its shapes") { + val empty = brokerAnswer[String]("examining an entry", topicFqn)(throw emptyTopicError) + val pastStart = brokerAnswer[String]("examining an entry", topicFqn)(throw pastStartOfLogError) + assertTrue(empty.isEmpty, pastStart.isEmpty) + }, + test("a broker that could not answer FAILS, naming the topic and the question") { + // A user has to be able to tell "your topic is empty" from "the broker is unwell", and + // the only place that can be said is here. + val cause = serverError + val result = Try(brokerAnswer[String]("examining an entry", topicFqn)(throw cause)) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + result.failed.toOption.exists(_.getMessage.contains(topicFqn)), + result.failed.toOption.exists(_.getMessage.contains("examining an entry")), + // The cause is kept, so the operator sees the broker's own words in the log. + result.failed.toOption.flatMap(err => Option(err.getCause)).exists(_ eq cause) + ) ?? s"result=$result" + } + ) + + /** The production shape: `resolveLatestN` walking entries back from the end of a log, with the + * broker behind [[brokerAnswer]] exactly as `entryFromLatest` puts it there. */ + private def walkBackFrom(answers: Long => Any): String => Long => Option[LogEntry[String]] = + _ => k => brokerAnswer("examining the entry", topicFqn)(answers(k)).map(_ => LogEntry(s"entry-$k", 1_000L, 1)) + + /** `entry-$k` counts back from the end, so a larger ordinal is strictly OLDER - the same order + * `MessageIdImpl.compareTo` gives real entry ids. */ + private def olderByEntryOrdinal(a: String, b: String): Boolean = a.split("-").last.toInt > b.split("-").last.toInt + + private def resolveOne(n: Long, lookup: String => Long => Option[LogEntry[String]]): LatestNSeek[String] = + resolveLatestN(n, Vector(topicFqn), lookup, olderByEntryOrdinal)(topicFqn) + + private val latestNSuite = suite("latest n: a failed lookup must not become 'the log is exhausted'")( + test("A TRANSIENT BROKER FAILURE FAILS THE RESOLUTION instead of seeking to earliest") { + // THE defect. `entryFromLatest` answering None means "this log has no entry there", and + // the walk reads that as the end of the log - so the caller shows all of it. A 500 on + // the third entry of a large log therefore turned "the latest 5" into the whole backlog + // - with the session reporting success. + val lookup = walkBackFrom(k => if k >= 3 then throw serverError else s"message-$k") + val result = Try(resolveOne(5, lookup)) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]) + ) ?? s"a broker failure resolved to $result, which the caller reads as 'show the whole log'" + }, + test("a log genuinely shorter than n still shows all of it") { + // The control, and the reason the two states cannot simply be merged into a failure: + // this is an ordinary, correct outcome - anchored at the oldest entry the walk took, + // so the retention re-check can later prove the anchor still exists. + val lookup = walkBackFrom(k => if k > 3 then throw pastStartOfLogError else s"message-$k") + assertTrue(resolveOne(5, lookup) == LatestNSeek.FromEntry("entry-3", 0L)) + }, + test("an empty log maps to EVERYTHING - its whole content at seek time is post-inspection live traffic") { + // The classifier still answers None (not a failure); the MAPPING is what changed: + // seeking earliest preserves anything appended between the inspection and the seek, + // where seek-time "latest" silently lost it. + val lookup = walkBackFrom(_ => throw emptyTopicError) + assertTrue(resolveOne(5, lookup) == LatestNSeek.Everything) + }, + test("a log long enough still resolves exactly") { + val lookup = walkBackFrom(k => if k > 10 then throw pastStartOfLogError else s"message-$k") + assertTrue(resolveOne(3, lookup) == LatestNSeek.FromEntry("entry-3", 0L)) + } + ) + + private val timeSpanSuite = suite("approximate publish time: a partition that could not be read must not be dropped from the range")( + test("A PARTITION THE BROKER COULD NOT READ FAILS THE RESOLUTION") { + // The range is min(first) .. max(last) across every partition. Silently leaving out the + // partition that failed produced a confident cutoff over a narrower range - a different + // position, reported as success. + val spanOf = (topic: String) => + brokerAnswer("reading the publish-time span", topic) { + if topic == otherTopicFqn then throw serverError else TopicPublishTimeSpan(1_000L, 2_000L) + } + val result = Try(resolveApproximatePublishTimePosition(0.5, Vector(topicFqn, otherTopicFqn), spanOf)) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]) + ) ?? s"the cutoff was computed from only the partitions that answered: $result" + }, + test("a partition that genuinely holds nothing is still skipped, as it must be") { + // An empty partition has no retained boundary entry; counting it as time zero would + // drag the range back to 1970. + val spanOf = (topic: String) => + brokerAnswer("reading the publish-time span", topic) { + if topic == otherTopicFqn then throw emptyTopicError else TopicPublishTimeSpan(1_000L, 2_000L) + } + assertTrue( + resolveApproximatePublishTimePosition(0.5, Vector(topicFqn, otherTopicFqn), spanOf) == + ApproximatePublishTimeSeek.Timestamp(1_500L) + ) + } + ) + + /** A consumer whose `getLastMessageIds` behaves as told. */ + private def consumerOn(topicFqn: String, lastMessageIds: () => java.util.List[org.apache.pulsar.client.api.MessageId]): Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => "cs-failing-0" + case "getLastMessageIds" => lastMessageIds() + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + private val streamsSuite = suite("global ordering: a stream whose end could not be read must not be called drained")( + test("A BROKER THAT WILL NOT SAY WHERE A PARTITION ENDS FAILS THE SESSION") { + // A stream recorded as "empty" is never waited for, so a global skip or latest simply + // leaves that whole partition out of the merge - it delivers messages that are neither + // counted nor ordered against the rest. + val consumer = consumerOn(topicFqn, () => throw serverError) + val result = Try(startFromStreamsAt(Vector(consumer))) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[StartFromUnresolvableException]), + result.failed.toOption.exists(_.getMessage.contains(topicFqn)) + ) ?? s"result=$result" + }, + test("a NON-PERSISTENT topic is still recorded as drained, without asking") { + // It retains nothing by definition, so there is no end to read and nothing to wait for. + // Decided from the FQN rather than from whatever the call happens to throw. + val nonPersistent = "non-persistent://public/default/live-only" + val consumer = consumerOn(nonPersistent, () => throw new UnsupportedOperationException("must not be asked")) + val streams = startFromStreamsAt(Vector(consumer)) + assertTrue(streams.map(_.lastAtStart) == Vector(EntryPosition.empty)) + }, + test("a partition that answers is recorded at the end it reported") { + val consumer = consumerOn( + topicFqn, + () => java.util.List.of(new org.apache.pulsar.client.impl.MessageIdImpl(7L, 3L, 0)) + ) + assertTrue(startFromStreamsAt(Vector(consumer)).map(_.lastAtStart) == Vector(EntryPosition(7L, 3L, -1, 1))) + }, + test("an EMPTY partition answers with MessageId.earliest and is drained from the start") { + val consumer = consumerOn(topicFqn, () => java.util.List.of(org.apache.pulsar.client.api.MessageId.earliest)) + assertTrue(startFromStreamsAt(Vector(consumer)).map(_.lastAtStart) == Vector(EntryPosition.empty)) + } + ) + + def spec = suite(this.getClass.toString)(classificationSuite, lookupSuite, latestNSuite, timeSpanSuite, streamsSuite) diff --git a/server/src/test/scala/consumer/session_runner/startFromCountValidationTest.scala b/server/src/test/scala/consumer/session_runner/startFromCountValidationTest.scala new file mode 100644 index 000000000..7b9ffb424 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromCountValidationTest.scala @@ -0,0 +1,283 @@ +package consumer.session_runner + +import _root_.consumer.start_from.{ + ApproximateEntryPosition, + ApproximatePublishTimePosition, + ConsumerSessionStartFrom, + DateTime, + DateTimeUnit, + EarliestMessage, + LatestMessage, + MessageId, + NthMessageAfterEarliest, + NthMessageBeforeLatest, + RelativeDateTime +} +import org.apache.pulsar.client.admin.PulsarAdmin +import org.apache.pulsar.client.api.PulsarClient +import zio.test.* + +import java.util.concurrent.TimeUnit +import scala.util.Try + +/** THE COUNT ON A COUNTING START-FROM IS A NUMBER SOMEBODY TYPED, and it arrives over gRPC as a + * plain `int64` that any client can fill in with anything. + * + * The server used to CLAMP rather than refuse, and a clamp answers a different question without + * saying so: + * + * - "skip the first -1 messages" was read as EARLIEST - the whole topic; + * - "the latest -1 messages" was read as LATEST - nothing retained at all; + * - "the latest 3,000,000,000 messages" was truncated to `Int.MaxValue` inside the retain heap. + * + * Each of those is a valid position the user did not ask for, delivered with a successful session. + * + * THERE IS DELIBERATELY NO UPPER BOUND ON SKIP-N. Skipping n messages is O(n) by nature - Pulsar + * keeps no message-ordinal index - so any cap would be an arbitrary number rather than a limit of + * the design, and the progress API exists precisely so a long skip can be watched. The bound on + * LATEST-N is not a policy choice: its retained set is an in-memory heap of exactly n, indexed by + * Int, so a larger n cannot be represented at all. + */ +object startFromCountValidationTest extends ZIOSpecDefault: + + private def reason(startFrom: ConsumerSessionStartFrom): Option[String] = startFromCountRejectionReason(startFrom) + + private val rejectionSuite = suite("which counts the server refuses")( + test("a NEGATIVE skip-n is refused, not read as 'start at the beginning'") { + val refusals = Vector(-1L, -5L, Long.MinValue).map(n => n -> reason(NthMessageAfterEarliest(n = n))) + assertTrue(refusals.forall((_, why) => why.isDefined)) ?? s"$refusals" + }, + test("a NEGATIVE latest-n is refused, not read as 'show nothing'") { + val refusals = Vector(-1L, -5L, Long.MinValue).map(n => n -> reason(NthMessageBeforeLatest(n = n))) + assertTrue(refusals.forall((_, why) => why.isDefined)) ?? s"$refusals" + }, + test("the refusal says which control was wrong and what it was set to") { + // A session can only be fixed if the error names the field. Both counting modes carry an + // n, so "n must be positive" on its own is not enough. + val skip = reason(NthMessageAfterEarliest(n = -3)).getOrElse("") + val latest = reason(NthMessageBeforeLatest(n = -3)).getOrElse("") + assertTrue( + skip.contains("-3") && skip.toLowerCase.contains("skip"), + latest.contains("-3") && latest.toLowerCase.contains("latest"), + skip != latest + ) ?? s"skip=$skip latest=$latest" + }, + test("ZERO is accepted by both - it is a real position, not a mistake") { + // "skip nothing" is the beginning, and "the latest 0 messages" is the live tail. + assertTrue(reason(NthMessageAfterEarliest(n = 0)).isEmpty, reason(NthMessageBeforeLatest(n = 0)).isEmpty) + }, + test("SKIP-N HAS NO UPPER BOUND - not even an enormous one is refused") { + // Deliberate. Skipping is O(n) whatever the number, the progress API exists to show it + // happening, and a cap would be an invented limit rather than a real one. + val enormous = Vector(1_000_000L, Int.MaxValue.toLong + 1, Long.MaxValue) + val wronglyRefused = enormous.filter(n => reason(NthMessageAfterEarliest(n = n)).isDefined) + assertTrue(wronglyRefused.isEmpty) ?? s"a cap was introduced on skip-n: $wronglyRefused" + }, + test("a latest-n above the OPERATIONAL boundary is REFUSED, not truncated") { + // It used to be silently narrowed to Int.MaxValue, which answers a different request - + // and Int.MaxValue itself, a leftover of a heap that no longer exists, still admitted a + // request the server would grind on for hours holding the lifecycle lock: the walk + // costs one synchronous broker lookup per entry with no progress to show. The boundary + // is an operational one now, and the refusal points at skip-n, which streams. + val why = reason(NthMessageBeforeLatest(n = latestNMaxAccepted + 1)) + val enormous = reason(NthMessageBeforeLatest(n = Int.MaxValue.toLong)) + assertTrue( + why.isDefined, + why.exists(_.contains(latestNMaxAccepted.toString)), + enormous.isDefined + ) ?? s"why=$why enormous=$enormous" + }, + test("a latest-n of exactly the boundary is still accepted") { + assertTrue(reason(NthMessageBeforeLatest(n = latestNMaxAccepted)).isEmpty) + }, + test("every mode that carries no count is untouched") { + val countless: Vector[ConsumerSessionStartFrom] = Vector( + EarliestMessage(), + LatestMessage(), + DateTime(dateTime = java.time.Instant.EPOCH), + RelativeDateTime(value = 1, unit = DateTimeUnit.Hour, isRoundedToUnitStart = false), + MessageId(messageIdBytes = Array.empty), + ApproximateEntryPosition(fraction = 0.6), + ApproximatePublishTimePosition(fraction = 0.6) + ) + val wronglyRefused = countless.filter(mode => reason(mode).isDefined) + assertTrue(wronglyRefused.isEmpty) ?? s"${wronglyRefused.map(_.getClass.getSimpleName)}" + } + ) + + /** Real clients aimed at a closed port. An empty consumer set reaches no broker on this path, so + * an accidental broker call would surface as a connection error rather than as an NPE that + * makes the assertion pass for the wrong reason. */ + private def withOfflineClients[A](f: (PulsarClient, PulsarAdmin) => A): A = + val client = PulsarClient.builder.serviceUrl("pulsar://127.0.0.1:1").operationTimeout(2, TimeUnit.SECONDS).build + val admin = PulsarAdmin.builder + .serviceHttpUrl("http://127.0.0.1:1") + .connectionTimeout(2, TimeUnit.SECONDS) + .readTimeout(2, TimeUnit.SECONDS) + .requestTimeout(2, TimeUnit.SECONDS) + .build + try f(client, admin) + finally + Try(client.close()) + Try(admin.close()) + + private def plan(startFrom: ConsumerSessionStartFrom): Try[StartFromPlan] = + withOfflineClients((client, admin) => + Try(handleStartFrom( + startFrom = startFrom, + consumers = Vector.empty, + adminClient = admin, + pulsarClient = client, + nonPartitionedTopicFqns = Vector.empty + )) + ) + + private val boundarySuite = suite("the refusal happens at the trust boundary")( + test("a negative skip-n fails the session instead of seeking to earliest") { + val result = plan(NthMessageAfterEarliest(n = -5)) + assertTrue( + result.isFailure, + result.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + result.failed.toOption.exists(_.getMessage.contains("-5")) + ) ?? s"result=$result" + }, + test("a negative latest-n fails the session instead of seeking to latest") { + val result = plan(NthMessageBeforeLatest(n = -5)) + assertTrue(result.isFailure, result.failed.toOption.exists(_.getMessage.contains("-5"))) ?? s"result=$result" + }, + test("an ordinary count still plans normally") { + assertTrue(plan(NthMessageAfterEarliest(n = 5)).map(_.discard) == scala.util.Success(StartFromDiscardPlan.SharedTotal(5))) + } + ) + + /** A log of `entries` unbatched entries, newest first, all with the same publish time. */ + private def oneTopic(entries: Int): String => Long => Option[LogEntry[String]] = + _ => k => Option.when(k >= 1 && k <= entries)(LogEntry(s"entry-$k", 1_000L, 1)) + + /** `entry-$k` counts back from the end, so a larger ordinal is strictly OLDER - the same order + * `MessageIdImpl.compareTo` gives real entry ids. */ + private def olderByEntryOrdinal(a: String, b: String): Boolean = a.split("-").last.toInt > b.split("-").last.toInt + + private val memorySuite = suite("LATEST-N BUFFERS NOTHING, whatever n is")( + test("a latest-n session arms no ordering layer, so it can hold nothing at all") { + // THE structural guarantee. Latest-n used to narrow its answer with a top-n heap of n + // DELIVERED messages plus an unbounded queue of live traffic beside it, so its memory + // was a number the user typed and a live message could evict a historical one. The cut + // is now resolved from entry metadata before a single message is delivered. + val armed = Vector(1, 2, 5, 50).map(streams => streams -> needsGlobalOrdering(NthMessageBeforeLatest(n = 1_000_000), streams)) + assertTrue(armed.forall((_, needed) => !needed)) ?? s"latest-n asked for a buffering layer: $armed" + }, + test("the resolved cut is O(topics) in memory - it holds one entry per topic, never n") { + // 1,000,000 messages asked for over 3 topics resolves to at most 3 answers. + val cut = resolveLatestN(1_000_000L, Vector("a", "b", "c"), oneTopic(10), olderByEntryOrdinal) + assertTrue(cut.size == 3) ?? s"cut=$cut" + }, + test("the walk is O(n / batch size) lookups and never touches a message payload") { + var lookups = 0 + val entryFromLatest: String => Long => Option[LogEntry[String]] = _ => + k => + lookups += 1 + Option.when(k >= 1 && k <= 1000)(LogEntry(s"entry-$k", 1_000L - k, 10)) + resolveLatestN(50L, Vector("a"), entryFromLatest, olderByEntryOrdinal) + assertTrue(lookups == 5) ?? s"$lookups lookups for 50 messages over batches of 10" + } + ) + + /** Target-aware refusals: sessions whose retained-log-derived mode cannot mean what it promises on these + * targets are refused at creation, with the reason naming what to change. */ + private val targetAwareSuite = suite("target-aware refusals for retained-log-derived modes")( + test("latest-n is refused when any enabled target reads COMPACTED") { + val refused = readCompactedStartFromRejectionReason(NthMessageBeforeLatest(n = 3), readCompactedTargetIndexes = Vector(1)) + val fine = readCompactedStartFromRejectionReason(NthMessageBeforeLatest(n = 3), readCompactedTargetIndexes = Vector.empty) + assertTrue( + refused.exists(_.contains("compacted")), + refused.exists(_.contains("1")), + fine.isEmpty + ) ?? s"refused=$refused fine=$fine" + }, + test("both percentage modes are refused because their raw-log positions do not describe a compacted view") { + val targetIndexes = Vector(4, 1) + val entry = readCompactedStartFromRejectionReason(ApproximateEntryPosition(0.5), targetIndexes) + val publishTime = readCompactedStartFromRejectionReason(ApproximatePublishTimePosition(0.5), targetIndexes) + assertTrue( + entry.exists(message => message.contains("Approximate position (% of data)") && message.contains("1, 4") && message.contains("raw stored entries")), + publishTime.exists(message => + message.contains("Approximate position (% of time)") && message.contains("1, 4") && message.contains("raw retained log") + ) + ) ?? s"entry=$entry publishTime=$publishTime" + }, + test("latest-0 and delivery-counted skip remain valid on a compacted view") { + assertTrue( + readCompactedStartFromRejectionReason(NthMessageBeforeLatest(n = 0), Vector(0)).isEmpty, + readCompactedStartFromRejectionReason(NthMessageAfterEarliest(n = 5), Vector(0)).isEmpty + ) + }, + test("without a read-compacted target no mode is refused") { + assertTrue( + readCompactedStartFromRejectionReason(ApproximateEntryPosition(0.5), Vector.empty).isEmpty, + readCompactedStartFromRejectionReason(ApproximatePublishTimePosition(0.5), Vector.empty).isEmpty, + readCompactedStartFromRejectionReason(NthMessageBeforeLatest(n = 3), Vector.empty).isEmpty + ) + }, + test("skip-n is refused when two enabled targets share a physical topic, and the reason NAMES it") { + val shared = "persistent://t/ns/shared-partition-0" + val refused = skipOverlapRejectionReason( + NthMessageAfterEarliest(n = 5), + topicsPerEnabledTarget = Vector(Vector(shared, "persistent://t/ns/a"), Vector(shared)) + ) + assertTrue( + refused.exists(_.contains(shared)), + refused.exists(_.contains("same topic")) + ) ?? s"refused=$refused" + }, + test("skip-n with DISJOINT targets, one target, or n = 0 is not refused") { + assertTrue( + skipOverlapRejectionReason( + NthMessageAfterEarliest(n = 5), + Vector(Vector("persistent://t/ns/a"), Vector("persistent://t/ns/b")) + ).isEmpty, + skipOverlapRejectionReason(NthMessageAfterEarliest(n = 5), Vector(Vector("persistent://t/ns/a"))).isEmpty, + skipOverlapRejectionReason( + NthMessageAfterEarliest(n = 0), + Vector(Vector("persistent://t/ns/a"), Vector("persistent://t/ns/a")) + ).isEmpty, + // Latest-n keeps its per-view duplicate contract: overlap is not its problem. + skipOverlapRejectionReason( + NthMessageBeforeLatest(n = 5), + Vector(Vector("persistent://t/ns/a"), Vector("persistent://t/ns/a")) + ).isEmpty + ) + }, + test("a latest-n anchor trimmed between resolving and seeking is refused, not silently shortened") { + // Entry ids compare as the production comparator does; the topic now retains only + // entry 5, and the resolved anchor was entry 3 - gone. + val cut = Map("persistent://t/ns/a" -> LatestNSeek.FromEntry(3L, 0L)) + val trimmed = latestNAnchorRejectionReason[Long](cut, _ => Some(5L), (a, b) => a < b) + val intact = latestNAnchorRejectionReason[Long](cut, _ => Some(2L), (a, b) => a < b) + val emptied = latestNAnchorRejectionReason[Long](cut, _ => None, (a, b) => a < b) + assertTrue( + trimmed.exists(_.contains("retention removed")), + intact.isEmpty, + emptied.isDefined // retains nothing at all: the anchor is gone by definition + ) ?? s"trimmed=$trimmed intact=$intact emptied=$emptied" + }, + test("an EXHAUSTED CONTRIBUTOR's trimmed anchor is caught - it is FromEntry now, never Everything") { + // The escape this closes: a topic whose whole backlog was counted resolved to + // `Everything`, the re-check only inspected FromEntry, and retention deleting that + // topic's contribution between resolving and seeking silently shrank the result + // below the promised n. Exhausted contributors now carry their oldest-entry anchor, + // so the same trim is refused like any other. + val cut = Map("persistent://t/ns/exhausted" -> LatestNSeek.FromEntry(1L, 0L)) + val trimmed = latestNAnchorRejectionReason[Long](cut, _ => None, (a, b) => a < b) + assertTrue(trimmed.exists(_.contains("retention removed"))) + }, + test("EVERYTHING and NOTHING cuts need no anchor - only FromEntry is re-checked") { + val cut = Map( + "persistent://t/ns/a" -> LatestNSeek.Everything, + "persistent://t/ns/b" -> (LatestNSeek.Nothing: LatestNSeek[Long]) + ) + assertTrue(latestNAnchorRejectionReason[Long](cut, _ => None, (a, b) => a < b).isEmpty) + } + ) + + def spec = suite(this.getClass.toString)(rejectionSuite, boundarySuite, memorySuite, targetAwareSuite) diff --git a/server/src/test/scala/consumer/session_runner/startFromDiscardOnceTest.scala b/server/src/test/scala/consumer/session_runner/startFromDiscardOnceTest.scala new file mode 100644 index 000000000..28a63f349 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromDiscardOnceTest.scala @@ -0,0 +1,353 @@ +package consumer.session_runner + +import java.util.concurrent.atomic.AtomicLong + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.api.{Consumer, Message as PulsarMessage, Schema} +import org.apache.pulsar.client.impl.{MessageIdImpl, MessageImpl} +import org.apache.pulsar.common.api.proto.MessageMetadata +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.nio.ByteBuffer +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.jdk.CollectionConverters.* + +/** The start-from discard must be applied EXACTLY ONCE, at session start. + * + * This is the regression that seek + discard invites: the counter lives next to the pause/resume + * machinery, and re-arming it on resume would silently skip a fresh n messages every time the user + * hits play - a bug that only shows up on the second play and looks like data loss. + * + * Everything here runs offline. `ConsumerSessionTargetRunner.resume` and `pause` are the REAL + * production methods; they are driven with an empty consumer map, so the only work left in them is + * the listener state they touch - which is exactly what is under test. `ConsumerListener.decide` is + * the real decision `received` makes, one call per delivered message. + */ +object startFromDiscardOnceTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/discard-once" + + /** A listener already open for business. It starts CLOSED in production - nothing may be + * consumed before the session is armed and a client has resumed it - and + * `ConsumerSessionTargetRunner.resume` is what opens it. */ + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetConfig(topicFqns: Vector[String]): ConsumerSessionTarget = + ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = topicFqns)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def targetRunner(consumerListener: ConsumerListener, topicFqns: Vector[String] = Vector(topicFqn)): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = targetConfig(topicFqns), + nonPartitionedTopicFqns = topicFqns, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + /** Drives the production `resume` with no-op callbacks - the same call `ConsumerSessionRunner` + * makes on every play. */ + private def resume(runner: ConsumerSessionTargetRunner): Unit = + runner.resume( + onNext = (_, _, _, _, _) => (), + isDebug = false, + incrementNumMessageProcessed = () => (), + onStartFromDiscardProgress = () => (), + admitDelivery = _ => DeliveryAdmission.Prepare + ) + + /** How the listener would treat `count` consecutively delivered messages. */ + private def deliver(l: ConsumerListener, count: Int): Vector[ConsumerListener.Action] = + Vector.fill(count)(l.decide(topicFqn, canAcknowledge = true)) + + import ConsumerListener.Action.* + + def spec = suite(this.getClass.toString)( + test("a listener that has never been resumed consumes NOTHING - it hands every message back") { + // VERIFIED against Pulsar 3.2.1, and the reason this matters: the start-from set-up does + // broker round trips (the backward entry walk, reading each topic's last message id) + // while the session is still being built. If the listener accepted during that window it + // would ACKNOWLEDGE messages into a message handler that is still a no-op, and a session + // could swallow its whole backlog and then deliver nothing at all. A live 3-partition + // "skip first 4" delivered ZERO of its 12 messages until this was closed. + val fresh = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + fresh.startFromDiscard = StartFromDiscard.shared(3) + val beforeAnyResume = deliver(fresh, 5) + assertTrue( + beforeAnyResume.forall(_ == Reject), + fresh.startFromDiscard.remaining == 3L + ) ?? s"an unarmed session consumed messages: $beforeAnyResume" + }, + test("resuming the target is what opens the listener") { + val fresh = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + fresh.startFromDiscard = StartFromDiscard.shared(1) + val runner = targetRunner(fresh) + val closed = deliver(fresh, 1) + resume(runner) + val open = deliver(fresh, 2) + assertTrue(closed == Vector(Reject), open == Vector(Drop, Deliver)) + }, + test("the first n delivered messages are dropped and everything after them is delivered") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + assertTrue(deliver(l, 6) == Vector(Drop, Drop, Drop, Deliver, Deliver, Deliver), l.startFromDiscard.remaining == 0L) + }, + test("resuming a target does not re-arm the discard") { + // The regression: 3 to skip, 2 already skipped, then the user pauses and plays again. + // Re-arming would drop 3 MORE messages here. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = targetRunner(l) + + resume(runner) + val beforePause = deliver(l, 2) + runner.pause() + resume(runner) + val afterResume = deliver(l, 3) + + assertTrue( + beforePause == Vector(Drop, Drop), + afterResume == Vector(Drop, Deliver, Deliver), + l.startFromDiscard.remaining == 0L + ) ?? "exactly 3 messages may ever be dropped, however many times the session is resumed" + }, + test("a discard already spent stays spent across further pause/resume cycles") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(2) + val runner = targetRunner(l) + resume(runner) + deliver(l, 2) + + val laterRounds = (1 to 3).flatMap { _ => + runner.pause() + resume(runner) + deliver(l, 2) + }.toVector + + assertTrue(laterRounds.forall(_ == Deliver), l.startFromDiscard.remaining == 0L) ?? + s"a spent discard was re-armed by resume: $laterRounds" + }, + test("a message rejected while paused does not consume the discard") { + // A paused listener nacks, and `negativeAckRedeliveryDelay(0)` brings the message + // straight back. Counting it as dropped would skip n+1 messages. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(2) + val runner = targetRunner(l) + + resume(runner) + val first = l.decide(topicFqn, canAcknowledge = true) + runner.pause() + val whilePaused = deliver(l, 5) + resume(runner) + val afterResume = deliver(l, 3) + + assertTrue( + first == Drop, + whilePaused.forall(_ == Reject), + afterResume == Vector(Drop, Deliver, Deliver), + l.startFromDiscard.remaining == 0L + ) + }, + test("resume leaves the counter object itself alone") { + // Belt and braces on the mechanism rather than the effect: `resume` must not swap the + // discard for a fresh one either. + val l = listener() + val armed = StartFromDiscard.shared(5) + l.startFromDiscard = armed + val runner = targetRunner(l) + resume(runner) + runner.pause() + resume(runner) + assertTrue(l.startFromDiscard eq armed) + }, + test("a session reports the remaining discard, counting a shared counter once") { + // Two targets over one shared counter: summing per target would report double and make + // "skipped exactly n" unassertable. + val shared = StartFromDiscard.shared(4) + val firstListener = listener() + val secondListener = listener() + firstListener.startFromDiscard = shared + secondListener.startFromDiscard = shared + + val session = ConsumerSessionRunner( + sessionName = "cs-discard", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> targetRunner(firstListener), 1 -> targetRunner(secondListener)) + ) + + val atStart = session.remainingStartFromDiscard + deliver(firstListener, 1) + deliver(secondListener, 2) + + assertTrue(atStart == 4L, session.remainingStartFromDiscard == 1L) + }, + test("the MERGE-owned skip budget survives pause/resume: no re-arm, no double count, no gate leak") { + // The branch's MAIN path. On a multi-stream skip-n the listener's own discard is NONE + // and the budget lives inside the global merge (`handleStartFrom` arms + // StartFromDiscardPlan.Nothing + GlobalSkip) - so every exactly-once case above + // drives the WRONG counter for it. Three claims, each of which held only by + // inspection until now: a pause/resume cycle leaves the merge's budget untouched + // (resume never touches `startFromOrdering`), a message the closed gate rejects + // never reaches the merge and so cannot consume the budget, and after the cycle the + // skip still drops exactly the globally-first n - the same SET a pause-free run + // would have dropped. + val consumerName = "cs-discard-once-0" + val p0 = "persistent://public/default/discard-once-p0" + val p1 = "persistent://public/default/discard-once-p1" + + final class RecordingConsumer(topicFqn: String): + val acknowledged = ConcurrentLinkedQueue[String]() + val handedBack = ConcurrentLinkedQueue[String]() + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "acknowledgeAsync" => + acknowledged.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + CompletableFuture.completedFuture(null) + case "negativeAcknowledge" => + handedBack.add(args(0).asInstanceOf[PulsarMessage[Array[Byte]]].getKey) + null + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + def message(topicFqn: String, key: String, publishTime: Long, entryId: Long): MessageImpl[Array[Byte]] = + val md = new MessageMetadata() + md.setPublishTime(publishTime) + md.setPartitionKey(key) + val msg = MessageImpl.create[Array[Byte]](md, ByteBuffer.wrap(s"""{"k":"$key"}""".getBytes("UTF-8")), Schema.BYTES, topicFqn) + msg.setMessageId(new MessageIdImpl(1L, entryId, -1)) + msg + + val delivered = ConcurrentLinkedQueue[String]() + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = msg => { delivered.add(msg.getKey); () })) + // The production resume rewires the target handler onto the full pipeline; this test + // asserts on the raw delivery SET, so the recorder is reinstalled after each resume. + def recordDeliveries(): Unit = + l.targetMessageHandler.onNext = msg => { delivered.add(msg.getKey); () } + // The listener-owned counter stays NONE - the merge owns the budget. + val armed = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip( + 3, + Vector( + StartFromStream(startFromStreamId(consumerName, p0), EntryPosition(1L, 9L, -1, 1)), + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition(1L, 9L, -1, 1)) + ) + ) + ) + l.startFromOrdering = armed + val runner = targetRunner(l, Vector(p0, p1)) + val c0 = RecordingConsumer(p0) + val c1 = RecordingConsumer(p1) + + resume(runner) + recordDeliveries() + // Publish times interleave the streams: the globally-first 3 are a1(10), b1(20), a2(30). + l.received(c0.consumer, message(p0, "a1", 10L, 0L)) // held: p1 is blind + l.received(c1.consumer, message(p1, "b1", 20L, 0L)) // resolves: drop a1; p0 blind + l.received(c0.consumer, message(p0, "a2", 30L, 1L)) // resolves: drop b1; p1 blind + val remainingMidSkip = l.effectiveDiscard.remaining + + runner.pause() + // The closed gate rejects b2 BEFORE the merge ever sees it: handed back, budget intact. + l.received(c1.consumer, message(p1, "b2", 40L, 1L)) + val remainingWhilePaused = l.effectiveDiscard.remaining + val handedBackWhilePaused = c1.handedBack.asScala.toVector + + resume(runner) + recordDeliveries() + val sameOrderingAfterResume = l.startFromOrdering eq armed + val remainingAfterResume = l.effectiveDiscard.remaining + + // The broker redelivers the rejected b2; the skip finishes on a2 and b2 is the first + // message the user sees, then the inert merge passes a3 straight through. + l.received(c1.consumer, message(p1, "b2", 40L, 1L)) + l.received(c0.consumer, message(p0, "a3", 50L, 2L)) + + assertTrue( + remainingMidSkip == 1L, + remainingWhilePaused == 1L, // the gate leak would have burned the last unit here + handedBackWhilePaused == Vector("b2"), + sameOrderingAfterResume, // resume must not re-arm (or swap) the ordering layer + remainingAfterResume == 1L, // and must not refill or spend the budget + // The cut is EXACTLY the globally-first 3 by publish time - a1, b1, a2 - however + // the pause interleaved; b2 and a3 are what the user sees, in order. + delivered.asScala.toVector == Vector("b2", "a3"), + c0.acknowledged.asScala.toVector == Vector("a1", "a2", "a3"), + c1.acknowledged.asScala.toVector == Vector("b1", "b2"), + l.effectiveDiscard.remaining == 0L, + l.startFromDiscard.remaining == 0L // the listener-owned counter never took part + ) ?? (s"midSkip=$remainingMidSkip paused=$remainingWhilePaused afterResume=$remainingAfterResume " + + s"delivered=${delivered.asScala.toVector} acked0=${c0.acknowledged.asScala.toVector} " + + s"acked1=${c1.acknowledged.asScala.toVector}") + }, + test("a session sums per-topic counters across its targets") { + val firstListener = listener() + val secondListener = listener() + firstListener.startFromDiscard = StartFromDiscard.perTopic(Map(topicFqn -> 3L)) + secondListener.startFromDiscard = StartFromDiscard.perTopic(Map(topicFqn -> 3L)) + + val session = ConsumerSessionRunner( + sessionName = "cs-discard-per-topic", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = Map(0 -> targetRunner(firstListener), 1 -> targetRunner(secondListener)) + ) + + deliver(firstListener, 1) + assertTrue(session.remainingStartFromDiscard == 5L) + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/startFromDiscardTest.scala b/server/src/test/scala/consumer/session_runner/startFromDiscardTest.scala new file mode 100644 index 000000000..b9f79d59a --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromDiscardTest.scala @@ -0,0 +1,213 @@ +package consumer.session_runner + +import zio.test.* + +/** The exactness of "Skip first n messages" and "Latest n messages". + * + * Regression context: both modes were built on `PulsarAdmin.examineMessage`, which is + * ENTRY-addressed, not message-addressed - and the Java producer batches by default. VERIFIED + * against Pulsar 3.2.1: 100 messages sent with a default producer became ONE broker entry, and + * `examineMessage(topic, "earliest", k)` answered with that same entry for every k from 1 to 101, + * silently clamping instead of failing. "Skip the first 5" therefore asked for entry 6, got the + * last entry, and skipped fifty messages - on the most ordinary setup there is, one non-partitioned + * topic with a default producer. Counting back from "latest" past the start does NOT clamp on 3.2: + * it fails, and the failure was swallowed into a fallback that seeked to EARLIEST, so "the latest + * 5" showed every message in the topic. + * + * The fix is seek + discard: a seek can only land on an ENTRY boundary (batch-index positions such + * as `1696:1:25` are rejected by the broker), so the only exact way to land on a MESSAGE is to seek + * to the entry holding it and drop the messages in front of it. + * + * [[resolveLatestN]] is pure - the broker sits behind a `Long => Option[(id, batchSize)]` lookup - + * so every log shape is driven here with a plain lambda: no broker, and no mock. + */ +object startFromDiscardTest extends ZIOSpecDefault: + + private val oneTopic = "persistent://public/default/one" + + /** A log described from its END: element 0 is the LAST entry, and each element is the number of + * messages that entry holds. All entries share one publish time, so nothing here depends on the + * cross-topic ordering - that is `globalStartFromTest`'s job. Returns the lookup + * [[resolveLatestN]] expects plus a counter of how many lookups it made (the walk must be + * O(n / batch size), not O(log size)). + */ + private def logFromLatest(entriesFromLatest: Int*): (String => Long => Option[LogEntry[String]], () => Int) = + var lookups = 0 + val lookup = (_: String) => + (k: Long) => + lookups += 1 + Option.when(k >= 1 && k <= entriesFromLatest.size)(LogEntry(s"entry-$k", 1_000L, entriesFromLatest(k.toInt - 1))) + (lookup, () => lookups) + + /** A broker that CLAMPS instead of failing once the walk runs past the start of the log - the + * behaviour `examineMessage` shows on the "earliest" side, guarded against here so a Pulsar + * version that clamps on "latest" too cannot turn the walk into an infinite loop. + */ + private def clampingLogFromLatest(entriesFromLatest: Int*): (String => Long => Option[LogEntry[String]], () => Int) = + var lookups = 0 + val lookup = (_: String) => + (k: Long) => + lookups += 1 + val clamped = k.min(entriesFromLatest.size).max(1) + Option.when(entriesFromLatest.nonEmpty)(LogEntry(s"entry-$clamped", 1_000L, entriesFromLatest(clamped.toInt - 1))) + (lookup, () => lookups) + + /** The `entry-$k` labels carry the walk's entry order: `k` counts back from the end, so a larger + * ordinal is strictly OLDER (the production comparator is `MessageIdImpl.compareTo`). */ + private def olderByEntryOrdinal(a: String, b: String): Boolean = a.split("-").last.toInt > b.split("-").last.toInt + + private def resolveOne(n: Long, lookup: String => Long => Option[LogEntry[String]]): LatestNSeek[String] = + resolveLatestN(n, Vector(oneTopic), lookup, olderByEntryOrdinal)(oneTopic) + + private val resolveLatestNSuite = suite("resolveLatestN over a single log")( + test("one batched entry: the last 5 of 100 seek to that entry and discard the 95 in front") { + // THE regression. Entry-addressing answered "entry 1" and discarded nothing, so all 100 + // messages were shown for a request of 5. + val (lookup, _) = logFromLatest(100) + assertTrue(resolveOne(5, lookup) == LatestNSeek.FromEntry("entry-1", 95L)) + }, + test("unbatched log: the last 5 land on the 5th entry from the end with nothing to discard") { + // The control: with one message per entry, entry-addressing and message-addressing + // coincide - which is exactly why unbatched fixtures never exposed the defect. + val (lookup, lookups) = logFromLatest(1, 1, 1, 1, 1, 1, 1, 1, 1, 1) + assertTrue(resolveOne(5, lookup) == LatestNSeek.FromEntry("entry-5", 0L), lookups() == 5) + }, + test("uneven batches: the walk stops on the entry that covers the n-th message") { + // From the end: 2, then 3 -> exactly 5 accounted for on the second entry. + val (lookup, lookups) = logFromLatest(2, 3, 10, 10) + assertTrue(resolveOne(5, lookup) == LatestNSeek.FromEntry("entry-2", 0L), lookups() == 2) + }, + test("uneven batches: the overshoot inside the stopping entry is discarded") { + // From the end: 2, then 3 -> 5 accounted for, but only 4 were asked for, so drop 1. + val (lookup, _) = logFromLatest(2, 3, 10) + assertTrue(resolveOne(4, lookup) == LatestNSeek.FromEntry("entry-2", 1L)) + }, + test("the last 1 of a 10-message batch discards the other 9") { + val (lookup, _) = logFromLatest(10, 10) + assertTrue(resolveOne(1, lookup) == LatestNSeek.FromEntry("entry-1", 9L)) + }, + test("a log shorter than n shows all of it - anchored at its oldest entry, re-verifiably") { + // FromEntry(oldest, 0) covers the same messages a seek-to-earliest would, and unlike + // `Everything` the retention re-check can later prove the anchor still exists. + val (lookup, _) = logFromLatest(10, 10, 10) + assertTrue(resolveOne(100, lookup) == LatestNSeek.FromEntry("entry-3", 0L)) + }, + test("an empty log maps to EVERYTHING - all it will ever hold is post-inspection live traffic") { + // Empty when inspected means whatever exists at seek time arrived AFTER the inspection; + // seeking EARLIEST delivers exactly that. Seeking "latest" at seek time raced those + // same appends and silently lost them. + val (lookup, _) = logFromLatest() + assertTrue(resolveOne(5, lookup) == LatestNSeek.Everything) + }, + test("a clamping broker is detected within a bounded number of steps, not by the first repeat") { + // Without a termination guard this walk never ends: the lookup keeps answering with the + // last entry and the running total keeps growing by 3 forever, so it would eventually + // "reach" n and seek to the WRONG entry with a nonsense discard. + // + // CHANGED EXPECTATION, deliberately: the old guard stopped on the FIRST repeated id, in + // exactly 3 lookups - but a single concurrent append produces that same first repeat (the + // moving anchor), so treating it as exhaustion was the whole-backlog bug. A clamp is now + // told apart by ONE VERIFICATION LOOKUP at the k that produced the last accepted entry - + // a clamped end never moves, a grown end answers newer - so the topic still resolves + // as exhausted (anchored at its oldest entry), one lookup later than the old guard. + val (lookup, lookups) = clampingLogFromLatest(3, 3) + assertTrue( + resolveOne(100, lookup) == LatestNSeek.FromEntry("entry-2", 0L), + lookups() > 3, + lookups() <= maxLatestNReanchorSteps + 4 + ) ?? s"resolved after ${lookups()} lookups (bound $maxLatestNReanchorSteps)" + }, + test("a clamping broker still resolves exactly when the log does hold n messages") { + val (lookup, _) = clampingLogFromLatest(3, 4, 50) + assertTrue(resolveOne(5, lookup) == LatestNSeek.FromEntry("entry-2", 2L)) + }, + test("the walk costs O(n / batch size) lookups, not one per message") { + // 50 messages over batches of 10 is 5 admin calls - and it does not matter that the log + // behind them is 1000 entries long. + val (lookup, lookups) = logFromLatest(Vector.fill(1000)(10)*) + assertTrue(resolveOne(50, lookup) == LatestNSeek.FromEntry("entry-5", 0L), lookups() == 5) + }, + test("n = 0 shows nothing retained, without asking the broker anything") { + val (lookup, lookups) = logFromLatest(10, 10) + assertTrue(resolveOne(0, lookup) == LatestNSeek.Nothing, lookups() == 0) + } + ) + + private val topicA = "persistent://public/default/a" + private val topicB = "persistent://public/default/b" + + private val startFromDiscardSuite = suite("StartFromDiscard")( + test("a shared counter drops exactly n messages of the merged stream, then stops") { + // "Skip first n" across partitions: the count is exact even though the interleaving is + // not - which is the whole point of counting the merged stream instead of each log. + val discard = StartFromDiscard.shared(4) + val topics = Vector(topicA, topicB, topicA, topicB, topicA, topicB, topicA) + val dropped = topics.map(discard.claim) + assertTrue(dropped == Vector(true, true, true, true, false, false, false), discard.remaining == 0) + }, + test("a shared counter reports what is left and never goes below zero") { + val discard = StartFromDiscard.shared(2) + val afterNone = discard.remaining + discard.claim(topicA) + val afterOne = discard.remaining + discard.claim(topicA) + discard.claim(topicA) + discard.claim(topicA) + assertTrue(afterNone == 2L, afterOne == 1L, discard.remaining == 0L) + }, + test("per-topic counters are independent - one topic's overshoot does not eat another's") { + // "Latest n" on a partitioned topic: each partition seeks to its own entry and owes its + // own overshoot. + val discard = StartFromDiscard.perTopic(Map(topicA -> 2, topicB -> 1)) + val a = Vector(discard.claim(topicA), discard.claim(topicA), discard.claim(topicA)) + val b = Vector(discard.claim(topicB), discard.claim(topicB)) + assertTrue(a == Vector(true, true, false), b == Vector(true, false), discard.remaining == 0) + }, + test("a topic with no counter is never dropped") { + val discard = StartFromDiscard.perTopic(Map(topicA -> 5)) + assertTrue(!discard.claim(topicB), discard.remaining == 5L) + }, + test("the empty discard drops nothing") { + assertTrue(!StartFromDiscard.none.claim(topicA), StartFromDiscard.none.remaining == 0L) + }, + test("concurrent claims on a shared counter drop exactly n in total") { + // One listener per consumer, each on its own Pulsar client io thread, all claiming from + // the same counter. A read-then-write counter would over-drop here. + val discard = StartFromDiscard.shared(100) + val claims = 8 + val perThread = 100 + val dropped = java.util.concurrent.atomic.AtomicLong(0) + val threads = (1 to claims).map(i => + Thread(() => (1 to perThread).foreach(_ => if discard.claim(s"topic-$i") then dropped.incrementAndGet())) + ) + threads.foreach(_.start()) + threads.foreach(_.join()) + assertTrue(dropped.get == 100L, discard.remaining == 0L) + }, + test("forTarget hands every target the SAME shared counter") { + val plan = StartFromDiscardPlan.SharedTotal(3) + val shared = StartFromDiscard.shared(3) + val first = StartFromDiscard.forTarget(plan, shared, Vector(topicA)) + val second = StartFromDiscard.forTarget(plan, shared, Vector(topicB)) + first.claim(topicA) + first.claim(topicA) + assertTrue(first eq second, second.remaining == 1L) + }, + test("forTarget gives each target its OWN per-topic counters, scoped to its topics") { + // Two targets may select the same topic; each has its own consumer, and each has to + // drop its own overshoot. + val plan = StartFromDiscardPlan.PerTopic(Map(topicA -> 2L, topicB -> 7L)) + val first = StartFromDiscard.forTarget(plan, StartFromDiscard.none, Vector(topicA)) + val second = StartFromDiscard.forTarget(plan, StartFromDiscard.none, Vector(topicA)) + first.claim(topicA) + first.claim(topicA) + assertTrue( + !(first eq second), + first.remaining == 0L, + second.remaining == 2L, // untouched by the other target + second.claim(topicB) == false // topicB belongs to neither target + ) + } + ) + + def spec = suite(this.getClass.toString)(resolveLatestNSuite, startFromDiscardSuite) diff --git a/server/src/test/scala/consumer/session_runner/startFromLiveEdgePlanTest.scala b/server/src/test/scala/consumer/session_runner/startFromLiveEdgePlanTest.scala new file mode 100644 index 000000000..747f2b115 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromLiveEdgePlanTest.scala @@ -0,0 +1,137 @@ +package consumer.session_runner + +import _root_.consumer.session_config.MessageDeliveryOrder +import _root_.consumer.start_from.{ApproximateEntryPosition, ConsumerSessionStartFrom, EarliestMessage, LatestMessage, NthMessageBeforeLatest} +import org.apache.pulsar.client.admin.PulsarAdmin +import org.apache.pulsar.client.api.{Consumer, PulsarClient, MessageId as PulsarMessageId} +import org.apache.pulsar.client.impl.MessageIdImpl +import zio.test.* + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.{ConcurrentLinkedQueue, TimeUnit} +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters.* +import scala.util.Try + +/** THE CREATE-TIME BOUNDARY OF A LIVE-EDGE SEEK IS EMPTY, AND COSTS NO BROKER READ. + * + * Under the GUARANTEED exact replay, "everything recorded up to the moment Play was pressed" is + * bounded by the per-stream ends captured at session build - and a seek that lands AT THE LIVE + * EDGE (Latest, latest-n = 0, the approximate-entry endpoint at 1.0) leaves nothing recorded + * behind the cursor, so its boundary is decided EMPTY by the seek itself, without asking the + * broker for the topic's end at all. These pins drive the real `handleStartFrom` over proxy + * consumers whose end-reads are counted, with real Pulsar clients aimed at a closed port so an + * accidental broker call fails the test loudly instead of hanging or passing by accident. + * + * The EMPTY boundary is what makes Latest x Guaranteed an instant caught-up on a NON-EMPTY topic: + * the broker's real end still names the backlog behind the cursor, and reading it as the boundary + * arms the barrier over a range the consumer can never deliver (e2e CS-DM-R3B). The resume-path + * half of that contract - the first Play CONSUMES these boundaries instead of re-reading the + * broker - is pinned in replayBoundaryTest. + */ +object startFromLiveEdgePlanTest extends ZIOSpecDefault: + + private val consumerName = "cs-live-edge-plan-0" + private def topic(i: Int): String = s"persistent://public/default/cs-live-edge-plan-$i" + + /** A consumer whose SEEKS are recorded and whose end-reads are counted. The broker end it + * would answer with is a real entry - the non-empty-backlog shape - so a plan that wrongly + * captured ends for a live-edge seek is caught by the boundary VALUE as well as the count. */ + private final class SeekRecordingConsumer(topicFqn: String): + val seeks = ConcurrentLinkedQueue[Object]() + val lastIdsReads = AtomicLong(0) + @volatile var lastIds: java.util.List[PulsarMessageId] = java.util.List.of(new MessageIdImpl(1L, 7L, -1)) + + val consumer: Consumer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "getTopic" => topicFqn + case "getConsumerName" => consumerName + case "isConnected" => java.lang.Boolean.TRUE + case "seek" => + seeks.add(args(0)) + null + case "pause" => null + case "getLastMessageIds" => + lastIdsReads.incrementAndGet() + lastIds + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(topicFqn.hashCode) + case "toString" => s"proxy-consumer($topicFqn)" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Consumer[Array[Byte]]]), handler) + .asInstanceOf[Consumer[Array[Byte]]] + + /** Real clients aimed at a closed port, exactly as startFromCountValidationTest uses them: the + * live-edge paths must ask the broker NOTHING, so an accidental admin or client call surfaces + * as a connection error rather than as a mock quietly answering. */ + private def withOfflineClients[A](f: (PulsarClient, PulsarAdmin) => A): A = + val client = PulsarClient.builder.serviceUrl("pulsar://127.0.0.1:1").operationTimeout(2, TimeUnit.SECONDS).build + val admin = PulsarAdmin.builder + .serviceHttpUrl("http://127.0.0.1:1") + .connectionTimeout(2, TimeUnit.SECONDS) + .readTimeout(2, TimeUnit.SECONDS) + .requestTimeout(2, TimeUnit.SECONDS) + .build + try f(client, admin) + finally + Try(client.close()) + Try(admin.close()) + + private def guaranteedPlan(startFrom: ConsumerSessionStartFrom, consumers: Vector[SeekRecordingConsumer]): StartFromPlan = + withOfflineClients((client, admin) => + handleStartFrom( + startFrom = startFrom, + consumers = consumers.map(_.consumer), + adminClient = admin, + pulsarClient = client, + nonPartitionedTopicFqns = consumers.map(c => c.consumer.getTopic), + deliveryOrdering = MessageDeliveryOrder.Guaranteed + ) + ) + + private def orderedStreamsOf(plan: StartFromPlan): Vector[StartFromStream] = plan.ordering match + case StartFromOrderingPlan.Ordered(streams, MessageDeliveryOrder.Guaranteed) => streams + case other => throw new AssertionError(s"expected a Guaranteed Ordered plan, got $other") + + private def liveEdgeCase(startFrom: ConsumerSessionStartFrom) = + val consumers = Vector(SeekRecordingConsumer(topic(0)), SeekRecordingConsumer(topic(1))) + val streams = orderedStreamsOf(guaranteedPlan(startFrom, consumers)) + assertTrue( + streams.size == 2, + // The boundary is decided EMPTY by the seek itself: nothing recorded behind a + // live-edge cursor, so nothing to replay - the instant-caught-up input. + streams.forall(_.lastAtStart == EntryPosition.empty), + // And it costs no broker end-read: the seek already knows. + consumers.forall(_.lastIdsReads.get == 0L), + // The seek that produced that knowledge really was to the live edge. + consumers.forall(_.seeks.asScala.toVector == Vector(PulsarMessageId.latest)) + ) ?? (s"streams=$streams endReads=${consumers.map(_.lastIdsReads.get)} " + + s"seeks=${consumers.map(_.seeks.asScala.toVector)}") + + def spec = suite(this.getClass.toString)( + test("LATEST x GUARANTEED: every boundary is empty, no end is read, every consumer sits at the live edge") { + liveEdgeCase(LatestMessage()) + }, + test("LATEST-N = 0 x GUARANTEED: 'the last 0 messages' is the same live edge - empty boundaries, no end reads") { + liveEdgeCase(NthMessageBeforeLatest(n = 0)) + }, + test("APPROXIMATE ENTRY AT 1.0 x GUARANTEED: the endpoint fast path is the live edge - empty boundaries, no end reads") { + liveEdgeCase(ApproximateEntryPosition(fraction = 1.0)) + }, + test("THE CONTRAST: EARLIEST x GUARANTEED captures the real recorded ends, one read per consumer") { + // A history seek's boundary is the topic's real end - the broker IS asked, exactly + // once per consumer, and the captured position is the one it answered with. This is + // what keeps the live-edge pins above honest: the counter can tell the two apart. + val consumers = Vector(SeekRecordingConsumer(topic(0)), SeekRecordingConsumer(topic(1))) + val streams = orderedStreamsOf(guaranteedPlan(EarliestMessage(), consumers)) + assertTrue( + streams.size == 2, + streams.forall(_.lastAtStart == EntryPosition(1L, 7L, -1, 1)), + consumers.forall(_.lastIdsReads.get == 1L), + consumers.forall(_.seeks.asScala.toVector == Vector(PulsarMessageId.earliest)) + ) ?? s"streams=$streams endReads=${consumers.map(_.lastIdsReads.get)}" + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/consumer/session_runner/startFromOrderingTest.scala b/server/src/test/scala/consumer/session_runner/startFromOrderingTest.scala new file mode 100644 index 000000000..661203162 --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromOrderingTest.scala @@ -0,0 +1,453 @@ +package consumer.session_runner + +import java.util.concurrent.atomic.AtomicLong + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.{ConsumerSessionStartFrom, DateTime, DateTimeUnit, EarliestMessage, LatestMessage, MessageId, NthMessageAfterEarliest, NthMessageBeforeLatest, ApproximateEntryPosition, ApproximatePublishTimePosition, RelativeDateTime} +import _root_.consumer.value_projections.ValueProjectionList +import org.apache.pulsar.client.impl.MessageIdImpl +import zio.test.* + +/** Wiring the two GLOBAL start-from modes into the session: which delivery stream a message belongs + * to, when that stream has run out of the messages it held at session start, and where the client + * reads the skip's progress from once the merge owns the counting. + * + * `StartFromOrdering` is generic in what it holds, so the whole routing is driven here with real + * Pulsar message ids and plain payloads - no broker, no mock. The layer never looks inside what it + * holds; it only decides drop-or-deliver and hands the payload back. + */ +object startFromOrderingTest extends ZIOSpecDefault: + + private val p0 = "persistent://public/default/ord-partition-0" + private val p1 = "persistent://public/default/ord-partition-1" + private val consumerName = "cs-ordering-0" + private val otherConsumerName = "cs-ordering-1" + + private def id(entryId: Long): MessageIdImpl = new MessageIdImpl(1L, entryId, 0) + + /** A stream whose backlog ended on `lastEntryId`. */ + private def stream(consumer: String, topicFqn: String, lastEntryId: Long): StartFromStream = + StartFromStream(startFromStreamId(consumer, topicFqn), EntryPosition(1L, lastEntryId, -1, 1)) + + private def emptyStream(consumer: String, topicFqn: String): StartFromStream = + StartFromStream(startFromStreamId(consumer, topicFqn), EntryPosition.empty) + + extension (ordering: StartFromOrdering[String]) + /** One delivered message, as the listener hands it over. */ + def deliver(consumer: String, topicFqn: String, publishTime: Long, entryId: Long, value: String): Vector[(String, StartFromOutcome)] = + ordering.offer(consumer, topicFqn, publishTime, id(entryId), value) + + def delivered(resolved: Vector[(String, StartFromOutcome)]): Vector[String] = + resolved.collect { case (value, StartFromOutcome.Deliver) => value } + + private val planSuite = suite("which sessions get a global ordering layer at all")( + test("SKIP-N gets one as soon as there is more than one stream") { + assertTrue( + needsGlobalOrdering(NthMessageAfterEarliest(n = 5), streamCount = 2), + needsGlobalOrdering(NthMessageAfterEarliest(n = 5), streamCount = 3) + ) + }, + test("LATEST-N NEVER GETS ONE - it resolves its cut before anything is delivered") { + // Changed expectation, deliberately: latest-n used to arm a top-n heap of DELIVERED + // messages here, which made a session's memory a number the user typed and let live + // traffic evict the historical tail. The cut now comes from entry metadata + // (`resolveLatestN`), so there is nothing left to hold and nothing left to reorder. + val armed = Vector(1, 2, 3, 50).filter(streams => needsGlobalOrdering(NthMessageBeforeLatest(n = 5), streams)) + assertTrue(armed.isEmpty) ?? s"latest-n asked for a buffering layer at stream counts $armed" + }, + test("a single stream keeps the FAST PATH - the ordinary non-partitioned session pays nothing") { + // One log is already in append order: the head-drop counter is exact on its own, and a + // merge would hold messages for no reason. + assertTrue( + !needsGlobalOrdering(NthMessageAfterEarliest(n = 5), streamCount = 1), + !needsGlobalOrdering(NthMessageAfterEarliest(n = 5), streamCount = 0) + ) + }, + test("n = 0 needs no ordering, however many streams there are") { + // A negative n is REFUSED at the trust boundary before this gate is reached (see + // `startFromCountRejectionReason`), so it can no longer arrive here from a request. The + // gate stays defensive about it anyway - answering "no ordering" is the safe reading if + // one ever did. + assertTrue( + !needsGlobalOrdering(NthMessageAfterEarliest(n = 0), streamCount = 4), + !needsGlobalOrdering(NthMessageAfterEarliest(n = -3), streamCount = 4) + ) + }, + test("percentage positions resolve before delivery and do not request a merge layer") { + // Deliberate, not an oversight: a count of n is reached by streaming n messages and + // stopping, but a proportion of the merged stream is only known once all of it has been + // measured, which is O(topic) at any n. + val exactSeekModes: Vector[ConsumerSessionStartFrom] = Vector( + ApproximateEntryPosition(fraction = 0.6), + ApproximatePublishTimePosition(fraction = 0.6), + EarliestMessage(), + LatestMessage(), + DateTime(dateTime = java.time.Instant.EPOCH), + RelativeDateTime(value = 1, unit = DateTimeUnit.Hour, isRoundedToUnitStart = false), + MessageId(messageIdBytes = Array.empty) + ) + val wrongly = exactSeekModes.filter(mode => needsGlobalOrdering(mode, streamCount = 4)) + assertTrue(wrongly.isEmpty) ?? s"these modes asked for a global ordering they must not have: ${wrongly.map(_.getClass.getSimpleName)}" + }, + test("only skip-n gets the streaming MERGE, and only it") { + val streams = Vector(stream(consumerName, p0, 9), stream(consumerName, p1, 9)) + val skip = globalOrderingPlanFor(NthMessageAfterEarliest(n = 7), streams) + val latest = globalOrderingPlanFor(NthMessageBeforeLatest(n = 7), streams) + assertTrue( + skip == StartFromOrderingPlan.GlobalSkip(7, streams), + latest == StartFromOrderingPlan.PassThrough + ) ?? s"skip=$skip latest=$latest" + }, + test("the fast path and the exact-seek modes build no layer at all") { + val oneStream = Vector(stream(consumerName, p0, 9)) + val twoStreams = Vector(stream(consumerName, p0, 9), stream(consumerName, p1, 9)) + assertTrue( + globalOrderingPlanFor(NthMessageAfterEarliest(n = 7), oneStream) == StartFromOrderingPlan.PassThrough, + globalOrderingPlanFor(NthMessageBeforeLatest(n = 7), oneStream) == StartFromOrderingPlan.PassThrough, + globalOrderingPlanFor(ApproximateEntryPosition(fraction = 0.6), twoStreams) == StartFromOrderingPlan.PassThrough, + globalOrderingPlanFor(ApproximatePublishTimePosition(fraction = 0.6), twoStreams) == StartFromOrderingPlan.PassThrough, + globalOrderingPlanFor(EarliestMessage(), twoStreams) == StartFromOrderingPlan.PassThrough + ) + } + ) + + private val identitySuite = suite("which stream a message belongs to")( + test("the same topic reached by two targets is TWO streams") { + // Each enabled target has its OWN consumer on the topic and delivers it independently. + // Merging both under the topic name would let one target's head hide the other's. + assertTrue(startFromStreamId(consumerName, p0) != startFromStreamId(otherConsumerName, p0)) + }, + test("one target's two partitions are two streams") { + assertTrue(startFromStreamId(consumerName, p0) != startFromStreamId(consumerName, p1)) + }, + test("a skip merge waits for BOTH targets on one topic before it decides") { + // The regression this guards: keyed by topic alone, the second target's consumer would + // overwrite the first's head and the merge would drop whichever arrived last. + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(consumerName, p0, 5), stream(otherConsumerName, p0, 5))) + ) + val afterFirst = ordering.deliver(consumerName, p0, 200L, 0, "later-target-a") + val afterSecond = ordering.deliver(otherConsumerName, p0, 100L, 0, "earlier-target-b") + assertTrue( + afterFirst.isEmpty, + // The drop spends the budget's only unit, so the OTHER target's held message is + // released in the same resolution - the merge does not wait for a head it can no + // longer use. + afterSecond == Vector("earlier-target-b" -> StartFromOutcome.Drop, "later-target-a" -> StartFromOutcome.Deliver) + ) ?? s"afterFirst=$afterFirst afterSecond=$afterSecond" + } + ) + + private val backlogEndSuite = suite("when a stream stops being waited for")( + test("a stream is waited for until its recorded last message arrives") { + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(consumerName, p0, 9), stream(consumerName, p1, 0))) + ) + // p1's whole backlog is entry 0, so its first delivery also ends it - and the merge can + // decide without ever hearing from p1 again. + val held = ordering.deliver(consumerName, p0, 100L, 0, "a1") + val decided = ordering.deliver(consumerName, p1, 50L, 0, "b1") + val next = ordering.deliver(consumerName, p0, 200L, 1, "a2") + assertTrue( + held.isEmpty, + decided == Vector("b1" -> StartFromOutcome.Drop, "a1" -> StartFromOutcome.Deliver), + next == Vector("a2" -> StartFromOutcome.Deliver) + ) ?? s"held=$held decided=$decided next=$next" + }, + test("a topic that was empty at session start is never waited for") { + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(consumerName, p0, 9), emptyStream(consumerName, p1))) + ) + assertTrue(ordering.deliver(consumerName, p0, 100L, 0, "a1") == Vector("a1" -> StartFromOutcome.Drop)) + }, + test("a stream the plan never heard of is never waited for") { + // A topic that appears under the session's feet must not hang it. It delivers once and + // goes quiet - if it had joined the waited-for set, everything after it would be held + // forever. Two things keep it out of that set, deliberately: the waited-for set is built + // from the PLAN, and an unknown stream reports itself at its backlog end. + val ordering = StartFromOrdering.make[String](StartFromOrderingPlan.GlobalSkip(1, Vector(emptyStream(consumerName, p0)))) + val fromUnknown = ordering.deliver(consumerName, p1, 100L, 0, "surprise") + val fromKnown = ordering.deliver(consumerName, p0, 200L, 0, "known") + assertTrue( + fromUnknown == Vector("surprise" -> StartFromOutcome.Drop), + fromKnown == Vector("known" -> StartFromOutcome.Deliver), + ordering.heldCount == 0 + ) ?? s"fromUnknown=$fromUnknown fromKnown=$fromKnown" + }, + test("a partition that answered with MessageId.earliest is drained, not waited for") { + // The end-to-end shape of the same bug: the plan records what getLastMessageIds said, + // and for an empty partition that is MessageId.earliest. + val emptyByEarliest = + StartFromStream(startFromStreamId(consumerName, p1), EntryPosition.of(org.apache.pulsar.client.api.MessageId.earliest)) + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip(1, Vector(stream(consumerName, p0, 1), emptyByEarliest)) + ) + val a1 = ordering.deliver(consumerName, p0, 10L, 0, "a1") + val a2 = ordering.deliver(consumerName, p0, 20L, 1, "a2") + assertTrue( + a1 == Vector("a1" -> StartFromOutcome.Drop), + ordering.delivered(a2) == Vector("a2") + ) ?? s"an empty partition held the merge back: a1=$a1 a2=$a2" + } + ) + + private val passThroughSuite = suite("no reordering at all")( + test("pass-through delivers every message untouched and holds nothing") { + val ordering = StartFromOrdering.passThrough[String] + val resolved = ordering.deliver(consumerName, p0, 10L, 0, "a1") + assertTrue(resolved == Vector("a1" -> StartFromOutcome.Deliver), ordering.heldCount == 0, ordering.progressDiscard.isEmpty) + }, + test("a plan with no ordering makes a pass-through") { + val ordering = StartFromOrdering.make[String](StartFromOrderingPlan.PassThrough) + assertTrue(ordering.progressDiscard.isEmpty, ordering.deliver(consumerName, p0, 10L, 0, "a1").size == 1) + } + ) + + /** A listener already open for business. It starts CLOSED in production - nothing may be + * consumed before the session is armed and a client has resumed it - and + * `ConsumerSessionTargetRunner.resume` is what opens it. */ + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetRunner(consumerListener: ConsumerListener): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = Vector(p0))), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + nonPartitionedTopicFqns = Vector(p0), + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(targets: (Int, ConsumerSessionTargetRunner)*): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-ordering", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = targets.toMap + ) + + private def skipOrdering(n: Long): StartFromOrdering[HeldMessage] = + StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip(n, Vector(stream(consumerName, p0, 9), stream(consumerName, p1, 9))) + ) + + private val progressSuite = suite("where the client reads the skip's progress from")( + test("a listener with no ordering reads its own head-drop counter") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(6) + assertTrue(l.progressDiscard eq l.startFromDiscard) + }, + test("a listener whose merge owns the counting reads the MERGE's counter") { + // The merge decides the drops itself, so the listener's own counter is deliberately + // empty - reading that one would report a session with nothing to skip. + val l = listener() + val ordering = skipOrdering(4) + l.startFromOrdering = ordering + assertTrue(l.progressDiscard.total == 4L, !(l.progressDiscard eq l.startFromDiscard)) + }, + test("a latest-n reports NOTHING - its head-drop is a seek correction, not a user skip") { + // A latest-n session's only counter is the overshoot inside the single entry its + // backward walk stopped on. That is the session reaching the requested position, not + // the user skipping anything: reporting it told a client asking for the last 5 messages + // that it was "skipping 95". + val l = listener() + l.startFromDiscard = StartFromDiscard.perTopic(Map(p0 -> 3L)) + assertTrue(l.progressDiscard.total == 0L, l.effectiveDiscard.remaining == 3L) ?? + s"latest-n reported a skip of ${l.progressDiscard.total}" + }, + test("a session reports the merge's progress, and completes when its budget is spent") { + val l = listener() + val ordering = skipOrdering(4) + l.startFromOrdering = ordering + val runner = session(0 -> targetRunner(l)) + val budget = ordering.progressDiscard.get + + val atStart = runner.startFromProgress.map(p => (p.messagesSkipped, p.messagesToSkip, p.complete)) + budget.claim(p0) + budget.claim(p1) + val midway = runner.startFromProgress.map(p => (p.messagesSkipped, p.messagesToSkip, p.complete)) + budget.claim(p0) + budget.claim(p1) + val atEnd = runner.startFromProgress.map(p => (p.messagesSkipped, p.messagesToSkip, p.complete)) + + assertTrue( + atStart == Some((0L, 4L, false)), + midway == Some((2L, 4L, false)), + atEnd == Some((4L, 4L, true)), + runner.remainingStartFromDiscard == 0L + ) ?? s"atStart=$atStart midway=$midway atEnd=$atEnd" + }, + test("two targets sharing one merge report its counter ONCE, not twice") { + val ordering = skipOrdering(10) + val first = listener() + val second = listener() + first.startFromOrdering = ordering + second.startFromOrdering = ordering + val runner = session(0 -> targetRunner(first), 1 -> targetRunner(second)) + assertTrue(runner.startFromProgress.map(_.messagesToSkip) == Some(10L)) + } + ) + + private val armOnceSuite = suite("the ordering layer is armed once")( + test("resuming a target does not swap the ordering layer") { + // The same regression the discard counter has: re-arming on play would start the merge + // over and skip a fresh n every time the user hits it. + val l = listener() + val armed = skipOrdering(3) + l.startFromOrdering = armed + val runner = targetRunner(l) + runner.resume( + onNext = (_, _, _, _, _) => (), + isDebug = false, + incrementNumMessageProcessed = () => (), + onStartFromDiscardProgress = () => (), + admitDelivery = _ => DeliveryAdmission.Prepare + ) + runner.pause() + runner.resume( + onNext = (_, _, _, _, _) => (), + isDebug = false, + incrementNumMessageProcessed = () => (), + onStartFromDiscardProgress = () => (), + admitDelivery = _ => DeliveryAdmission.Prepare + ) + assertTrue(l.startFromOrdering eq armed) + }, + test("a merge partway through a skip stays partway through it across a pause") { + val l = listener() + val ordering = skipOrdering(5) + l.startFromOrdering = ordering + val runner = targetRunner(l) + ordering.progressDiscard.foreach(_.claim(p0)) + ordering.progressDiscard.foreach(_.claim(p0)) + runner.pause() + runner.resume( + onNext = (_, _, _, _, _) => (), + isDebug = false, + incrementNumMessageProcessed = () => (), + onStartFromDiscardProgress = () => (), + admitDelivery = _ => DeliveryAdmission.Prepare + ) + assertTrue(l.progressDiscard.remaining == 3L, l.progressDiscard.total == 5L) + } + ) @@ TestAspect.sequential + + /** THE FLOW-CONTROL WIRING, end to end at the ordering layer: the merge's desired-paused set + * actually reaching (and releasing) the per-stream pause hooks. The merge-side watermark + * logic is pinned in globalStartFromTest; THIS is the half nothing else exercises - e2e + * counts never cross the 1000-message watermark. + */ + private val flowControlWiringSuite = suite("the desired-paused set reaches the consumer hooks")( + test("pause fires ONCE at the watermark (transitions only), and resume fires at the cut") { + // Two real streams: p1 never delivers (blind), p0 crosses the per-stream watermark. + // The hooks land on per-consumer pause arbiters in production, so external resumes + // cannot stomp a hold any more - which is what lets the reconcile apply DIFFS: one + // pause call at the crossing, however many thousand offers follow, one resume at the + // cut. The call counts pin exactly that. + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip( + 1, + Vector(stream(consumerName, p0, 5_000), stream(otherConsumerName, p1, 5_000)) + ) + ) + var p0Paused = 0 + var p0Resumed = 0 + var p1Paused = 0 + ordering.registerStreamPauseHooks(startFromStreamId(consumerName, p0), () => { p0Paused += 1; true }, () => { p0Resumed += 1; true }) + ordering.registerStreamPauseHooks(startFromStreamId(otherConsumerName, p1), () => { p1Paused += 1; true }, () => true) + + // Cross the production per-stream watermark (1000) while p1 stays silent, then keep + // going: the extra offers must not produce extra pause calls. + (1L to (startFromMergePauseStreamAt + 50L)).foreach { i => + ordering.deliver(consumerName, p0, 100L + i, i, s"a$i") + ordering.reconcileFlowControl() + } + val pausedAtWatermark = p0Paused + val blindNeverPaused = p1Paused + + // p1 finally speaks with the globally-oldest message: the budget's only unit is spent + // on it and everything held drains - the cut releases the pause. + val atCut = ordering.deliver(otherConsumerName, p1, 1L, 0, "b1") + ordering.settleIfDone() + ordering.reconcileFlowControl() + + assertTrue( + pausedAtWatermark == 1, // TRANSITIONS only - not one call per offer past the mark + blindNeverPaused == 0, + atCut.head == ("b1" -> StartFromOutcome.Drop), + atCut.count(_._2 == StartFromOutcome.Deliver) == (startFromMergePauseStreamAt + 50).toInt, + p0Resumed >= 1 + ) ?? s"pausedAtWatermark=$pausedAtWatermark resumed=$p0Resumed p1Paused=$p1Paused atCutHead=${atCut.headOption}" + }, + test("a pause that FAILS once is retried until it takes - the books never say what did not happen") { + // The client call behind a hook can throw (a consumer mid-reconnect answers false + // through its arbiter). Recording the transition anyway meant no retry ever came: the + // stream kept delivering into a merge that believed it was paused. Success-only + // bookkeeping turns every later reconcile - each handled batch, each sweep tick - + // into the retry. + val ordering = StartFromOrdering.make[String]( + StartFromOrderingPlan.GlobalSkip( + 1, + Vector(stream(consumerName, p0, 5_000), stream(otherConsumerName, p1, 5_000)) + ) + ) + var pauseAttempts = 0 + var resumeAttempts = 0 + ordering.registerStreamPauseHooks( + startFromStreamId(consumerName, p0), + // The FIRST attempt fails; every later one succeeds. + pause = () => { pauseAttempts += 1; pauseAttempts > 1 }, + resume = () => { resumeAttempts += 1; true } + ) + ordering.registerStreamPauseHooks(startFromStreamId(otherConsumerName, p1), () => true, () => true) + + (1L to (startFromMergePauseStreamAt + 1L)).foreach { i => + ordering.deliver(consumerName, p0, 100L + i, i, s"a$i") + } + ordering.reconcileFlowControl() // attempt 1: refused, must NOT be recorded as applied + val attemptsAfterFailure = pauseAttempts + ordering.reconcileFlowControl() // the retry: nothing changed but the books were honest + val attemptsAfterRetry = pauseAttempts + ordering.reconcileFlowControl() // applied now - no further calls + assertTrue( + attemptsAfterFailure == 1, + attemptsAfterRetry == 2, + pauseAttempts == 2, + resumeAttempts == 0 + ) ?? s"pauseAttempts=$pauseAttempts resumeAttempts=$resumeAttempts" + } + ) + + def spec = suite(this.getClass.toString)(planSuite, identitySuite, backlogEndSuite, passThroughSuite, progressSuite, armOnceSuite, flowControlWiringSuite) diff --git a/server/src/test/scala/consumer/session_runner/startFromProgressTest.scala b/server/src/test/scala/consumer/session_runner/startFromProgressTest.scala new file mode 100644 index 000000000..1bf1de67c --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/startFromProgressTest.scala @@ -0,0 +1,619 @@ +package consumer.session_runner + +import _root_.consumer.coloring_rules.ColoringRuleChain +import _root_.consumer.deserializer.Deserializer +import _root_.consumer.deserializer.deserializers.UseLatestTopicSchema +import _root_.consumer.message_filter.MessageFilterChain +import _root_.consumer.pause_trigger.ConsumerSessionPauseTriggerChain +import _root_.consumer.session_config.ConsumerSessionConfig +import _root_.consumer.session_target.ConsumerSessionTarget +import _root_.consumer.session_target.consumption_mode.ConsumerSessionTargetConsumptionMode +import _root_.consumer.session_target.consumption_mode.modes.RegularConsumptionMode +import _root_.consumer.session_target.topic_selector.{MultiTopicSelector, TopicSelector} +import _root_.consumer.start_from.EarliestMessage +import _root_.consumer.value_projections.ValueProjectionList +import com.tools.teal.pulsar.ui.api.v1.consumer as consumerPb +import zio.test.* + +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters.* + +/** Progress reporting for a start-from position that has to be reached by COUNTING. + * + * "Skip the first n messages" is O(n) - Pulsar keeps no message-ordinal index, so the only exact + * way to land on message n is to stream n messages and throw them away. n is a number the user + * typed and is deliberately uncapped, so a session can spend a long time delivering nothing at all. + * Without a report the UI cannot tell that from a hung session. + * + * The awkward part is that a discarded message never reaches the delivery path - the listener drops + * it before the message handler - so nothing on the normal response path fires while the skip is in + * flight. The report therefore has to be pushed from the discard itself, and throttled, because one + * gRPC frame per skipped message would be millions of frames. + */ +object startFromProgressTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/progress" + private val otherTopicFqn = "persistent://public/default/progress-other" + + private final class RecordingObserver extends io.grpc.stub.StreamObserver[consumerPb.ResumeResponse]: + private val responses = java.util.concurrent.ConcurrentLinkedQueue[consumerPb.ResumeResponse]() + override def onNext(value: consumerPb.ResumeResponse): Unit = responses.add(value) + override def onError(t: Throwable): Unit = () + override def onCompleted(): Unit = () + def received: Vector[consumerPb.ResumeResponse] = responses.asScala.toVector + def progressReports: Vector[(Long, Long, Boolean)] = + received.flatMap(_.consumerStats).flatMap(_.startFromProgress).map(p => (p.messagesSkipped, p.messagesToSkip, p.complete)) + + /** A listener already open for business. It starts CLOSED in production - nothing may be + * consumed before the session is armed and a client has resumed it - and + * `ConsumerSessionTargetRunner.resume` is what opens it. */ + private def listener(): ConsumerListener = + val l = ConsumerListener(ConsumerSessionTargetMessageHandler(onNext = _ => ())) + l.startAcceptingNewMessages() + l + + private def targetConfig(topicFqns: Vector[String]): ConsumerSessionTarget = + ConsumerSessionTarget( + isEnabled = true, + consumptionMode = ConsumerSessionTargetConsumptionMode(mode = RegularConsumptionMode()), + messageValueDeserializer = Deserializer(deserializer = UseLatestTopicSchema()), + topicSelector = TopicSelector(topicSelector = MultiTopicSelector(topicFqns = topicFqns)), + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ) + + private def targetRunner(consumerListener: ConsumerListener, topicFqns: Vector[String] = Vector(topicFqn)): ConsumerSessionTargetRunner = + ConsumerSessionTargetRunner( + targetIndex = 0, + targetConfig = targetConfig(topicFqns), + nonPartitionedTopicFqns = topicFqns, + schemasByTopic = Map.empty, + sessionContextPool = ConsumerSessionContextPool(), + consumers = Map.empty, + pauseArbiters = Map.empty, + consumerListener = consumerListener, + stats = ConsumerSessionTargetStats(messageProcessed = AtomicLong(0)) + ) + + private def session(targets: (Int, ConsumerSessionTargetRunner)*): ConsumerSessionRunner = + ConsumerSessionRunner( + sessionName = "cs-progress", + sessionConfig = ConsumerSessionConfig( + startFrom = EarliestMessage(), + targets = Vector.empty, + messageFilterChain = MessageFilterChain.empty, + coloringRuleChain = ColoringRuleChain.empty, + pauseTriggerChain = ConsumerSessionPauseTriggerChain.empty, + valueProjectionList = ValueProjectionList(isEnabled = false, projections = Vector.empty) + ), + sessionContextPool = ConsumerSessionContextPool(), + grpcResponseObserver = None, + schemasByTopic = Map.empty, + targets = targets.toMap + ) + + private def progressOf(runner: ConsumerSessionRunner): Option[(Long, Long, Boolean)] = + runner.startFromProgress.map(p => (p.messagesSkipped, p.messagesToSkip, p.complete)) + + private val totalSuite = suite("what the discard was armed with")( + test("a shared discard remembers its total after the counter has been spent") { + // messagesToSkip is the number the user asked for; reading it off the live counter + // would report a total that shrinks to zero as the skip progresses. + val discard = StartFromDiscard.shared(5) + discard.claim(topicFqn) + discard.claim(topicFqn) + assertTrue(discard.total == 5L, discard.remaining == 3L) + }, + test("a per-topic discard totals every topic it covers") { + val discard = StartFromDiscard.perTopic(Map(topicFqn -> 2L, otherTopicFqn -> 3L)) + discard.claim(topicFqn) + assertTrue(discard.total == 5L, discard.remaining == 4L) + }, + test("the empty discard has nothing to skip") { + assertTrue(StartFromDiscard.none.total == 0L) + }, + test("a negative arming is clamped in the total too") { + // DEFENCE IN DEPTH, and no longer reachable from a request: a negative count is now + // REFUSED at the trust boundary (see `startFromCountRejectionReason`) instead of being + // clamped into a different valid position. This pins the counter's own behaviour so an + // internal caller that ever armed one negative cannot produce a nonsense total. + assertTrue(StartFromDiscard.shared(-7).total == 0L) + } + ) + + private val throttleSuite = suite("how often a skip in flight is reported")( + test("the very first skipped message is reported, so the UI learns the total immediately") { + assertTrue(shouldReportStartFromProgress(skipped = 1, total = 1_000_000, reportEvery = 10_000)) + }, + test("the last skipped message is reported, so the UI sees the run complete") { + assertTrue(shouldReportStartFromProgress(skipped = 1_000_000, total = 1_000_000, reportEvery = 10_000)) + }, + test("only every reportEvery-th message in between is reported") { + val reported = (1L to 30_000L).filter(skipped => shouldReportStartFromProgress(skipped, total = 1_000_000, reportEvery = 10_000)) + assertTrue(reported == Vector(1L, 10_000L, 20_000L, 30_000L)) ?? + s"a skip of a million must not put a frame on the wire per message, got ${reported.size} reports" + }, + test("a skip shorter than one interval still reports its start and its end") { + val reported = (1L to 5L).filter(skipped => shouldReportStartFromProgress(skipped, total = 5, reportEvery = 10_000)) + assertTrue(reported == Vector(1L, 5L)) + }, + test("a count past the total still reports - a race between listener threads must not lose the end") { + assertTrue(shouldReportStartFromProgress(skipped = 7, total = 5, reportEvery = 10_000)) + }, + test("nothing is reported for a session with nothing to skip") { + assertTrue( + !shouldReportStartFromProgress(skipped = 0, total = 0, reportEvery = 10_000), + !shouldReportStartFromProgress(skipped = 1, total = 0, reportEvery = 10_000) + ) + }, + test("a zero or negative interval degrades to first-and-last instead of dividing by zero") { + val reported = (1L to 20L).filter(skipped => shouldReportStartFromProgress(skipped, total = 20, reportEvery = 0)) + assertTrue(reported == Vector(1L, 20L)) + }, + test("the production interval is coarse enough that a million-message skip is a trickle") { + val reports = (1L to 1_000_000L).count(skipped => shouldReportStartFromProgress(skipped, 1_000_000, startFromProgressReportInterval)) + assertTrue(reports > 1, reports < 1000) ?? s"a 1,000,000 skip produced $reports progress frames" + } + ) + + private val sessionProgressSuite = suite("what the session reports")( + test("a session with no skip to do reports no progress at all") { + // Earliest / Latest / date-time / message-id / approximate all seek exactly: there is + // nothing to count, and a progress bar for them would be a lie. + val runner = session(0 -> targetRunner(listener())) + assertTrue(progressOf(runner).isEmpty) + }, + test("a skip in flight reports the total and how much of it is actually done") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(10) + val runner = session(0 -> targetRunner(l)) + val atStart = progressOf(runner) + (1 to 4).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + assertTrue(atStart == Some((0L, 10L, false)), progressOf(runner) == Some((4L, 10L, false))) + }, + test("progress completes exactly when the last message has been skipped") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = session(0 -> targetRunner(l)) + (1 to 2).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + val beforeLast = progressOf(runner) + l.decide(topicFqn, canAcknowledge = true) + val afterLast = progressOf(runner) + l.decide(topicFqn, canAcknowledge = true) // now delivering normally + assertTrue(beforeLast == Some((2L, 3L, false)), afterLast == Some((3L, 3L, true)), progressOf(runner) == Some((3L, 3L, true))) + }, + test("a shared counter across two targets is counted once, not twice") { + // Both targets hold the SAME counter; summing per target would report 20 to skip. + val shared = StartFromDiscard.shared(10) + val first = listener() + val second = listener() + first.startFromDiscard = shared + second.startFromDiscard = shared + val runner = session(0 -> targetRunner(first), 1 -> targetRunner(second)) + first.decide(topicFqn, canAcknowledge = true) + second.decide(topicFqn, canAcknowledge = true) + assertTrue(progressOf(runner) == Some((2L, 10L, false))) + }, + test("a LATEST-N seek correction is not user-facing progress") { + // The per-topic counter of a "latest n" is INTERNAL: the seek can only land on an entry + // boundary, so each topic over-fetches and drops its own overshoot. That is the session + // reaching the position the user asked for, not the user's own skip - and the proto + // says so ("Only NthMessageAfterEarliest needs this"). Reporting it told a client that + // asked for the last 5 messages that it was "skipping 95 messages". + val first = listener() + val second = listener() + first.startFromDiscard = StartFromDiscard.perTopic(Map(topicFqn -> 3L)) + second.startFromDiscard = StartFromDiscard.perTopic(Map(otherTopicFqn -> 2L)) + val runner = session(0 -> targetRunner(first), 1 -> targetRunner(second, Vector(otherTopicFqn))) + first.decide(topicFqn, canAcknowledge = true) + assertTrue(progressOf(runner).isEmpty) ?? + s"a latest-n overshoot was reported to the client as a skip: ${progressOf(runner)}" + }, + test("the latest-n correction still HAPPENS, it is just not reported") { + // The counter must keep dropping the overshoot - only its visibility changed. + val l = listener() + l.startFromDiscard = StartFromDiscard.perTopic(Map(topicFqn -> 2L)) + session(0 -> targetRunner(l)) + val actions = (1 to 4).map(_ => l.decide(topicFqn, canAcknowledge = true)) + assertTrue(actions.count(_ == ConsumerListener.Action.Drop) == 2, l.startFromDiscard.remaining == 0L) + }, + test("an ORDERING layer does not mask the latest-n overshoot from the effective counter") { + // "Latest n" + a delivery order (the DEFAULT) arms BOTH a per-topic overshoot counter + // on the listener AND an ordering-only layer whose own budget is a shared ZERO. The + // ordering layer does no dropping, so the zero must not shadow the listener's counter: + // `remainingStartFromDiscard` is what suppresses the rate limiter's permit hold while + // the overshoot is still being dropped, and reporting 0 there un-suppressed it in + // exactly the window the suppression protects. + val l = listener() + l.startFromDiscard = StartFromDiscard.perTopic(Map(topicFqn -> 3L)) + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.Ordered( + // Best effort carries EMPTY recorded ends - the continuous rules need none. + Vector( + StartFromStream(startFromStreamId("cs-progress-0", topicFqn), EntryPosition.empty), + StartFromStream(startFromStreamId("cs-progress-0", otherTopicFqn), EntryPosition.empty) + ), + _root_.consumer.session_config.MessageDeliveryOrder.BestEffort + ) + ) + val runner = session(0 -> targetRunner(l)) + + val atStart = runner.remainingStartFromDiscard + l.decide(topicFqn, canAcknowledge = true) + + assertTrue( + atStart == 3L, + runner.remainingStartFromDiscard == 2L, + // Still NOT user-facing progress: the overshoot is internal, whatever carries it. + progressOf(runner).isEmpty + ) ?? s"atStart=$atStart remaining=${runner.remainingStartFromDiscard} progress=${progressOf(runner)}" + }, + test("a MERGE-owned skip budget still answers through the effective counter") { + // The control for the fix above: when the ordering layer DOES own the budget (a + // multi-stream skip-n), its counter is the one doing the dropping and must keep + // being the one reported. + val l = listener() + l.startFromOrdering = StartFromOrdering.make[HeldMessage]( + StartFromOrderingPlan.GlobalSkip( + 4, + Vector( + StartFromStream(startFromStreamId("cs-progress-0", topicFqn), EntryPosition(1L, 9L, -1, 1)), + StartFromStream(startFromStreamId("cs-progress-0", otherTopicFqn), EntryPosition(1L, 9L, -1, 1)) + ) + ) + ) + val runner = session(0 -> targetRunner(l)) + assertTrue(runner.remainingStartFromDiscard == 4L) + }, + test("a SKIP-N counter is user-facing progress and is still reported") { + // The control: the same machinery, armed by "skip the first n", is exactly what the + // progress API exists for. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(4) + val runner = session(0 -> targetRunner(l)) + l.decide(topicFqn, canAcknowledge = true) + assertTrue(progressOf(runner) == Some((1L, 4L, false))) + }, + test("a message rejected while paused does not count as skipped") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(4) + val target = targetRunner(l) + val runner = session(0 -> target) + l.decide(topicFqn, canAcknowledge = true) + target.pause() + (1 to 5).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + assertTrue(progressOf(runner) == Some((1L, 4L, false))) + } + ) + + private val responseSuite = suite("what goes on the wire")( + test("a response carrying messages also carries the progress") { + val progress = consumerPb.StartFromProgress(messagesSkipped = 7, messagesToSkip = 9, complete = false) + val response = resumeResponse(Seq(consumerPb.Message(numMessageProcessed = 3)), Vector.empty, Some(progress)) + assertTrue( + response.consumerStats.flatMap(_.startFromProgress) == Some(progress), + response.messages.size == 1 + ) + }, + test("a response for a session with nothing to skip carries no stats at all") { + val response = resumeResponse(Seq(consumerPb.Message()), Vector.empty, None) + assertTrue(response.consumerStats.isEmpty) + }, + test("errors still decide the status, whether or not progress is attached") { + val withErrors = resumeResponse(Seq.empty, Vector("boom", "bang"), None) + val withoutErrors = resumeResponse(Seq.empty, Vector.empty, None) + assertTrue( + withErrors.getStatus.code == com.google.rpc.code.Code.UNKNOWN.value, + withErrors.getStatus.message.contains("boom") && withErrors.getStatus.message.contains("bang"), + withoutErrors.getStatus.code == com.google.rpc.code.Code.OK.value + ) + }, + test("a delivered message carries the completed progress, so the client clears its panel") { + // The client hides its progress panel on `complete` or on an absent field. If the + // responses that resume normal delivery dropped the stats, the panel would be left on + // screen for the rest of the session. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(2) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + // Wired like production: sendResponse now refuses an observer that is not the + // session's current play stream, so the shape test resumes first. + runner.resume(observer, isDebug = false) + (1 to 2).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + runner.sendResponse(observer, Seq(consumerPb.Message(numMessageProcessed = 5)), Vector.empty) + + // The resumed listener also pushes real in-flight progress frames as the drops land; + // the pinned claim is about the DELIVERED message's frame - the last one out. + assertTrue( + observer.progressReports.last == (2L, 2L, true), + observer.received.last.messages.size == 1 + ) + }, + test("a delivered message on a session that skipped nothing carries no stats at all") { + val runner = session(0 -> targetRunner(listener())) + val observer = RecordingObserver() + runner.resume(observer, isDebug = false) + + runner.sendResponse(observer, Seq(consumerPb.Message()), Vector.empty) + + assertTrue(observer.received.size == 1, observer.received.head.consumerStats.isEmpty) + }, + test("delivery never begins while the skip is still reported as in flight") { + // The invariant the client's "clear on complete" rule depends on: the first message that + // is NOT dropped must already see a completed progress, never a stale in-flight one. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = session(0 -> targetRunner(l)) + + val atDelivery = (1 to 6).map(_ => (l.decide(topicFqn, canAcknowledge = true), progressOf(runner))).collect { + case (ConsumerListener.Action.Deliver, progress) => progress + } + + assertTrue(atDelivery.size == 3, atDelivery.forall(_.exists((_, _, complete) => complete))) ?? + s"a message was delivered while the skip still claimed to be running: $atDelivery" + }, + test("a progress-only push is an OK, message-less response") { + // The client reads the trailing message's counters only when a message is present, so a + // message-less progress frame must not pretend to carry one. + val response = resumeResponse(Seq.empty, Vector.empty, Some(consumerPb.StartFromProgress(messagesSkipped = 1, messagesToSkip = 9))) + assertTrue( + response.messages.isEmpty, + response.getStatus.code == com.google.rpc.code.Code.OK.value, + response.consumerStats.flatMap(_.startFromProgress).map(_.messagesSkipped) == Some(1L) + ) + } + ) + + private val pushSuite = suite("pushing progress while nothing is being delivered")( + test("a skip in flight pushes progress to the client although no message is delivered") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + + runner.resume(observer, isDebug = false) + (1 to 3).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue(observer.progressReports == Vector((1L, 3L, false), (3L, 3L, true))) ?? + s"expected a first and a completing report, got ${observer.progressReports}" + }, + test("a long skip is reported periodically, not per message") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(25_000) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + + runner.resume(observer, isDebug = false) + (1 to 25_000).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue( + observer.progressReports.head == (1L, 25_000L, false), + observer.progressReports.last == (25_000L, 25_000L, true), + observer.progressReports.size == 4 // 1, 10_000, 20_000, 25_000 + ) ?? s"got ${observer.progressReports}" + }, + test("a session with nothing to skip pushes nothing") { + val l = listener() + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + + runner.resume(observer, isDebug = false) + (1 to 100).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue(observer.received.isEmpty) + }, + test("nothing is pushed before the session is resumed") { + // The listener is armed at session creation, long before any client is listening. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + session(0 -> targetRunner(l)) + val dropped = (1 to 3).map(_ => l.decide(topicFqn, canAcknowledge = true)) + assertTrue(dropped.forall(_ == ConsumerListener.Action.Drop)) + }, + test("a client that did NOT ask for consumer stats is sent none") { + // `ResumeRequest.include_consumer_stats` is the client saying whether it wants them. + // It was read off the request and then dropped on the floor, so every client got the + // stats - and, worse, got message-LESS progress frames it never asked to handle. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + + runner.resume(observer, isDebug = false, includeConsumerStats = false) + (1 to 3).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + runner.sendResponse(observer, Seq(consumerPb.Message(numMessageProcessed = 1)), Vector.empty) + + assertTrue( + observer.received.size == 1, + observer.received.head.consumerStats.isEmpty + ) ?? s"a client that asked for no stats received ${observer.received.size} responses: ${observer.progressReports}" + }, + test("a client that DID ask for consumer stats still gets them") { + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(3) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + + runner.resume(observer, isDebug = false, includeConsumerStats = true) + (1 to 3).foreach(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue(observer.progressReports == Vector((1L, 3L, false), (3L, 3L, true))) + }, + test("resuming again keeps reporting to the NEW client without re-arming the skip") { + // Pause/resume must not skip a fresh batch - and the second client must still see where + // the skip got to. + val l = listener() + l.startFromDiscard = StartFromDiscard.shared(4) + val target = targetRunner(l) + val runner = session(0 -> target) + val first = RecordingObserver() + val second = RecordingObserver() + + runner.resume(first, isDebug = false) + l.decide(topicFqn, canAcknowledge = true) + runner.pause() + runner.resume(second, isDebug = false) + val afterResume = (1 to 5).map(_ => l.decide(topicFqn, canAcknowledge = true)) + + assertTrue( + first.progressReports == Vector((1L, 4L, false)), + second.progressReports == Vector((4L, 4L, true)), + afterResume.count(_ == ConsumerListener.Action.Drop) == 3, + l.startFromDiscard.remaining == 0L + ) ?? s"first=${first.progressReports} second=${second.progressReports} actions=$afterResume" + } + ) @@ TestAspect.sequential + + private val stallDisclosureSuite = suite("disclosing a stalled delivery order with no counted skip")( + test("a GUARANTEED stall pushes the waiting-streams disclosure although nothing is skipped or delivered") { + // H2's exact shape: an ordering-ONLY session (no skip-n) stalls under Guaranteed. Its + // budget is a shared ZERO, so `startFromProgress` is None - and the old gate hung the + // whole push on it, so no ResumeResponse was ever written: the session sat in state + // `running`, indistinguishable from an empty topic. The stall must reach the client + // through the same stats channel every response carries, throttled to one message-less + // frame per NEW stall warning - never a frame per held message or per sweep tick. + // + // The full production path is driven: a real GuaranteedOnly merge with an injected + // clock, the real listener sweep, the real runner closure, a recording observer. The + // runner is PAUSED after resume so the background sweep timer and the delivery pump + // stay out of the way - the sweep is driven by hand, deterministically. + val clock = new AtomicLong(0L) + val streamA = startFromStreamId("cs-progress-0", topicFqn) + val streamB = startFromStreamId("cs-progress-0", otherTopicFqn) + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector(streamA, streamB), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.get, + policy = OrderingPolicy.GuaranteedOnly + ) + val l = listener() + // MID-REPLAY streams: far-future recorded ends, so nothing offered here reads as + // finished - the stall being disclosed is a stream that still owes its recorded + // range. (Empty ends would mean "nothing to replay", which since the exact-replay + // redesign finishes the stream instead of waiting on it.) + l.startFromOrdering = new StartFromOrdering[HeldMessage]( + Some(merge), + Map( + streamA -> StartFromStream(streamA, EntryPosition(1L, 1_000_000L, -1, 1)), + streamB -> StartFromStream(streamB, EntryPosition(1L, 1_000_000L, -1, 1)) + ) + ) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + runner.resume(observer, isDebug = false) + // Cancels the real 250ms sweep timer and pauses the drain - which also short-circuits + // the guaranteed pump's pacing check, so the held payloads below are never dereferenced. + runner.pause() + + // Stream a speaks; b stays silent. Guaranteed holds a-100 and waits on b, forever. + var nextEntryId = 0L + def offer(streamFqn: String, orderTime: Long): Unit = + nextEntryId += 1 + l.startFromOrdering.offer( + consumerName = "cs-progress-0", + topicFqn = streamFqn, + orderTime = orderTime, + messageId = new org.apache.pulsar.client.impl.MessageIdImpl(1L, nextEntryId, -1), + // Never delivered in this test (the pump is short-circuited by the paused + // drain), so the payload's consumer and message are never touched. + payload = HeldMessage(null, null, l) + ) + () + offer(topicFqn, 100L) + l.sweepStartFromStall() // stamps the blind clock; the warning window has not passed + val beforeWarning = observer.received.size + + clock.set(startFromMergeStallWarnMs + 1) + l.sweepStartFromStall() // crosses the warn window: the disclosure must go out NOW + val atFirstWarning = observer.received.size + // The server-side diagnostic knows WHICH stream is waited on; the wire carries only + // the count (there is no per-stream proto field). + val stalledIds = l.startFromOrdering.stalledStreamIds + l.sweepStartFromStall() // still stalled, no NEW warning: the throttle holds the line + // A drop-path report while the SAME stall persists (what every latest-n overshoot drop + // fires) must not re-push either - the disclosure is once per NEW warning, wherever + // the report comes from. + l.onStartFromDiscardProgress() + val afterQuietSweep = observer.received.size + + // b finally speaks: the stall resolves... + offer(otherTopicFqn, 50L) + val waitingAfterRecovery = l.startFromOrdering.stalledStreamCount + // ...its head is consumed (by hand, through the barrier - no delivery machinery)... + l.startFromOrdering.inOrder { + l.startFromOrdering.peekGuaranteed() + l.startFromOrdering.commitGuaranteed() + } + // ...and b is silent again: a FRESH stall must disclose again, exactly once. + l.sweepStartFromStall() // stamps the new blind clock + clock.addAndGet(startFromMergeStallWarnMs + 1) + l.sweepStartFromStall() + + val frames = observer.received + assertTrue( + beforeWarning == 0, + atFirstWarning == 1, + stalledIds == Vector(streamB), + afterQuietSweep == 1, + waitingAfterRecovery == 0, + frames.size == 2, + frames.forall(_.messages.isEmpty), + frames.forall(_.consumerStats.exists(stats => + stats.deliveryOrderWaitingStreams == 1 && stats.deliveryOrderActive && stats.startFromProgress.isEmpty + )) + ) ?? (s"beforeWarning=$beforeWarning atFirstWarning=$atFirstWarning afterQuietSweep=$afterQuietSweep " + + s"frames=${frames.map(f => (f.messages.size, f.consumerStats))}") + }, + test("a client that asked for no stats is not sent the stall disclosure frame") { + val clock = new AtomicLong(0L) + val streamA = startFromStreamId("cs-progress-0", topicFqn) + val streamB = startFromStreamId("cs-progress-0", otherTopicFqn) + val merge = GlobalSkipMerge[HeldMessage]( + streamIds = Vector(streamA, streamB), + drainedAtStart = Set.empty, + discard = StartFromDiscard.shared(0), + nowMs = () => clock.get, + policy = OrderingPolicy.GuaranteedOnly + ) + val l = listener() + // Mid-replay shape, exactly as the disclosure test above: far-future recorded ends. + l.startFromOrdering = new StartFromOrdering[HeldMessage]( + Some(merge), + Map( + streamA -> StartFromStream(streamA, EntryPosition(1L, 1_000_000L, -1, 1)), + streamB -> StartFromStream(streamB, EntryPosition(1L, 1_000_000L, -1, 1)) + ) + ) + val runner = session(0 -> targetRunner(l)) + val observer = RecordingObserver() + runner.resume(observer, isDebug = false, includeConsumerStats = false) + runner.pause() + + l.startFromOrdering.offer("cs-progress-0", topicFqn, 100L, new org.apache.pulsar.client.impl.MessageIdImpl(1L, 1L, -1), HeldMessage(null, null, l)) + l.sweepStartFromStall() + clock.set(startFromMergeStallWarnMs + 1) + l.sweepStartFromStall() + + assertTrue(observer.received.isEmpty) ?? + s"a stats-less client received ${observer.received.size} stall frames" + }, + test("the stall diagnostic classifies empty-forever, drained-to-end, and lagging streams") { + // The debug log's classification is pure so it is pinned without a broker: a stream + // with nothing ever published, a stream the session has read to its end, and a stream + // that verifiably holds messages the session has not received yet must each read + // differently - the last is the case the guarantee exists for. + val end = EntryPosition(1L, 10L, -1, 1) + assertTrue( + stalledStreamEmptinessNote(None, None).contains("could not be read"), + stalledStreamEmptinessNote(Some(EntryPosition.empty), None).contains("no message has ever been published"), + stalledStreamEmptinessNote(Some(end), Some(EntryPosition(1L, 10L, -1, 1))).contains("waiting for a NEW message"), + stalledStreamEmptinessNote(Some(end), Some(EntryPosition(1L, 4L, -1, 1))).contains("delivery lag"), + stalledStreamEmptinessNote(Some(end), None).contains("delivery lag") + ) + } + ) @@ TestAspect.sequential + + def spec = suite(this.getClass.toString)(totalSuite, throttleSuite, sessionProgressSuite, responseSuite, pushSuite, stallDisclosureSuite) diff --git a/server/src/test/scala/consumer/session_runner/topicPositionsTest.scala b/server/src/test/scala/consumer/session_runner/topicPositionsTest.scala new file mode 100644 index 000000000..d3c65f4ee --- /dev/null +++ b/server/src/test/scala/consumer/session_runner/topicPositionsTest.scala @@ -0,0 +1,202 @@ +package consumer.session_runner + +import zio.test.* +import zio.test.Assertion.* +import org.apache.pulsar.client.impl.{BatchMessageIdImpl, MessageIdImpl} + +/** The per-topic debug view's arithmetic and its refusals. + * + * Every case here is one the broker actually produces - the open-ledger hole is measured, the empty + * topic and the non-persistent refusal are the failure modes `startFromLookups` already classifies, + * and the backwards range is what a producer clock that stepped back leaves behind. The point of + * the suite is that each of them reports NOTHING rather than a plausible-looking zero. + */ +object topicPositionsTest extends ZIOSpecDefault: + + private def msgId(ledger: Long, entry: Long) = MessageIdImpl(ledger, entry, -1) + private def endpoint(publishTime: Long, ledger: Long = 1, entry: Long = 0) = + LogEndpoint(msgId(ledger, entry), publishTime) + private def cursorAt(publishTime: Long, ledger: Long = 1, entry: Long = 0) = + TopicCursor(msgId(ledger, entry), publishTime) + + private val inputs = TopicPositionInputs( + topicFqn = "persistent://t/n/topic", + first = None, + last = None, + firstConsumed = None, + cursor = None, + ledgers = Vector.empty, + currentLedgerEntries = 0, + retainedEntries = 0, + unavailableReason = None + ) + + def spec = suite("topicPositions")( + suite("the open-ledger hole")( + test("the CURRENT ledger's real entry count is used where the list reports zero") { + // Measured on a live 6060-entry topic: the one open ledger reports entries 0 while + // currentLedgerEntries holds 6060. Walking the list as reported puts every cursor in + // that ledger at ordinal 1 - "0% through" for the whole life of the ledger. + val patched = retainedLedgerSpans(Vector(LedgerSpan(7057, 0)), currentLedgerEntries = 6060) + assertTrue(patched == Vector(LedgerSpan(7057, 6060))) + }, + test("a CLOSED ledger reporting zero in the middle of the list is left alone") { + val ledgers = Vector(LedgerSpan(1, 0), LedgerSpan(2, 50), LedgerSpan(3, 0)) + val patched = retainedLedgerSpans(ledgers, currentLedgerEntries = 7) + // Only the last is patched; the leading zero is somebody else's business. + assertTrue(patched == Vector(LedgerSpan(1, 0), LedgerSpan(2, 50), LedgerSpan(3, 7))) + }, + test("a last ledger that already reports entries is NOT overwritten") { + val ledgers = Vector(LedgerSpan(1, 10), LedgerSpan(2, 20)) + assertTrue(retainedLedgerSpans(ledgers, currentLedgerEntries = 999) == ledgers) + }, + test("an empty ledger list stays empty rather than growing a phantom ledger") { + assertTrue(retainedLedgerSpans(Vector.empty, currentLedgerEntries = 500).isEmpty) + } + ), + suite("entry ordinal")( + test("counts every entry in the ledgers BEFORE the cursor's own, and is 1-based") { + val ledgers = Vector(LedgerSpan(1, 100), LedgerSpan(2, 50), LedgerSpan(3, 10)) + // Entry 0 of ledger 2 is the 101st retained entry. + assertTrue(entryOrdinalOf(ledgers, 2, 0).contains(101L)) && + assertTrue(entryOrdinalOf(ledgers, 2, 49).contains(150L)) && + assertTrue(entryOrdinalOf(ledgers, 1, 0).contains(1L)) && + assertTrue(entryOrdinalOf(ledgers, 3, 9).contains(160L)) + }, + test("a cursor whose ledger has AGED OUT reports nothing, not the beginning") { + // Retention trimmed ledger 1 from under a session that had read it. Answering 1, or + // 0%, would claim the session is at the start when it is in fact past it. + val ledgers = Vector(LedgerSpan(2, 50), LedgerSpan(3, 10)) + assertTrue(entryOrdinalOf(ledgers, 1, 5).isEmpty) + } + ), + suite("time fraction")( + test("places the cursor proportionally between the endpoints") { + val f = cursorTimeFractionOf(Some(endpoint(1000)), Some(endpoint(2000)), Some(cursorAt(1500))) + assertTrue(f.contains(0.5)) + }, + test("a topic occupying ONE INSTANT has no interior, so no fraction describes it") { + // first == last: no position separates the messages. 0.0 and 1.0 would both be + // inventions - the same rule ApproximatePublishTimePosition follows. + val f = cursorTimeFractionOf(Some(endpoint(1000)), Some(endpoint(1000)), Some(cursorAt(1000))) + assertTrue(f.isEmpty) + }, + test("a range reported BACKWARDS is refused rather than turned into a negative fraction") { + // Publish time is stamped by the producer, so a clock that stepped back produces it. + val f = cursorTimeFractionOf(Some(endpoint(2000)), Some(endpoint(1000)), Some(cursorAt(1500))) + assertTrue(f.isEmpty) + }, + test("a cursor PAST the recorded end clamps to 1.0 - a stale denominator, not an overrun") { + // The endpoints and the cursor are not read atomically: a message published between + // the two lookups leaves the cursor beyond the last entry that was recorded. + val f = cursorTimeFractionOf(Some(endpoint(1000)), Some(endpoint(2000)), Some(cursorAt(9999))) + assertTrue(f.contains(1.0)) + }, + test("no cursor and no endpoints each yield nothing") { + assertTrue(cursorTimeFractionOf(Some(endpoint(1000)), Some(endpoint(2000)), None).isEmpty) && + assertTrue(cursorTimeFractionOf(None, Some(endpoint(2000)), Some(cursorAt(1500))).isEmpty) && + assertTrue(cursorTimeFractionOf(Some(endpoint(1000)), None, Some(cursorAt(1500))).isEmpty) + } + ), + suite("entry fraction")( + test("is the ordinal over the retained count") { + assertTrue(cursorEntryFractionOf(Some(50L), 100L).contains(0.5)) + }, + test("the BOUNDARIES read as proportion CONSUMED: first of N is 1/N, last is 1.0, a single entry is 1.0") { + // The contract the proto states: a cursor exists only once something was consumed, + // so there is no 0.0 with a cursor present - sitting ON the first of 100 entries + // means one entry consumed, 1%. A one-entry topic is fully consumed by its first + // read. Pinned here so a future switch to geometric position (ordinal-1 over N) + // has to change the contract on purpose, in both places. + assertTrue(cursorEntryFractionOf(Some(1L), 100L).contains(0.01)) && + assertTrue(cursorEntryFractionOf(Some(100L), 100L).contains(1.0)) && + assertTrue(cursorEntryFractionOf(Some(1L), 1L).contains(1.0)) + }, + test("a topic retaining NOTHING has no denominator, and 0/0 is not 0%") { + assertTrue(cursorEntryFractionOf(Some(1L), 0L).isEmpty) + }, + test("clamps rather than exceeding 1.0 when the count is staler than the cursor") { + assertTrue(cursorEntryFractionOf(Some(150L), 100L).contains(1.0)) + } + ), + suite("batch ids")( + test("two messages of ONE BATCH share the entry, so they share the ordinal") { + // The ordinal walk is entry-addressed. A batch index is a third coordinate inside the + // entry and must not shift the count. + val ledgers = Vector(LedgerSpan(1, 100)) + val first = BatchMessageIdImpl(1, 7, -1, 0) + val fifth = BatchMessageIdImpl(1, 7, -1, 4) + val rowOf = (id: org.apache.pulsar.client.api.MessageId) => + buildTopicPositionRow( + inputs.copy( + cursor = Some(TopicCursor(id, 1500)), + ledgers = ledgers, + retainedEntries = 100 + ) + ).cursorEntryOrdinal + assertTrue(rowOf(first) == rowOf(fifth)) && assertTrue(rowOf(first).contains(8L)) + } + ), + suite("assembled rows")( + test("an EMPTY topic reports no endpoints and no reason - it answered, with nothing in it") { + val row = buildTopicPositionRow(inputs) + assertTrue(row.first.isEmpty) && assertTrue(row.last.isEmpty) && + assertTrue(row.unavailableReason.isEmpty) && + assertTrue(row.cursorTimeFraction.isEmpty) && assertTrue(row.cursorEntryFraction.isEmpty) + }, + test("an UNAVAILABLE topic reports a reason and withholds the entry count") { + // A non-persistent topic: Pulsar refuses to examine it (405). Its retained count is 0 + // only because nothing filled it in, and a "0 entries" cell reads as an empty topic. + val row = buildTopicPositionRow(inputs.copy(unavailableReason = Some("non-persistent topic"))) + assertTrue(row.unavailableReason.contains("non-persistent topic")) && + assertTrue(row.retainedEntries.isEmpty) + }, + test("an UNAVAILABLE retained log still carries session-local consumed bounds") { + val firstConsumed = cursorAt(1200, ledger = 1, entry = 3) + val lastConsumed = cursorAt(1800, ledger = 1, entry = 9) + val row = buildTopicPositionRow( + inputs.copy( + firstConsumed = Some(firstConsumed), + cursor = Some(lastConsumed), + unavailableReason = Some("non-persistent topic") + ) + ) + + assertTrue(row.firstConsumed.contains(firstConsumed)) && + assertTrue(row.cursor.contains(lastConsumed)) && + assertTrue(row.retainedEntries.isEmpty) + }, + test("a fully populated row carries both fractions and the ordinal") { + val firstConsumed = cursorAt(1100, ledger = 1, entry = 9) + val row = buildTopicPositionRow( + inputs.copy( + first = Some(endpoint(1000, ledger = 1, entry = 0)), + last = Some(endpoint(2000, ledger = 1, entry = 99)), + firstConsumed = Some(firstConsumed), + cursor = Some(cursorAt(1500, ledger = 1, entry = 49)), + ledgers = Vector(LedgerSpan(1, 0)), + currentLedgerEntries = 100, + retainedEntries = 100 + ) + ) + assertTrue(row.cursorTimeFraction.contains(0.5)) && + assertTrue(row.firstConsumed.contains(firstConsumed)) && + assertTrue(row.cursorEntryOrdinal.contains(50L)) && + assertTrue(row.cursorEntryFraction.contains(0.5)) && + assertTrue(row.retainedEntries.contains(100L)) && + { + val serialized = topicPositionToPb(row) + assertTrue(serialized.firstConsumedMessageId.exists(_.toByteArray.sameElements(firstConsumed.messageId.toByteArray))) && + assertTrue(serialized.firstConsumedPublishTime.contains(1100L)) && + assertTrue(serialized.cursorPublishTime.contains(1500L)) + } + }, + test("unset consumed bounds stay absent on the wire") { + val serialized = topicPositionToPb(buildTopicPositionRow(inputs)) + assertTrue(serialized.firstConsumedMessageId.isEmpty) && + assertTrue(serialized.firstConsumedPublishTime.isEmpty) && + assertTrue(serialized.cursorMessageId.isEmpty) && + assertTrue(serialized.cursorPublishTime.isEmpty) + } + ) + ) diff --git a/server/src/test/scala/consumer/session_target/topic_selector/multiTopicSelectorTest.scala b/server/src/test/scala/consumer/session_target/topic_selector/multiTopicSelectorTest.scala new file mode 100644 index 000000000..d1a146bbe --- /dev/null +++ b/server/src/test/scala/consumer/session_target/topic_selector/multiTopicSelectorTest.scala @@ -0,0 +1,67 @@ +package consumer.session_target.topic_selector + +import org.apache.pulsar.client.admin.PulsarAdmin +import zio.test.* + +import java.util.concurrent.TimeUnit +import scala.util.Try + +/** `MultiTopicSelector` turns the FQNs the user picked into the concrete non-partitioned topics a + * session subscribes to. + * + * Regression context: a topic whose partitioning could not be read was logged with `println` and + * replaced by `Vector.empty`, i.e. silently dropped from the selection. With every topic + * unresolvable (an unreachable broker, a topic deleted between the picker and the session) the + * whole selector returned an empty vector, the session runner accepted a target with zero + * consumers, and `createConsumer` answered Code.OK - a session in state `running` that could never + * deliver a message. The sibling `NamespacedRegexTopicSelector` never swallowed these. + * + * Driven with a REAL PulsarAdmin aimed at a closed port: it constructs offline (see + * `pulsar_auth.ClientConstructionTest`) and then fails every call with a connection error, which is + * exactly the production failure. No broker, no mock. + */ +object multiTopicSelectorTest extends ZIOSpecDefault: + + private def withUnreachableAdmin[A](f: PulsarAdmin => A): A = + val admin = PulsarAdmin.builder + .serviceHttpUrl("http://127.0.0.1:1") + .connectionTimeout(2, TimeUnit.SECONDS) + .readTimeout(2, TimeUnit.SECONDS) + .requestTimeout(2, TimeUnit.SECONDS) + .build + try f(admin) + finally Try(admin.close()) + + def spec = suite(this.getClass.toString)( + test("a topic whose partitioning cannot be resolved fails loudly instead of being dropped") { + val topicFqn = "persistent://public/default/topic-that-cannot-be-resolved" + val result = withUnreachableAdmin(admin => Try(MultiTopicSelector(Vector(topicFqn)).getNonPartitionedTopics(admin))) + val message = result.failed.toOption.map(_.getMessage).getOrElse("") + + assertTrue(result.isFailure, message.contains(topicFqn)) ?? + s"an unresolvable topic must not be silently dropped, got: $result" + }, + test("one unresolvable topic fails the whole selection rather than returning the rest") { + // The partial-drop case: the user explicitly named three topics, so quietly consuming + // from a subset is just as wrong as quietly consuming from none. + val result = withUnreachableAdmin(admin => + Try( + MultiTopicSelector(Vector( + "persistent://public/default/t1", + "persistent://public/default/t2", + "persistent://public/default/t3" + )).getNonPartitionedTopics(admin) + ) + ) + + assertTrue(result.isFailure) + }, + test("a selector with no topics resolves to an empty vector without contacting the broker") { + // Control for the session-runner guard below: MultiTopicSelector reports an honest + // empty result here (no failure to report), which is why rejecting a target that + // resolves to nothing has to happen in ConsumerSessionRunner. + val result = withUnreachableAdmin(admin => Try(MultiTopicSelector(Vector.empty).getNonPartitionedTopics(admin))) + + assertTrue(result == scala.util.Success(Vector.empty)) + } + ) diff --git a/server/src/test/scala/consumer/start_from/startFromConversionsTest.scala b/server/src/test/scala/consumer/start_from/startFromConversionsTest.scala new file mode 100644 index 000000000..fca6fd70c --- /dev/null +++ b/server/src/test/scala/consumer/start_from/startFromConversionsTest.scala @@ -0,0 +1,143 @@ +package consumer.start_from + +import com.tools.teal.pulsar.ui.api.v1.consumer as pb +import com.tools.teal.pulsar.ui.library.v1.managed_items as managedPb +import zio.test.* + +import java.time.Instant +import scala.util.{Failure, Success, Try} + +/** Every start-from mode has to survive the trip to protobuf and back. + * + * `ConsumerSessionStartFrom` is a UNION type, so its `match`es are NOT checked for exhaustiveness - + * a mode the conversion forgot falls into `case _ => throw`, and the failure surfaces at RUNTIME as + * a generic FAILED_PRECONDITION on saving or loading a session, not at compile time. Adding a mode + * without adding it here is therefore silent until a user picks it, which is exactly what happened + * to the two Nth modes: they were readable (`fromPb`) but not writable (`toPb`). + * + * The sweep below is over ALL modes, so a mode added later fails here rather than in production. + */ +object startFromConversionsTest extends ZIOSpecDefault: + + private val allModes: Vector[ConsumerSessionStartFrom] = Vector( + EarliestMessage(), + LatestMessage(), + NthMessageAfterEarliest(n = 5), + NthMessageBeforeLatest(n = 7), + MessageId(messageIdBytes = Array[Byte](8, 1, 16, 2)), + DateTime(dateTime = Instant.ofEpochSecond(1_700_000_000L, 123_000_000)), + RelativeDateTime(value = 3, unit = DateTimeUnit.Hour, isRoundedToUnitStart = true), + ApproximateEntryPosition(fraction = 0.42), + ApproximatePublishTimePosition(fraction = 0.42) + ) + + /** MessageId holds an Array, whose `==` is reference identity - a round trip would never look + * equal without this. */ + private def sameMode(a: ConsumerSessionStartFrom, b: ConsumerSessionStartFrom): Boolean = + (a, b) match + case (x: MessageId, y: MessageId) => x.messageIdBytes.sameElements(y.messageIdBytes) + case _ => a == b + + private def roundTrip(mode: ConsumerSessionStartFrom): Try[ConsumerSessionStartFrom] = + Try(ConsumerSessionStartFrom.fromPb(ConsumerSessionStartFrom.toPb(mode))) + + def spec = suite(this.getClass.toString)( + test("every start-from mode survives a proto round trip") { + val failures = allModes.flatMap { mode => + roundTrip(mode) match + case Success(back) if sameMode(back, mode) => None + case Success(back) => Some(s"$mode came back as $back") + case Failure(err) => Some(s"$mode threw ${err.getClass.getSimpleName}: ${err.getMessage}") + } + assertTrue(failures.isEmpty) ?? s"start-from modes that do not round trip: ${failures.mkString("; ")}" + }, + test("an approximate ENTRY position travels in the start_from_approximate_entry_position field") { + val encoded = ConsumerSessionStartFrom.toPb(ApproximateEntryPosition(fraction = 0.6)) + assertTrue( + encoded.startFrom.isStartFromApproximateEntryPosition, + encoded.getStartFromApproximateEntryPosition.fraction == 0.6 + ) + }, + test("an approximate PUBLISH-TIME position travels in its own field, not the entry one") { + // The two modes carry the same payload - a single double - so a conversion that reached + // for the wrong oneof case would still round-trip a fraction and look correct. The only + // symptom would be a session positioned by the wrong rule. + val encoded = ConsumerSessionStartFrom.toPb(ApproximatePublishTimePosition(fraction = 0.6)) + assertTrue( + encoded.startFrom.isStartFromApproximatePublishTimePosition, + !encoded.startFrom.isStartFromApproximateEntryPosition, + encoded.getStartFromApproximatePublishTimePosition.fraction == 0.6 + ) + }, + test("an approximate entry position is read back off the wire as the model type") { + val decoded = ConsumerSessionStartFrom.fromPb( + pb.ConsumerSessionStartFrom( + startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximateEntryPosition( + pb.ApproximateEntryPosition(fraction = 0.35) + ) + ) + ) + assertTrue(decoded == ApproximateEntryPosition(fraction = 0.35)) + }, + test("an approximate publish-time position is read back off the wire as the model type") { + val decoded = ConsumerSessionStartFrom.fromPb( + pb.ConsumerSessionStartFrom( + startFrom = pb.ConsumerSessionStartFrom.StartFrom.StartFromApproximatePublishTimePosition( + pb.ApproximatePublishTimePosition(fraction = 0.35) + ) + ) + ) + assertTrue(decoded == ApproximatePublishTimePosition(fraction = 0.35)) + }, + test("the fraction is carried at full double precision, not rounded on the way") { + // A UI slider at 1/3 must not come back as 0.33: the entry ordinal and publish-time + // cutoff are both computed from it. + val precise = 1.0 / 3.0 + assertTrue( + roundTrip(ApproximateEntryPosition(fraction = precise)) == Success(ApproximateEntryPosition(fraction = precise)), + roundTrip(ApproximatePublishTimePosition(fraction = precise)) == Success(ApproximatePublishTimePosition(fraction = precise)) + ) + }, + test("an unset oneof is still rejected") { + // The catch-all `case _` must stay: a start-from with nothing selected is not a mode. + val result = Try(ConsumerSessionStartFrom.fromPb(pb.ConsumerSessionStartFrom())) + assertTrue(result.isFailure) + }, + test("every start-from mode survives a managed-item (library) round trip") { + // The saved-session side of the same union, with its own separate pair of matches. + import _root_.library.managed_items.ManagedConsumerSessionStartFromSpec + val managedModes = Vector( + EarliestMessage(), + LatestMessage(), + NthMessageAfterEarliest(n = 5), + NthMessageBeforeLatest(n = 7), + ApproximateEntryPosition(fraction = 0.42), + ApproximatePublishTimePosition(fraction = 0.42) + ) + val failures = managedModes.flatMap { mode => + val spec = ManagedConsumerSessionStartFromSpec(startFrom = mode) + Try(ManagedConsumerSessionStartFromSpec.fromPb(ManagedConsumerSessionStartFromSpec.toPb(spec))) match + case Success(back) if back == spec => None + case Success(back) => Some(s"$mode came back as ${back.startFrom}") + case Failure(err) => Some(s"$mode threw ${err.getClass.getSimpleName}: ${err.getMessage}") + } + assertTrue(failures.isEmpty) ?? s"managed start-from modes that do not round trip: ${failures.mkString("; ")}" + }, + test("saved pre-rename percentage modes remain wire-compatible") { + // Exact old-schema wire fixtures: managed oneof tag 8/9, containing a double fraction + // in nested field 1. Field names are absent from protobuf binary data, so retaining the + // tags must map the legacy fields to their clearer entry/publish-time names. + val nestedHalf = Array[Byte](0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0.toByte, 0x3f) + val oldDataMode = Array[Byte](0x42, nestedHalf.length.toByte) ++ nestedHalf + val oldTimeMode = Array[Byte](0x4a, nestedHalf.length.toByte) ++ nestedHalf + + import _root_.library.managed_items.ManagedConsumerSessionStartFromSpec + val entry = ManagedConsumerSessionStartFromSpec.fromPb(managedPb.ManagedConsumerSessionStartFromSpec.parseFrom(oldDataMode)) + val publishTime = ManagedConsumerSessionStartFromSpec.fromPb(managedPb.ManagedConsumerSessionStartFromSpec.parseFrom(oldTimeMode)) + + assertTrue( + entry.startFrom == ApproximateEntryPosition(0.5), + publishTime.startFrom == ApproximatePublishTimePosition(0.5) + ) + } + ) diff --git a/server/src/test/scala/conversions/primitiveConvTest.scala b/server/src/test/scala/conversions/primitiveConvTest.scala index a53210bb3..7ad2984a9 100644 --- a/server/src/test/scala/conversions/primitiveConvTest.scala +++ b/server/src/test/scala/conversions/primitiveConvTest.scala @@ -14,6 +14,13 @@ import com.google.common.primitives.{Bytes, Doubles, Ints, Shorts} object primitiveConvTest extends ZIOSpecDefault { private val floatPrecision = 0.000_000_1 + + /* Renders a byte array as hex so a failing table case is identifiable in the report. */ + private def hex(bytes: Array[Byte]): String = bytes.map(b => f"0x$b%02x").mkString("[", " ", "]") + + /* Keeps a label on a single line. */ + private def show(s: String): String = s.replace("\\", "\\\\").replace("\n", "\\n").replace("\t", "\\t") + def spec = suite(this.getClass.toString)( test("eqFloat") { case class TestCase(a: Double, b: Double, precision: Double, expected: Boolean) @@ -33,7 +40,11 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Float.NaN, Float.NaN, floatPrecision, false) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + val actual = primitiveConv.eqFloat(testCase.a, testCase.b, testCase.precision) + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: eqFloat(${testCase.a}, ${testCase.b}, ${testCase.precision}) = $actual, expected ${testCase.expected}" + }.reduce(_ && _) }, test("bytesToInt8") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Byte]) => Boolean) @@ -55,7 +66,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x01, 0x01).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToInt8(${hex(testCase.bytes)}) = ${primitiveConv.bytesToInt8(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToInt16") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Short]) => Boolean) @@ -78,7 +92,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x00, 0x01, 0x01).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToInt16(${hex(testCase.bytes)}) = ${primitiveConv.bytesToInt16(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToInt32") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Int]) => Boolean) @@ -104,7 +121,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToInt32(${hex(testCase.bytes)}) = ${primitiveConv.bytesToInt32(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToInt64") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Long]) => Boolean) @@ -139,7 +159,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToInt64(${hex(testCase.bytes)}) = ${primitiveConv.bytesToInt64(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToFloat32") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Float]) => Boolean) @@ -165,7 +188,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToFloat32(${hex(testCase.bytes)}) = ${primitiveConv.bytesToFloat32(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToFloat64") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Double]) => Boolean) @@ -200,7 +226,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToFloat64(${hex(testCase.bytes)}) = ${primitiveConv.bytesToFloat64(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToString") { case class TestCase(bytes: Array[Byte], expected: String) @@ -223,7 +252,11 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x71, 0x75, 0x22, 0x6f, 0x74, 0x65, 0x22, 0x73).map(_.toByte), """qu"ote"s""") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + val actual = show(primitiveConv.bytesToString(testCase.bytes)) + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToString(${hex(testCase.bytes)}) = $actual, expected ${show(testCase.expected)}" + }.reduce(_ && _) }, test("bytesToJsonString") { case class TestCase(bytes: Array[Byte], expected: String) @@ -244,7 +277,11 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x71, 0x75, 0x22, 0x6f, 0x74, 0x65, 0x22, 0x73).map(_.toByte), """"qu\"ote\"s"""") ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + val actual = show(primitiveConv.bytesToJsonString(testCase.bytes)) + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToJsonString(${hex(testCase.bytes)}) = $actual, expected ${show(testCase.expected)}" + }.reduce(_ && _) }, test("bytesToBoolean") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, Boolean]) => Boolean) @@ -261,7 +298,10 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x00, 0x01).map(_.toByte), _.isLeft) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToBoolean(${hex(testCase.bytes)}) = ${primitiveConv.bytesToBoolean(testCase.bytes)}" + }.reduce(_ && _) }, test("bytesToJson") { case class TestCase(bytes: Array[Byte], check: (result: Either[Throwable, String]) => Boolean) @@ -289,7 +329,11 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase("""{a:2,"b":{"c":3}}""".getBytes("UTF-8"), _.isLeft), ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + val actual = primitiveConv.bytesToJson(testCase.bytes) + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: bytesToJson(${show(primitiveConv.bytesToString(testCase.bytes))}) = $actual" + }.reduce(_ && _) }, test("leftPad") { case class TestCase(bytes: Array[Byte], size: Int, pad: Byte, expected: Array[Byte]) @@ -308,7 +352,11 @@ object primitiveConvTest extends ZIOSpecDefault { TestCase(Array(0x01).map(_.toByte), 1, 0, Array(0x01).map(_.toByte)) ) - assertTrue(testCases.forall(runTestCase)) + testCases.zipWithIndex.map { (testCase, idx) => + val actual = hex(primitiveConv.leftPad(testCase.bytes, testCase.size, testCase.pad)) + assertTrue(runTestCase(testCase)) ?? + s"case #$idx: leftPad(${hex(testCase.bytes)}, ${testCase.size}, ${testCase.pad}) = $actual, expected ${hex(testCase.expected)}" + }.reduce(_ && _) } ) } diff --git a/server/src/test/scala/library/libraryConcurrencyTest.scala b/server/src/test/scala/library/libraryConcurrencyTest.scala new file mode 100644 index 000000000..38b9cc2c4 --- /dev/null +++ b/server/src/test/scala/library/libraryConcurrencyTest.scala @@ -0,0 +1,89 @@ +package library + +import zio.* +import zio.test.* +import _root_.library.managed_items.{ManagedMarkdownDocument, ManagedMarkdownDocumentSpec} + +/** Concurrency contracts for `library/Library.scala`. + * + * `Library` is a per-instance singleton shared by every gRPC call (`LibraryServiceImpl.library`), so + * saveLibraryItem/deleteLibraryItem run concurrently on ONE object. Both mutators are + * "touch a file, then rescan the whole dir, then replace `db`": + * + * - two racing writers can interleave so an OLDER scan publishes LAST, dropping a just-written + * item from the snapshot even though its file is on disk (get/list stop seeing it until the + * next unrelated mutation); + * - deleteItem does exists-then-remove, so two racing deletes of the same id can both pass the + * exists check and both report OK. + * + * These tests race real threads (`ZIO.attemptBlocking` on the blocking pool) rather than asserting + * on structure, and repeat under `TestAspect.nonFlaky` because a single round can get lucky. + * Each test mints its own temp dir and its own Library, so parallel execution is safe. + */ +object libraryConcurrencyTest extends ZIOSpecDefault { + + private def tempDir(): os.Path = + os.temp.dir(prefix = "library-concurrency") + + private def tenantContext(tenant: String): ResourceMatcher = + ResourceMatcher(matcher = TenantMatcher(matcher = ExactTenantMatcher(tenant = tenant))) + + private def markdownItem(id: String): LibraryItem = + LibraryItem( + metadata = LibraryItemMetadata(updatedAt = "2026-07-25T00:00:00Z", availableForContexts = Vector(tenantContext("t1"))), + spec = ManagedMarkdownDocument( + metadata = ManagedItemMetadata( + `type` = ManagedItemType.MarkdownDocument, + id = id, + name = s"item-$id", + descriptionMarkdown = "" + ), + spec = ManagedMarkdownDocumentSpec(markdown = "hello") + ) + ) + + def spec = suite(this.getClass.toString)( + test("concurrent writes of distinct ids all survive in the in-memory snapshot") { + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + val ids = (0 until 12).map(i => f"item$i%02d").toVector + + for _ <- ZIO.foreachParDiscard(ids)(id => ZIO.attemptBlocking(library.writeItem(markdownItem(id)))) + yield + val missingOnDisk = ids.filterNot(id => os.exists(root / s"$id.binpb")) + val missingInDb = ids.filterNot(id => library.getItemById(id).isDefined) + assertTrue(missingOnDisk.isEmpty, missingInDb.isEmpty, library.size == ids.size) ?? + s"written to disk but lost from the snapshot: ${missingInDb.mkString(", ")}" + } @@ TestAspect.nonFlaky(25), + test("a concurrent write is not dropped by a concurrent delete's rescan") { + // The same lost-update, mixed: the deleter's scan may predate the writer's file. + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + library.writeItem(markdownItem("victim")) + + for _ <- ZIO.collectAllParDiscard(Seq( + ZIO.attemptBlocking(library.writeItem(markdownItem("newone"))), + ZIO.attemptBlocking(library.deleteItem("victim")) + )) + yield assertTrue( + library.getItemById("newone").isDefined, + library.getItemById("victim").isEmpty, + os.exists(root / "newone.binpb"), + library.size == 1 + ) + } @@ TestAspect.nonFlaky(25), + test("exactly one of several concurrent deletes of the same id succeeds") { + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + library.writeItem(markdownItem("dupdel")) + + for results <- ZIO.foreachPar(1 to 4)(_ => ZIO.attemptBlocking(library.deleteItem("dupdel")).either) + yield assertTrue( + results.count(_.isRight) == 1, + results.count(_.isLeft) == 3, + !os.exists(root / "dupdel.binpb"), + library.size == 0 + ) ?? s"delete attempts that reported success: ${results.count(_.isRight)} (expected exactly 1)" + } @@ TestAspect.nonFlaky(25) + ) +} diff --git a/server/src/test/scala/library/libraryScanTest.scala b/server/src/test/scala/library/libraryScanTest.scala new file mode 100644 index 000000000..4fcc46c67 --- /dev/null +++ b/server/src/test/scala/library/libraryScanTest.scala @@ -0,0 +1,373 @@ +package library + +import zio.test.* +import com.tools.teal.pulsar.ui.library.v1.library as pb +import com.tools.teal.pulsar.ui.library.v1.resource_matchers as pbm +import _root_.library.managed_items.{ManagedMarkdownDocument, ManagedMarkdownDocumentSpec} +import scala.util.Try +import ch.qos.logback.classic.{Level, Logger as LogbackLogger} +import ch.qos.logback.classic.spi.ILoggingEvent +import org.slf4j.LoggerFactory +import scala.jdk.CollectionConverters.* + +/** Unit coverage for the on-disk scan/refresh path in `library/Library.scala` - `scan`, `refreshDb` + * and `deleteItem`. `LibraryBugRegressionsTest` covers the write-side guards (id charset, empty + * contexts, empty search filter); the read side (what the scanner does with files it did NOT write) + * was untested. + * + * Every test mints its OWN `os.temp.dir` and its own `Library` instance, so the default parallel + * test execution is safe - there is no shared mutable state and no fixed path. + */ +object libraryScanTest extends ZIOSpecDefault { + + private def tempDir(): os.Path = + os.temp.dir(prefix = "library-scan") + + private def tenantContext(tenant: String): ResourceMatcher = + ResourceMatcher(matcher = TenantMatcher(matcher = ExactTenantMatcher(tenant = tenant))) + + private def markdownItem(id: String, contexts: Vector[ResourceMatcher] = Vector(tenantContext("t1"))): LibraryItem = + LibraryItem( + metadata = LibraryItemMetadata(updatedAt = "2026-07-25T00:00:00Z", availableForContexts = contexts), + spec = ManagedMarkdownDocument( + metadata = ManagedItemMetadata( + `type` = ManagedItemType.MarkdownDocument, + id = id, + name = s"item-$id", + descriptionMarkdown = "" + ), + spec = ManagedMarkdownDocumentSpec(markdown = "hello") + ) + ) + + private def itemBytes(item: LibraryItem): Array[Byte] = LibraryItem.toPb(item).toByteArray + + /** Capture the events emitted on the `library.Library` logger while `body` runs. The appender is + * attached before the scan and detached after, so it observes the scan's own logging. Other + * library suites log to the same logger under the default parallel execution, so the assertions + * below key on a file name UNIQUE to each test rather than on an event count - a foreign warning + * can never masquerade as the one under test. */ + // SLF4J hands a SubstituteLogger to callers that arrive while the backend is still initializing; + // under the default parallel suite execution our first `getLogger` can land in that window and a + // direct cast to the logback Logger throws ClassCastException. Re-fetch until the real logback + // binding is in place (initialization completes in milliseconds, so this resolves at once). + private def libraryLogbackLogger(): LogbackLogger = + var logger = LoggerFactory.getLogger("library.Library") + var attempts = 0 + while !logger.isInstanceOf[LogbackLogger] && attempts < 500 do + Thread.sleep(2) + logger = LoggerFactory.getLogger("library.Library") + attempts += 1 + logger match + case l: LogbackLogger => l + case other => throw new IllegalStateException(s"Expected a logback logger, got ${other.getClass.getName}") + + /** A THREAD-SAFE capture appender. Logback's own ListAppender backs onto a plain ArrayList, + * and the "library.Library" logger is shared: suites running in parallel append to it while + * this helper snapshots, which threw ConcurrentModificationException out of the FIXTURE and + * failed the test before its assertion ever ran - rarely, and only in full-suite runs. */ + private final class QueueAppender extends ch.qos.logback.core.AppenderBase[ILoggingEvent]: + val events = new java.util.concurrent.ConcurrentLinkedQueue[ILoggingEvent]() + override def append(event: ILoggingEvent): Unit = + events.add(event) + () + + private def withCapturedLibraryLogs[A](body: => A): (A, List[ILoggingEvent]) = + val logbackLogger = libraryLogbackLogger() + val appender = new QueueAppender() + appender.start() + logbackLogger.addAppender(appender) + try + val result = body + (result, appender.events.asScala.toList) + finally + logbackLogger.detachAppender(appender) + appender.stop() + + // Tag byte 0x0f = field 1 with wire type 7, which is not a valid protobuf wire type. + private val corruptBytes: Array[Byte] = Array[Byte](0x0f, 0x7f, 0x7f, 0x7f) + + def spec = suite(this.getClass.toString)( + test("a corrupt .binpb file is skipped without failing the scan") { + val root = tempDir() + os.write(root / "corrupt1.binpb", corruptBytes) + os.write(root / "valid1.binpb", itemBytes(markdownItem("valid1"))) + + val library = Library.createAndRefreshDb(root.toString) + + assertTrue( + // the fixture really is unparseable - otherwise this test would prove nothing + Try(pb.LibraryItem.parseFrom(corruptBytes)).isFailure, + // the corrupt file neither loads nor takes the whole scan down with it + library.getItemById("corrupt1").isEmpty, + library.getItemById("valid1").isDefined, + library.size == 1, + // the scanner is read-only: it does not quarantine or delete what it cannot parse + os.exists(root / "corrupt1.binpb") + ) + }, + test("a file whose embedded item id does not match its file name is not loaded") { + val bytes = itemBytes(markdownItem("aaaaaa")) + + val mismatchedRoot = tempDir() + os.write(mismatchedRoot / "bbbbbb.binpb", bytes) + val mismatched = Library.createAndRefreshDb(mismatchedRoot.toString) + + // control: the SAME bytes under the matching file name do load, so the rejection above is + // attributable to the name mismatch and not to bad content. + val matchingRoot = tempDir() + os.write(matchingRoot / "aaaaaa.binpb", bytes) + val matching = Library.createAndRefreshDb(matchingRoot.toString) + + assertTrue( + mismatched.size == 0, + mismatched.getItemById("aaaaaa").isEmpty, // not keyed by the embedded id + mismatched.getItemById("bbbbbb").isEmpty, // nor by the file name + os.exists(mismatchedRoot / "bbbbbb.binpb"), // left on disk untouched + matching.size == 1, + matching.getItemById("aaaaaa").isDefined + ) + }, + test("non-.binpb entries in the library directory are ignored") { + val root = tempDir() + os.write(root / "notes.txt", "not a library item") + os.write(root / "item.json", """{"metadata":{}}""") + os.write(root / "README", "no extension at all") + os.write(root / "valid1.binpb.bak", itemBytes(markdownItem("valid1"))) // real bytes, wrong ext + os.makeDir(root / "subdir.binpb") // right ext, but a directory - the os.isFile guard + os.write(root / "valid1.binpb", itemBytes(markdownItem("valid1"))) + + val library = Library.createAndRefreshDb(root.toString) + + assertTrue( + library.size == 1, + library.getItemById("valid1").isDefined, + library.getItemById("notes").isEmpty, + library.getItemById("item").isEmpty, + // the scan leaves foreign files alone + os.exists(root / "notes.txt"), + os.exists(root / "README"), + os.exists(root / "subdir.binpb") + ) + }, + test("a file whose name is not exactly `.binpb` is not loaded") { + // The file name is the ONLY handle the API has on an item: writeItem and deleteItem both + // derive `$itemId.binpb` from the id. The scan derived the id with + // `fileName.split('.').head`, so `aaaaaa.extra.binpb` was accepted as item `aaaaaa` - + // listed and gettable, but deleting it targets `aaaaaa.binpb` (NOT_FOUND) and saving it + // creates a SECOND file. Require the exact canonical name instead. + val root = tempDir() + os.write(root / "aaaaaa.extra.binpb", itemBytes(markdownItem("aaaaaa"))) + os.write(root / "bbbbbb.binpb", itemBytes(markdownItem("bbbbbb"))) // control: canonical name + + val library = Library.createAndRefreshDb(root.toString) + + assertTrue( + library.getItemById("aaaaaa").isEmpty, + library.size == 1, + library.getItemById("bbbbbb").isDefined, // the control really does load + // the scanner stays read-only about what it rejects + os.exists(root / "aaaaaa.extra.binpb") + ) + }, + test("a file whose id is outside the safe charset is not loaded") { + // `requireSafeItemId` guards writeItem/deleteItem but was never applied by the scan, so a + // file dropped into the library dir with e.g. `bad+id` loaded happily - and then every + // write/delete for that id was rejected with INVALID_ARGUMENT. An item the API cannot + // address must not be surfaced. + val root = tempDir() + os.write(root / "bad+id.binpb", itemBytes(markdownItem("bad+id"))) + os.write(root / "goodid.binpb", itemBytes(markdownItem("goodid"))) // control + + val library = Library.createAndRefreshDb(root.toString) + val deleteBad = Try(library.deleteItem("bad+id")) + + assertTrue( + library.getItemById("bad+id").isEmpty, + library.size == 1, + library.getItemById("goodid").isDefined, + // context: the write path really does refuse this id, which is why surfacing it is wrong + deleteBad.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + os.exists(root / "bad+id.binpb") + ) + }, + test("a file rejected for an unsafe id is logged as a warning, not dropped silently") { + // The rejection branch built a Left(...) that only refreshDb's Right-collector reads - + // nothing logged it, so an operator saw "Found N items" with no hint a file was skipped. + val root = tempDir() + // Names unique to this test so a parallel suite scanning its own unsafe-id file cannot + // supply the warning we assert on. + os.write(root / "unsafe+scanlog+id.binpb", itemBytes(markdownItem("unsafe+scanlog+id"))) + os.write(root / "goodscanlogid.binpb", itemBytes(markdownItem("goodscanlogid"))) // control + + val (library, logs) = withCapturedLibraryLogs(Library.createAndRefreshDb(root.toString)) + val warnings = logs.filter(_.getLevel == Level.WARN).map(_.getFormattedMessage) + + assertTrue( + // still excluded from the db ... + library.getItemById("unsafe+scanlog+id").isEmpty, + library.getItemById("goodscanlogid").isDefined, + // ... but no longer silently: a WARN names the offending file + warnings.exists(_.contains("unsafe+scanlog+id.binpb")) + ) + }, + test("a file rejected for a name/id mismatch is logged as a warning, not dropped silently") { + val root = tempDir() + os.write(root / "mismatchlogfile.binpb", itemBytes(markdownItem("mismatchlogid"))) + os.write(root / "goodscanlogid2.binpb", itemBytes(markdownItem("goodscanlogid2"))) // control + + val (library, logs) = withCapturedLibraryLogs(Library.createAndRefreshDb(root.toString)) + val warnings = logs.filter(_.getLevel == Level.WARN).map(_.getFormattedMessage) + + assertTrue( + library.getItemById("mismatchlogid").isEmpty, + library.getItemById("mismatchlogfile").isEmpty, + library.getItemById("goodscanlogid2").isDefined, + warnings.exists(_.contains("mismatchlogfile.binpb")) + ) + }, + test("every item the scan surfaces is addressable by its id") { + // The invariant behind the two tests above, stated directly: whatever getItemById returns + // must be deletable under that same id. + val root = tempDir() + os.write(root / "cccccc.extra.binpb", itemBytes(markdownItem("cccccc"))) + os.write(root / "bad+id.binpb", itemBytes(markdownItem("bad+id"))) + os.write(root / "dddddd.binpb", itemBytes(markdownItem("dddddd"))) + + val library = Library.createAndRefreshDb(root.toString) + val surfaced = List("cccccc", "bad+id", "dddddd").filter(library.getItemById(_).isDefined) + val notDeletable = surfaced.filterNot(id => Try(library.deleteItem(id)).isSuccess) + + assertTrue(surfaced == List("dddddd"), notDeletable.isEmpty) + }, + test("a stored item carrying the unimplemented namespace_regex still loads - the field is dropped with a warning") { + // REGRESSION - `allNamespaceMatcherFromPb` REFUSES a set `namespace_regex`, which is + // right for REQUEST data (accepting it would silently WIDEN the caller's scope). But + // the same conversion runs on every stored item during scan(), where refusing made the + // whole ITEM fail to parse: it silently vanished from the library with only a server + // WARN and no way to repair it through the product. A stored item is not a request - + // the scan must drop the FIELD (never applied by any version that wrote it) and keep + // the ITEM, naming the file and the ignored pattern. + val tenantPb = pbm.TenantMatcher(matcher = pbm.TenantMatcher.Matcher.Exact(pbm.ExactTenantMatcher(tenant = "t1"))) + def allNsPb(regex: String) = pbm.NamespaceMatcher(matcher = + pbm.NamespaceMatcher.Matcher.All(pbm.AllNamespaceMatcher(tenant = Some(tenantPb), namespaceRegex = regex)) + ) + // the field can sit directly under a namespace matcher or nested under a topic matcher + val directWithRegex = pbm.ResourceMatcher(matcher = pbm.ResourceMatcher.Matcher.Namespace(allNsPb("audit-.*"))) + val nestedWithRegex = pbm.ResourceMatcher(matcher = + pbm.ResourceMatcher.Matcher.Topic( + pbm.TopicMatcher(matcher = pbm.TopicMatcher.Matcher.All(pbm.AllTopicMatcher(namespace = Some(allNsPb("billing-.*"))))) + ) + ) + + // what the item must load AS: the same matchers with the regex field dropped + val tenantModel = TenantMatcher(matcher = ExactTenantMatcher(tenant = "t1")) + val expected = markdownItem( + "nsregexitem1", + contexts = Vector( + ResourceMatcher(matcher = NamespaceMatcher(matcher = AllNamespaceMatcher(tenant = tenantModel))), + ResourceMatcher(matcher = + TopicMatcher(matcher = AllTopicMatcher(namespace = NamespaceMatcher(matcher = AllNamespaceMatcher(tenant = tenantModel)))) + ) + ) + ) + val cleanPb = LibraryItem.toPb(expected) + val poisonedPb = cleanPb.copy(metadata = cleanPb.metadata.map(_.copy(availableForContexts = Seq(directWithRegex, nestedWithRegex)))) + + val root = tempDir() + os.write(root / "nsregexitem1.binpb", poisonedPb.toByteArray) + os.write(root / "nsregexcontrol1.binpb", itemBytes(markdownItem("nsregexcontrol1"))) // control + + val (library, logs) = withCapturedLibraryLogs(Library.createAndRefreshDb(root.toString)) + val warnings = logs.filter(_.getLevel == Level.WARN).map(_.getFormattedMessage) + + assertTrue( + // the ITEM survives, with the FIELD dropped in both matcher positions + library.getItemById("nsregexitem1").contains(expected), + library.getItemById("nsregexcontrol1").isDefined, + library.size == 2, + // and not silently: a WARN names the file and each ignored pattern + warnings.exists(m => m.contains("nsregexitem1") && m.contains("audit-.*")), + warnings.exists(m => m.contains("nsregexitem1") && m.contains("billing-.*")), + // the WRITE path stays strict: the same bytes are still refused as request data + // (saveLibraryItem converts the request with this exact call before writing) + Try(LibraryItem.fromPb(poisonedPb)).failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) ?? s"warnings=${warnings.mkString("|")}" + }, + test("mutations mark their lock-held directory rescan as blocking for the execution context") { + // REGRESSION - writeItem/deleteItem do O(N) file I/O (a rescan of the whole library + // dir) inside the process-global mutation lock, and LibraryServiceImpl binds on + // ExecutionContext.global - a bounded ForkJoinPool that only spawns compensation + // threads for blocking it is TOLD about via scala.concurrent.blocking{}. Unmarked, a + // slow disk plus a contended lock starves unrelated compute tasks pool-wide. + // BlockContext is the seam blocking{} reports to, so a recording context is the direct + // oracle: at least one blocking-marked region must cover each mutation. + import scala.concurrent.{BlockContext, CanAwait} + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + val blockOnCalls = new java.util.concurrent.atomic.AtomicInteger(0) + val recording = new BlockContext: + override def blockOn[T](thunk: => T)(implicit permission: CanAwait): T = + blockOnCalls.incrementAndGet() + thunk + + BlockContext.withBlockContext(recording)(library.writeItem(markdownItem("blockmark1"))) + val afterWrite = blockOnCalls.get() + BlockContext.withBlockContext(recording)(library.deleteItem("blockmark1")) + val afterDelete = blockOnCalls.get() + + assertTrue( + afterWrite >= 1, // the write (including its rescan) ran inside a blocking-marked region + afterDelete > afterWrite, // and so did the delete + // the marker changed reporting, not behavior + library.getItemById("blockmark1").isEmpty, + library.size == 0 + ) ?? s"afterWrite=$afterWrite afterDelete=$afterDelete" + }, + test("deleteItem removes the file and drops the item from the db") { + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + library.writeItem(markdownItem("keepme")) + library.writeItem(markdownItem("dropme")) + + val sizeBefore = library.size + library.deleteItem("dropme") + + assertTrue( + sizeBefore == 2, + !os.exists(root / "dropme.binpb"), + library.getItemById("dropme").isEmpty, + // the delete is surgical - the sibling item survives + os.exists(root / "keepme.binpb"), + library.getItemById("keepme").isDefined, + library.size == 1 + ) + }, + test("deleteItem on an unknown id reports not-found instead of succeeding silently") { + // REGRESSION (fixed 2026-07-25) - Library.scala used to call `os.remove(filePath)` alone, which in os-lib 0.9.3 is + // `Files.deleteIfExists` (returns false, throws nothing). deleteItem therefore returns + // normally for an id that never existed, and LibraryServiceImpl.deleteLibraryItem reports + // Code.OK - indistinguishable from a real delete. Its sibling getLibraryItem already + // returns NOT_FOUND for the same id, so the API is internally inconsistent and a UI + // "deleted" confirmation is unearned. Expected: deleteItem signals the miss (and the + // service maps it to NOT_FOUND); IllegalArgumentException is deliberately NOT the right + // answer here since that is already the INVALID_ARGUMENT channel for a malformed id. + val root = tempDir() + val library = Library.createAndRefreshDb(root.toString) + library.writeItem(markdownItem("present1")) + + val deleteMissing = Try(library.deleteItem("missing1")) + + assertTrue( + // context: the id is genuinely absent, and a malformed id DOES still fail loudly + library.getItemById("missing1").isEmpty, + Try(library.deleteItem("../../evil")).isFailure, + // nothing collateral happened - the real item is untouched + library.getItemById("present1").isDefined, + library.size == 1, + deleteMissing.isFailure, + !deleteMissing.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + } + ) +} diff --git a/server/src/test/scala/library/libraryServiceDeleteTest.scala b/server/src/test/scala/library/libraryServiceDeleteTest.scala new file mode 100644 index 000000000..6124a4078 --- /dev/null +++ b/server/src/test/scala/library/libraryServiceDeleteTest.scala @@ -0,0 +1,120 @@ +package library + +import zio.* +import zio.test.* + +import com.google.rpc.code.Code +import com.tools.teal.pulsar.ui.library.v1.library.{DeleteLibraryItemRequest, GetLibraryItemRequest} +import _root_.library.managed_items.{ManagedMarkdownDocument, ManagedMarkdownDocumentSpec} + +import scala.concurrent.duration.{Duration, SECONDS} +import scala.concurrent.{Await, Future} + +/** `LibraryServiceImpl.deleteLibraryItem` - the gRPC STATUS the UI is handed, not the store beneath + * it. + * + * `libraryScanTest` pins `Library.deleteItem`: it throws `NoSuchElementException` for an id with no + * file and `IllegalArgumentException` for an id outside the safe charset. Nothing pinned what the + * service does with those two exceptions, and they leave by different `catch` arms of the same + * `try`. Adding a `case e: Exception` arm above them, or reordering them, silently collapses both + * into INTERNAL - the store stays correct while every caller starts seeing a server error, and no + * test notices. `getLibraryItem` already answers NOT_FOUND for a missing id, so a delete that + * answered INTERNAL (or, before 2026-07-25, OK) for the same id also makes the API inconsistent + * with itself. + * + * Each test mints its own `os.temp.dir` and its own service, so parallel execution is safe. + */ +object libraryServiceDeleteTest extends ZIOSpecDefault: + + private def markdownItem(id: String): LibraryItem = + LibraryItem( + metadata = LibraryItemMetadata( + updatedAt = "2026-07-26T00:00:00Z", + availableForContexts = Vector(ResourceMatcher(matcher = TenantMatcher(matcher = ExactTenantMatcher(tenant = "t1")))) + ), + spec = ManagedMarkdownDocument( + metadata = ManagedItemMetadata( + `type` = ManagedItemType.MarkdownDocument, + id = id, + name = s"item-$id", + descriptionMarkdown = "" + ), + spec = ManagedMarkdownDocumentSpec(markdown = "hello") + ) + ) + + /** A service over a throwaway library directory. The process-wide `libraryRoot` is fixed for the + * JVM, so the store is passed in - the same device `PulsarAuthRoutes.routesWith` uses. */ + private def serviceOver(root: os.Path): LibraryServiceImpl = + LibraryServiceImpl(Library.createAndRefreshDb(root.toString)) + + private def await[A](future: Future[A]): A = Await.result(future, Duration(30, SECONDS)) + + def spec = suite(this.getClass.toString)( + test("deleting an id that has no file answers NOT_FOUND, not OK and not INTERNAL") { + // REGRESSION - `Library.deleteItem` used `os.remove`, i.e. `Files.deleteIfExists`, which + // returns false rather than throwing, so deleting an id that never existed reported + // Code.OK: the UI showed a "deleted" confirmation for something it had not deleted. + val root = os.temp.dir(prefix = "library-service-delete") + val service = serviceOver(root) + await(service.saveLibraryItem(pbSave(markdownItem("present1")))) + + val missing = await(service.deleteLibraryItem(DeleteLibraryItemRequest(id = "missing1"))) + val present = await(service.getLibraryItem(GetLibraryItemRequest(id = "present1"))) + + assertTrue( + missing.getStatus.code == Code.NOT_FOUND.value, + // ... and it is the SAME verdict its sibling read gives for the same id + await(service.getLibraryItem(GetLibraryItemRequest(id = "missing1"))).getStatus.code == Code.NOT_FOUND.value, + // nothing collateral happened + present.getStatus.code == Code.OK.value, + os.exists(root / "present1.binpb") + ) ?? s"missing=${missing.getStatus} present=${present.getStatus}" + }, + test("deleting a malformed id answers INVALID_ARGUMENT, which is not the missing-item verdict") { + // Item ids become file names, so ids outside the safe charset are refused before any + // filesystem call - a caller error, distinct from "there is no such item". Collapsing + // the two would tell the UI to retry a path-traversal attempt as if it were a typo. + val root = os.temp.dir(prefix = "library-service-delete") + val service = serviceOver(root) + + val traversal = await(service.deleteLibraryItem(DeleteLibraryItemRequest(id = "../../evil"))) + val badCharset = await(service.deleteLibraryItem(DeleteLibraryItemRequest(id = "bad+id"))) + + assertTrue( + traversal.getStatus.code == Code.INVALID_ARGUMENT.value, + badCharset.getStatus.code == Code.INVALID_ARGUMENT.value, + // nothing escaped the library root + !os.exists(root / os.up / "evil.binpb"), + !os.exists(root / os.up / os.up / "evil.binpb") + ) ?? s"traversal=${traversal.getStatus} badCharset=${badCharset.getStatus}" + }, + test("deleting a real item answers OK and the item is gone from disk and from reads") { + // The control: the two refusals above prove nothing unless a genuine delete still works, + // and OK has to mean the file is actually gone - not merely that no exception escaped. + val root = os.temp.dir(prefix = "library-service-delete") + val service = serviceOver(root) + await(service.saveLibraryItem(pbSave(markdownItem("dropme")))) + await(service.saveLibraryItem(pbSave(markdownItem("keepme")))) + + val readBefore = await(service.getLibraryItem(GetLibraryItemRequest(id = "dropme"))) + val deleted = await(service.deleteLibraryItem(DeleteLibraryItemRequest(id = "dropme"))) + val readAfter = await(service.getLibraryItem(GetLibraryItemRequest(id = "dropme"))) + val deletedAgain = await(service.deleteLibraryItem(DeleteLibraryItemRequest(id = "dropme"))) + + assertTrue( + readBefore.getStatus.code == Code.OK.value, + deleted.getStatus.code == Code.OK.value, + !os.exists(root / "dropme.binpb"), + readAfter.getStatus.code == Code.NOT_FOUND.value, + // a repeated delete is now a miss, not a second success + deletedAgain.getStatus.code == Code.NOT_FOUND.value, + // the delete is surgical + await(service.getLibraryItem(GetLibraryItemRequest(id = "keepme"))).getStatus.code == Code.OK.value, + os.exists(root / "keepme.binpb") + ) ?? s"deleted=${deleted.getStatus} readAfter=${readAfter.getStatus} deletedAgain=${deletedAgain.getStatus}" + } + ) + + private def pbSave(item: LibraryItem) = + com.tools.teal.pulsar.ui.library.v1.library.SaveLibraryItemRequest(item = Some(LibraryItem.toPb(item))) diff --git a/server/src/test/scala/library/managedItemsConversionsTest.scala b/server/src/test/scala/library/managedItemsConversionsTest.scala new file mode 100644 index 000000000..81f107c9a --- /dev/null +++ b/server/src/test/scala/library/managedItemsConversionsTest.scala @@ -0,0 +1,94 @@ +package library.managed_items + +import zio.test.* +import com.tools.teal.pulsar.ui.library.v1.managed_items as pb +import _root_.consumer.start_from.DateTimeUnit +import library.{ManagedItemMetadata, ManagedItemType} +import scala.util.Try + +/** Conversion-layer coverage for `library/managed_items/`. + * + * - Defect 4 (trust boundary): `ManagedRelativeDateTimeSpec` carries a `Long` because + * managed_items.proto stores `value` as int64, but the api form (consumer.proto + * `RelativeDateTime`) is int32. A library file written by a non-Dekaf client can persist a + * value that is negative or out of int32 range; `fromPb` is the server-side ingestion boundary + * (reached by both SaveLibraryItem and the on-disk scan), so it must reject such a value loudly + * rather than let a later narrowing to int32 silently truncate it. + * - Defect 3 (dead union member): `ManagedConsumerSessionStartFromValOrRef` was removed from the + * `ManagedConsumerSessionStartFromSpec.startFrom` union (it is absent from the proto oneof and + * no conversion ever produced it). The relative-date-time member - structurally adjacent to the + * one removed - must still round-trip through both directions. + */ +object managedItemsConversionsTest extends ZIOSpecDefault { + + private def relativeDateTimeSpecPb(value: Long): pb.ManagedRelativeDateTimeSpec = + pb.ManagedRelativeDateTimeSpec( + value = value, + unit = DateTimeUnit.toPb(DateTimeUnit.Hour), + isRoundedToUnitStart = false + ) + + def spec = suite(this.getClass.toString)( + test("ManagedRelativeDateTimeSpec.fromPb accepts an in-range non-negative value") { + val small = ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(5L)) + val zero = ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(0L)) + // Int.MaxValue is the largest value the api int32 can hold - it must be accepted. + val maxInt = ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(Int.MaxValue.toLong)) + assertTrue( + small.value == 5L, + zero.value == 0L, + maxInt.value == Int.MaxValue.toLong + ) + }, + test("ManagedRelativeDateTimeSpec.fromPb rejects a value above int32 range") { + // One past Int.MaxValue: fits int64 on disk, cannot fit the api int32 without truncation. + val tooBig = Try(ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(Int.MaxValue.toLong + 1L))) + val wayBig = Try(ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(Long.MaxValue))) + assertTrue( + tooBig.isFailure, + tooBig.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + wayBig.isFailure, + wayBig.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + }, + test("ManagedRelativeDateTimeSpec.fromPb rejects a negative value") { + val negOne = Try(ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(-1L))) + val negBig = Try(ManagedRelativeDateTimeSpec.fromPb(relativeDateTimeSpecPb(Long.MinValue))) + assertTrue( + negOne.isFailure, + negOne.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + negBig.isFailure, + negBig.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + }, + test("a relative-date-time start-from still round-trips after the dead union member is removed") { + val original = ManagedConsumerSessionStartFromSpec( + startFrom = ManagedRelativeDateTimeValOrRef( + value = Some( + ManagedRelativeDateTime( + metadata = ManagedItemMetadata( + `type` = ManagedItemType.RelativeDateTime, + id = "rdt-roundtrip", + name = "rdt-roundtrip-name", + descriptionMarkdown = "" + ), + spec = ManagedRelativeDateTimeSpec(value = 7L, unit = DateTimeUnit.Day, isRoundedToUnitStart = true) + ) + ), + reference = None + ) + ) + + val roundTripped = ManagedConsumerSessionStartFromSpec.fromPb(ManagedConsumerSessionStartFromSpec.toPb(original)) + + val recoveredValue = roundTripped.startFrom match + case v: ManagedRelativeDateTimeValOrRef => v.value.map(_.spec.value) + case _ => None + + assertTrue( + roundTripped.startFrom.isInstanceOf[ManagedRelativeDateTimeValOrRef], + recoveredValue.contains(7L) + ) + } + ) +} diff --git a/server/src/test/scala/library/managed_items/managedConsumerSessionDeliveryOrderTest.scala b/server/src/test/scala/library/managed_items/managedConsumerSessionDeliveryOrderTest.scala new file mode 100644 index 000000000..8e44d5866 --- /dev/null +++ b/server/src/test/scala/library/managed_items/managedConsumerSessionDeliveryOrderTest.scala @@ -0,0 +1,153 @@ +package library.managed_items + +import com.tools.teal.pulsar.ui.api.v1.consumer as apiPb +import com.tools.teal.pulsar.ui.library.v1.managed_items as pb +import zio.test.* + +/** The managed-library protobuf is a second compatibility boundary in addition to the runtime + * session request. Proto3 represents an absent enum as zero, so both old binary items and old JSON + * items must become the explicit current default when they cross this boundary. + * + * That default is GUARANTEED. Owner decision (2026-08-11, direct instruction) - the third move + * of this default; the plan file's decision log is the record. A saved session that never named + * a delivery order replays recorded history exactly and auto-pauses when caught up. Explicit + * Best effort and explicit Fastest are choices and are preserved. */ +object managedConsumerSessionDeliveryOrderTest extends ZIOSpecDefault: + + private val ref = "delivery-order-fixture" + + private def specWith(order: Option[apiPb.MessageDeliveryOrder]): ManagedConsumerSessionConfigSpec = + ManagedConsumerSessionConfigSpec( + startFrom = ManagedConsumerSessionStartFromValOrRef(value = None, reference = Some(ref)), + targets = Vector.empty, + messageFilterChain = ManagedMessageFilterChainValOrRef(value = None, reference = Some(ref)), + pauseTriggerChain = ManagedConsumerSessionPauseTriggerChainValOrRef(value = None, reference = Some(ref)), + coloringRuleChain = ManagedColoringRuleChainValOrRef(value = None, reference = Some(ref)), + valueProjectionList = ManagedValueProjectionListValOrRef(value = None, reference = Some(ref)), + numDisplayItems = None, + messageDeliveryOrder = order, + deliveryOrderKey = None + ) + + /** Which top-level fields the encoded item actually carries. A test that only sets an enum to + * zero in memory proves nothing about a file written before the field existed; this reads the + * tags back off the bytes, so "the field is absent" is a statement about the wire. */ + private def wireFieldNumbers(bytes: Array[Byte]): Set[Int] = + val in = com.google.protobuf.CodedInputStream.newInstance(bytes) + val numbers = Set.newBuilder[Int] + var tag = in.readTag() + while tag != 0 do + // A protobuf tag is (fieldNumber << 3) | wireType. + numbers += (tag >>> 3) + in.skipField(tag) + tag = in.readTag() + numbers.result() + + /** Field 8 in `ManagedConsumerSessionConfigSpec`. */ + private val deliveryOrderFieldNumber = 8 + + def spec = suite(this.getClass.toString)( + test("an UNSPECIFIED saved value loads as Guaranteed, the default that also covers legacy items") { + // Pre-existing saved sessions carry no delivery-order field at all. Zero is proto3 + // absence, and absence resolves to the product default - Guaranteed - never to an + // ordering the saved session never chose. The base spec names explicit BEST EFFORT so + // the expected result can only come from the UNSPECIFIED mapping, not the fixture. + val otherwiseValid = ManagedConsumerSessionConfigSpec.toPb( + specWith(Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME)) + ) + val oldWireItem = otherwiseValid.copy( + messageDeliveryOrder = apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED + ) + + assertTrue( + ManagedConsumerSessionConfigSpec.fromPb(oldWireItem).messageDeliveryOrder.contains( + apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ) + ) + }, + test("an item SERIALIZED before the field existed carries no delivery-order field and still loads as Guaranteed, the default") { + // The compatibility case that matters: BYTES on disk, written by a build that had no + // field 8 at all - not an in-memory model with the enum set to zero. + val preFieldBytes = ManagedConsumerSessionConfigSpec + .toPb(specWith(Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME))) + .copy(messageDeliveryOrder = apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED) + .toByteArray + + val parsed = pb.ManagedConsumerSessionConfigSpec.parseFrom(preFieldBytes) + val loaded = ManagedConsumerSessionConfigSpec.fromPb(parsed) + + assertTrue( + !wireFieldNumbers(preFieldBytes).contains(deliveryOrderFieldNumber), + parsed.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED, + loaded.messageDeliveryOrder.contains(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED) + ) ?? s"wireFields=${wireFieldNumbers(preFieldBytes)} loaded=${loaded.messageDeliveryOrder}" + }, + test("a spec built without naming a delivery order carries Guaranteed, not an absent value") { + // The case-class default itself: every in-process construction that predates the field + // (and every future one that ignores it) has to land on the same product default. + val builtWithoutTheField = ManagedConsumerSessionConfigSpec( + startFrom = ManagedConsumerSessionStartFromValOrRef(value = None, reference = Some(ref)), + targets = Vector.empty, + messageFilterChain = ManagedMessageFilterChainValOrRef(value = None, reference = Some(ref)), + pauseTriggerChain = ManagedConsumerSessionPauseTriggerChainValOrRef(value = None, reference = Some(ref)), + coloringRuleChain = ManagedColoringRuleChainValOrRef(value = None, reference = Some(ref)), + valueProjectionList = ManagedValueProjectionListValOrRef(value = None, reference = Some(ref)), + numDisplayItems = None + ) + + assertTrue( + builtWithoutTheField.messageDeliveryOrder.contains( + apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ) + ) ?? s"messageDeliveryOrder=${builtWithoutTheField.messageDeliveryOrder}" + }, + test("a missing older JSON value is written back as explicit Guaranteed, the default") { + val saved = ManagedConsumerSessionConfigSpec.toPb(specWith(None)) + + assertTrue( + saved.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ) + }, + test("an unrecognized future mode falls back to the Guaranteed default, never to Fastest") { + val saved = ManagedConsumerSessionConfigSpec.toPb( + specWith(Some(apiPb.MessageDeliveryOrder.Unrecognized(99))) + ) + + assertTrue( + saved.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ) + }, + test("an explicit Best effort choice round-trips and is never replaced by the Guaranteed default") { + val saved = ManagedConsumerSessionConfigSpec.toPb( + specWith(Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME)) + ) + val loaded = ManagedConsumerSessionConfigSpec.fromPb(saved) + + assertTrue( + saved.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME, + loaded.messageDeliveryOrder.contains(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME) + ) + }, + test("an explicit Guaranteed choice round-trips - it now coincides with the default, and stays explicit") { + val saved = ManagedConsumerSessionConfigSpec.toPb( + specWith(Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED)) + ) + val loaded = ManagedConsumerSessionConfigSpec.fromPb(saved) + + assertTrue( + saved.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED, + loaded.messageDeliveryOrder.contains(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED) + ) + }, + test("an explicit Fastest choice is preserved, not replaced by the Best effort default") { + val saved = ManagedConsumerSessionConfigSpec.toPb( + specWith(Some(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED)) + ) + val loaded = ManagedConsumerSessionConfigSpec.fromPb(saved) + + assertTrue( + saved.messageDeliveryOrder == apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED, + loaded.messageDeliveryOrder.contains(apiPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED) + ) + } + ) diff --git a/server/src/test/scala/library/resourceMatchersConversionsTest.scala b/server/src/test/scala/library/resourceMatchersConversionsTest.scala new file mode 100644 index 000000000..4a99a985f --- /dev/null +++ b/server/src/test/scala/library/resourceMatchersConversionsTest.scala @@ -0,0 +1,218 @@ +package library + +import zio.test.* +import com.tools.teal.pulsar.ui.library.v1.resource_matchers as pb +import scala.util.Try + +/** Unit coverage for `library/resourceMatchersConversions.scala` - the proto <-> model boundary. + * + * `resourceMatchersTest.scala` only exercises the `.test()` predicates in `resourceMatchers.scala`; + * the conversions were untested, even though `LibraryServiceImpl.listLibraryItems` / + * `getLibraryItemsCount` call `resourceMatcherFromPb` directly on UNTRUSTED request data, and + * `LibraryItemMetadata.fromPb` calls it on every matcher of every item read off disk. + * + * The proto has no regex matcher variant - the shape vocabulary is exactly + * instance | tenant{exact,all} | namespace{exact,all} | topic{exact,all}, enumerated below. + * (`AllNamespaceMatcher.namespace_regex` is a declared proto FIELD with no model counterpart - + * setting it is rejected rather than ignored; pinned by its own two tests. The disk-read path is + * the one exception: `Library.scan` strips a set regex before converting, so a stored ITEM keeps + * loading - pinned in libraryScanTest.) + * + * No shared mutable state, so the default parallel test execution is safe here. + */ +object resourceMatchersConversionsTest extends ZIOSpecDefault { + + private val tenantExact = TenantMatcher(matcher = ExactTenantMatcher(tenant = "tenant-a")) + private val tenantAll = TenantMatcher(matcher = AllTenantMatcher()) + + private val nsExactUnderExactTenant = + NamespaceMatcher(matcher = ExactNamespaceMatcher(tenant = tenantExact, namespace = "ns-a")) + private val nsExactUnderAllTenant = + NamespaceMatcher(matcher = ExactNamespaceMatcher(tenant = tenantAll, namespace = "ns-a")) + private val nsAllUnderExactTenant = NamespaceMatcher(matcher = AllNamespaceMatcher(tenant = tenantExact)) + private val nsAllUnderAllTenant = NamespaceMatcher(matcher = AllNamespaceMatcher(tenant = tenantAll)) + + /** Every ResourceMatcher shape the model can express, including each nested tenant/namespace variant. */ + private val allShapes: List[ResourceMatcher] = List( + ResourceMatcher(matcher = InstanceMatcher()), + ResourceMatcher(matcher = tenantExact), + ResourceMatcher(matcher = tenantAll), + ResourceMatcher(matcher = nsExactUnderExactTenant), + ResourceMatcher(matcher = nsExactUnderAllTenant), + ResourceMatcher(matcher = nsAllUnderExactTenant), + ResourceMatcher(matcher = nsAllUnderAllTenant), + ResourceMatcher(matcher = TopicMatcher(matcher = ExactTopicMatcher(namespace = nsExactUnderExactTenant, topic = "topic-a"))), + ResourceMatcher(matcher = TopicMatcher(matcher = ExactTopicMatcher(namespace = nsAllUnderAllTenant, topic = "topic-a"))), + ResourceMatcher(matcher = TopicMatcher(matcher = AllTopicMatcher(namespace = nsExactUnderExactTenant))), + ResourceMatcher(matcher = TopicMatcher(matcher = AllTopicMatcher(namespace = nsAllUnderAllTenant))) + ) + + /** The deepest shape: topic -> namespace -> tenant, every level an `exact`. */ + private val deepestShape = + ResourceMatcher(matcher = TopicMatcher(matcher = ExactTopicMatcher(namespace = nsExactUnderExactTenant, topic = "topic-a"))) + + def spec = suite(this.getClass.toString)( + test("every matcher shape survives a model -> pb -> model round-trip") { + val broken = allShapes.filter(shape => resourceMatcherFromPb(resourceMatcherToPb(shape)) != shape) + assertTrue( + allShapes.size == 11, // instance + 2 tenant + 4 namespace + 4 topic + allShapes.distinct.size == 11, // the shapes really are distinct - equality discriminates + broken.isEmpty + ) + }, + test("toPb populates the nested pb message fields instead of leaving them unset") { + // A dropped nested field would round-trip "fine" only because fromPb would blow up on it - + // assert the pb intermediate directly so the two directions cannot hide each other's bug. + val encoded = resourceMatcherToPb(deepestShape) + val exactTopic = encoded.matcher.topic.flatMap(_.matcher.exact) + val namespace = exactTopic.flatMap(_.namespace) + val exactNamespace = namespace.flatMap(_.matcher.exact) + val tenant = exactNamespace.flatMap(_.tenant) + + assertTrue( + exactTopic.map(_.topic).contains("topic-a"), + namespace.isDefined, + exactNamespace.map(_.namespace).contains("ns-a"), + tenant.isDefined, + tenant.flatMap(_.matcher.exact).map(_.tenant).contains("tenant-a") + ) + }, + test("a wire-encoded matcher decodes back into the same model value") { + // The on-disk library format and the gRPC surface are both real protobuf bytes, so exercise + // the encode/parse legs too - not just the in-memory case-class hop. + val bytes = resourceMatcherToPb(deepestShape).toByteArray + val decoded = resourceMatcherFromPb(pb.ResourceMatcher.parseFrom(bytes)) + assertTrue( + bytes.nonEmpty, + decoded == deepestShape, + // a different-tenant shape must NOT decode equal - guards against a degenerate compare + decoded != ResourceMatcher(matcher = + TopicMatcher(matcher = + ExactTopicMatcher( + namespace = NamespaceMatcher(matcher = + ExactNamespaceMatcher(tenant = TenantMatcher(matcher = ExactTenantMatcher("tenant-b")), namespace = "ns-a") + ), + topic = "topic-a" + ) + ) + ) + ) + }, + test("an unset oneof is rejected with IllegalArgumentException") { + // The oneof discriminators ARE guarded (`case _ => throw new IllegalArgumentException`), + // which LibraryServiceImpl maps to INVALID_ARGUMENT. This is the contrast case for the + // unset-nested-message tests below. + val resource = Try(resourceMatcherFromPb(pb.ResourceMatcher())) + val tenant = Try(tenantMatcherFromPb(pb.TenantMatcher())) + val namespace = Try(namespaceMatcherFromPb(pb.NamespaceMatcher())) + val topic = Try(topicMatcherFromPb(pb.TopicMatcher())) + + assertTrue( + resource.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + tenant.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + namespace.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + topic.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + }, + test("an unset nested message field is rejected cleanly, not with NoSuchElementException") { + // REGRESSION (fixed 2026-07-25) - resourceMatchersConversions.scala used to call `.get` on the + // Option-typed nested proto field. A default-constructed proto (an older client, or any + // request that simply omits the field) therefore raises NoSuchElementException, which + // LibraryServiceImpl's `case e: Exception` maps to INTERNAL - a 500 for what is plainly + // caller-supplied malformed input. It should be an IllegalArgumentException like every + // other malformed-input path in this same file (-> INVALID_ARGUMENT). + val exactNamespace = Try(exactNamespaceMatcherFromPb(pb.ExactNamespaceMatcher(namespace = "ns-a"))) + val allNamespace = Try(allNamespaceMatcherFromPb(pb.AllNamespaceMatcher())) + val exactTopic = Try(exactTopicMatcherFromPb(pb.ExactTopicMatcher(topic = "topic-a"))) + val allTopic = Try(allTopicMatcherFromPb(pb.AllTopicMatcher())) + + assertTrue( + // all four do fail - the defect is the TYPE of failure + exactNamespace.isFailure, + allNamespace.isFailure, + exactTopic.isFailure, + allTopic.isFailure, + exactNamespace.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + allNamespace.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + exactTopic.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + allTopic.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + }, + test("a request whose nested matcher field is unset is rejected cleanly at the public entry point") { + // REGRESSION (fixed 2026-07-25) - same root cause as above, reached the way a real client does: + // ListLibraryItemsRequest.contexts -> resourceMatcherFromPb. A client that sends + // {namespace: {exact: {namespace: "ns-a"}}} (tenant omitted) gets INTERNAL instead of + // INVALID_ARGUMENT. Expected: IllegalArgumentException. + val namespaceWithoutTenant = pb.ResourceMatcher(matcher = + pb.ResourceMatcher.Matcher.Namespace( + pb.NamespaceMatcher(matcher = pb.NamespaceMatcher.Matcher.Exact(pb.ExactNamespaceMatcher(namespace = "ns-a"))) + ) + ) + val topicWithoutNamespace = pb.ResourceMatcher(matcher = + pb.ResourceMatcher.Matcher.Topic( + pb.TopicMatcher(matcher = pb.TopicMatcher.Matcher.Exact(pb.ExactTopicMatcher(topic = "topic-a"))) + ) + ) + val namespaceResult = Try(resourceMatcherFromPb(namespaceWithoutTenant)) + val topicResult = Try(resourceMatcherFromPb(topicWithoutNamespace)) + + assertTrue( + namespaceResult.isFailure, + topicResult.isFailure, + namespaceResult.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + topicResult.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]) + ) + }, + test("a set AllNamespaceMatcher.namespace_regex is rejected instead of silently widening the scope") { + // REGRESSION (fixed 2026-07-26) - the conversion used to ACCEPT a nonempty + // `namespace_regex`, log a server-side warning, drop it, and return a plain + // all-namespaces matcher. A caller asking for `audit-.*` therefore got success plus a + // strictly WIDER scope than it requested, and the only record of that was a log line it + // cannot see. Scope widening must never be the quiet outcome of an unimplemented field. + // + // The field stays unimplemented on purpose - matchers are tested against other + // MATCHERS, not against a concrete namespace, so All-vs-All would have to decide + // whether one regex subsumes another, undecidable in general. Narrow with + // ExactNamespaceMatcher instead. "Unimplemented" therefore has to mean rejected, not + // ignored: IllegalArgumentException is the INVALID_ARGUMENT channel LibraryServiceImpl + // already maps for malformed request data. + val allWithRegex = pb.AllNamespaceMatcher(tenant = Some(tenantMatcherToPb(tenantAll)), namespaceRegex = "audit-.*") + val withRegex = pb.NamespaceMatcher(matcher = pb.NamespaceMatcher.Matcher.All(allWithRegex)) + + val direct = Try(allNamespaceMatcherFromPb(allWithRegex)) + val viaNamespace = Try(namespaceMatcherFromPb(withRegex)) + // the way a real client reaches it: ListLibraryItemsRequest.contexts -> resourceMatcherFromPb + val viaEntryPoint = Try(resourceMatcherFromPb(pb.ResourceMatcher(matcher = pb.ResourceMatcher.Matcher.Namespace(withRegex)))) + + assertTrue( + direct.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + viaNamespace.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + viaEntryPoint.failed.toOption.exists(_.isInstanceOf[IllegalArgumentException]), + // the message must name the field and echo the pattern, or the caller cannot tell + // which part of its request was refused + viaEntryPoint.failed.toOption.exists(_.getMessage.contains("namespace_regex")), + viaEntryPoint.failed.toOption.exists(_.getMessage.contains("audit-.*")) + ) + }, + test("an unset namespace_regex still converts - the rejection is scoped to the field being SET") { + // This conversion runs on every stored item (LibraryItemMetadata.fromPb), so it must be + // inert for the default value - refusing an UNSET regex would make every library item + // unreadable. Nothing writes the field: the UI never sets it, and decoding all 329 items + // in the dogfooding data dir with `protoc --decode` found zero occurrences. (A SET regex + // on the disk-read path is handled BEFORE this conversion - Library.scan strips it and + // keeps the item; libraryScanTest pins that. The refusal above stays load-bearing for + // request data: list/count contexts and saveLibraryItem.) + val allNamespaces = pb.AllNamespaceMatcher(tenant = Some(tenantMatcherToPb(tenantAll))) + val expected = AllNamespaceMatcher(tenant = tenantAll) + + val decoded = Try(allNamespaceMatcherFromPb(allNamespaces)) + // the exact shape an on-disk item arrives in: parsed back from real protobuf bytes + val fromBytes = Try(allNamespaceMatcherFromPb(pb.AllNamespaceMatcher.parseFrom(allNamespaces.toByteArray))) + + assertTrue( + decoded.toOption.contains(expected), + fromBytes.toOption.contains(expected) + ) + } + ) +} diff --git a/server/src/test/scala/producer/ProducerRegistryTest.scala b/server/src/test/scala/producer/ProducerRegistryTest.scala new file mode 100644 index 000000000..9765234a7 --- /dev/null +++ b/server/src/test/scala/producer/ProducerRegistryTest.scala @@ -0,0 +1,242 @@ +package producer + +import zio.* +import zio.test.* + +import com.google.rpc.code.Code +import com.tools.teal.pulsar.ui.api.v1.producer.{ + CreateProducerRequest, + CreateProducerResponse, + DeleteProducerRequest, + DeleteProducerResponse +} +import org.apache.pulsar.client.api.{Producer, ProducerBuilder, PulsarClient} +import pulsar_auth.RequestContext + +import java.lang.reflect.{InvocationHandler, Method, Proxy} +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, CyclicBarrier} +import scala.concurrent.duration.{Duration, SECONDS} +import scala.concurrent.{Await, Future} +import scala.jdk.CollectionConverters.* + +/** The producer REGISTRY: what `createProducer`/`deleteProducer` do to the live broker producers the + * service is holding on behalf of the UI. + * + * Regression context: the registry was a plain `var producers: Map[...]`, and every mutation was a + * read-modify-write (`producers = producers + (name -> p)`) performed from `ExecutionContext.global` + * - the service is a singleton bound on that pool in `GrpcServer`. Two creates that read the same + * old map lose one another's entry, and a create under a name that is already registered simply + * overwrote its predecessor. Either way a producer that is LIVE on the broker disappears from the + * only map that knows its name, so `deleteProducer` can never close it: it holds its topic + * connection (and, on a topic with exclusive access, blocks the next producer) until Dekaf exits. + * + * Everything is asserted through the SERVICE, never through the map field, so the tests describe + * the contract rather than the data structure: a producer the registry no longer holds must have + * been closed, and every name a create reported OK for must still be deletable. + * + * No broker here (the server test tier runs before Pulsar is up), so `PulsarClient`, + * `ProducerBuilder` and `Producer` are implemented in-test with `java.lang.reflect.Proxy` - the same + * device `consumerServiceDeleteTest` uses. `close()` appends to a queue, and that queue is the + * oracle for "what did NOT leak". + */ +object ProducerRegistryTest extends ZIOSpecDefault: + + private val topicFqn = "persistent://public/default/registry-test" + + /** A producer that records the fact it was closed, under a label the test can recognise. */ + private def fakeProducer(label: String, closed: ConcurrentLinkedQueue[String]): Producer[Array[Byte]] = + val handler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "close" => + closed.add(label) + null + case "getProducerName" => label + case "getTopic" => topicFqn + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(label.hashCode) + case "toString" => s"producer-$label" + case _ => null + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[Producer[Array[Byte]]]), handler) + .asInstanceOf[Producer[Array[Byte]]] + + /** A client whose only job is to hand back the next producer when `create()` is called. The + * supplier may block - that is how a create is held inside the broker call while another one + * races past it. */ + private def fakeClient(nextProducer: () => Producer[Array[Byte]]): PulsarClient = + val builderHandler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "create" => nextProducer() + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(java.lang.System.identityHashCode(proxy)) + case "toString" => "producer-builder" + // accessMode/producerName/topic are fluent - they return the same builder + case _ => proxy + + val builder = Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[ProducerBuilder[Array[Byte]]]), builderHandler) + .asInstanceOf[ProducerBuilder[Array[Byte]]] + + val clientHandler = new InvocationHandler: + override def invoke(proxy: Object, method: Method, args: Array[Object]): Object = + method.getName match + case "newProducer" => builder + case "equals" => java.lang.Boolean.valueOf(proxy eq args(0)) + case "hashCode" => java.lang.Integer.valueOf(java.lang.System.identityHashCode(proxy)) + case "toString" => "fake-pulsar-client" + case _ => null + + Proxy + .newProxyInstance(getClass.getClassLoader, Array[Class[?]](classOf[PulsarClient]), clientHandler) + .asInstanceOf[PulsarClient] + + private def await[A](future: Future[A]): A = Await.result(future, Duration(30, SECONDS)) + + private def create(service: ProducerServiceImpl, client: PulsarClient, name: String): CreateProducerResponse = + await( + io.grpc.Context + .current() + .withValue(RequestContext.pulsarClient, client) + .call(() => service.createProducer(CreateProducerRequest(producerName = name, topic = topicFqn))) + ) + + private def delete(service: ProducerServiceImpl, name: String): DeleteProducerResponse = + await(service.deleteProducer(DeleteProducerRequest(producerName = name))) + + def spec = suite(this.getClass.toString)( + test("creating a producer under a name that is already registered closes the one it replaces") { + // REGRESSION - the second create just overwrote the map entry. The first producer is + // still connected to the topic and its name now points at the second one, so nothing can + // ever close it. Reusing a producer name is ordinary: the UI re-creates a producer for + // the same topic after an edit or a page reload. + val closed = ConcurrentLinkedQueue[String]() + val supply = ConcurrentLinkedQueue(List("first", "second").map(fakeProducer(_, closed)).asJava) + val service = ProducerServiceImpl() + val client = fakeClient(() => supply.poll()) + + val firstCreate = create(service, client, "p") + val secondCreate = create(service, client, "p") + val closedAfterReplacement = closed.asScala.toList + + val deleted = delete(service, "p") + + assertTrue( + firstCreate.getStatus.code == Code.OK.value, + secondCreate.getStatus.code == Code.OK.value, + // the predecessor is closed AT the moment it is replaced, not left to chance + closedAfterReplacement == List("first"), + deleted.getStatus.code == Code.OK.value, + // and the survivor is the one delete closes - nothing is left live on the broker + closed.asScala.toList == List("first", "second") + ) ?? s"closedAfterReplacement=$closedAfterReplacement closed=${closed.asScala.toList}" + }, + test("when two creates race under the same name, the producer that loses is closed, not orphaned") { + // REGRESSION, deterministically ordered: the losing create is held inside the broker call + // until the winning one has fully registered, so the late writer overwrites a map entry + // it can see. With a `var Map` the overwritten producer is simply dropped; the registry + // has to hand back whatever it displaced so the caller can close it. + val closed = ConcurrentLinkedQueue[String]() + val slow = fakeProducer("slow", closed) + val fast = fakeProducer("fast", closed) + + val slowIsInsideTheBrokerCall = CountDownLatch(1) + val releaseSlow = CountDownLatch(1) + + val service = ProducerServiceImpl() + val slowClient = fakeClient { () => + slowIsInsideTheBrokerCall.countDown() + releaseSlow.await() + slow + } + val fastClient = fakeClient(() => fast) + + val slowThread = Thread(() => { create(service, slowClient, "p"); () }) + slowThread.start() + + for + _ <- ZIO.attemptBlocking(slowIsInsideTheBrokerCall.await()) + fastCreate <- ZIO.attemptBlocking(create(service, fastClient, "p")) + _ <- ZIO.attemptBlocking { + releaseSlow.countDown() + slowThread.join() + } + closedAfterRace = closed.asScala.toList + deleted <- ZIO.attemptBlocking(delete(service, "p")) + yield assertTrue( + fastCreate.getStatus.code == Code.OK.value, + // the create that arrived last displaced `fast`, so `fast` is the one to close + closedAfterRace == List("fast"), + deleted.getStatus.code == Code.OK.value, + // whichever won, BOTH broker producers are accounted for + closed.asScala.toSet == Set("fast", "slow") + ) ?? s"closedAfterRace=$closedAfterRace closed=${closed.asScala.toList}" + }, + test("concurrent creates under distinct names all stay deletable") { + // REGRESSION - `producers = producers + (name -> p)` reads the map, builds a new one and + // assigns it. Two creates that read the same snapshot lose one entry: that producer is + // live on the broker and its name is no longer in the registry, so `deleteProducer` + // answers FAILED_PRECONDITION forever and the connection is held until Dekaf exits. + // Asserted as "every create the service reported OK for is still deletable", which is + // the property a UI actually depends on. + val closed = ConcurrentLinkedQueue[String]() + val threadCount = 8 + val perThread = 40 + val names = (0 until threadCount * perThread).map(i => f"p$i%04d").toList + + val minted = AtomicInteger(0) + val service = ProducerServiceImpl() + val client = fakeClient(() => fakeProducer(s"producer-${minted.incrementAndGet()}", closed)) + + val startTogether = CyclicBarrier(threadCount) + val threads = (0 until threadCount).map { t => + Thread { () => + startTogether.await() + (0 until perThread).foreach(i => create(service, client, names(t * perThread + i))) + } + } + + for + _ <- ZIO.attemptBlocking { + threads.foreach(_.start()) + threads.foreach(_.join()) + } + deleteResults <- ZIO.attemptBlocking(names.map(name => name -> delete(service, name).getStatus.code)) + undeletable = deleteResults.collect { case (name, code) if code != Code.OK.value => name } + yield assertTrue( + minted.get == names.size, // control: every create really did reach the broker + undeletable.isEmpty, + // nothing survives the sweep - a lost producer is never closed by anything + closed.size == names.size + ) ?? s"undeletable=${undeletable.take(10)} (${undeletable.size}) closed=${closed.size} minted=${minted.get}" + }, + test("deleting a producer closes exactly that producer and leaves its siblings registered") { + // The control for the three above, and the guard on the delete path itself: removal and + // close must stay a matched pair, a second delete of the same name must be refused + // rather than closing something else, and an unrelated name must survive untouched. + val closed = ConcurrentLinkedQueue[String]() + val supply = ConcurrentLinkedQueue(List("a", "b").map(fakeProducer(_, closed)).asJava) + val service = ProducerServiceImpl() + val client = fakeClient(() => supply.poll()) + + create(service, client, "a") + create(service, client, "b") + + val deletedA = delete(service, "a") + val closedAfterFirstDelete = closed.asScala.toList + val deletedAgain = delete(service, "a") + val closedAfterSecondDelete = closed.asScala.toList + val deletedB = delete(service, "b") + + assertTrue( + deletedA.getStatus.code == Code.OK.value, + closedAfterFirstDelete == List("a"), + deletedAgain.getStatus.code == Code.FAILED_PRECONDITION.value, + closedAfterSecondDelete == List("a"), // the refusal closed nothing + deletedB.getStatus.code == Code.OK.value, + closed.asScala.toList == List("a", "b") + ) ?? s"closed=${closed.asScala.toList}" + } + ) diff --git a/server/src/test/scala/producer/ProducerSendTest.scala b/server/src/test/scala/producer/ProducerSendTest.scala new file mode 100644 index 000000000..6db7fc356 --- /dev/null +++ b/server/src/test/scala/producer/ProducerSendTest.scala @@ -0,0 +1,324 @@ +package producer + +import zio.* +import zio.test.* + +import com.google.protobuf.ByteString +import com.google.rpc.code.Code +import com.tools.teal.pulsar.ui.api.v1.producer.{MessageFormat, ProducerMessage, SendRequest, SendResponse} +import org.apache.pulsar.client.admin.{PulsarAdmin, Schemas} +import org.apache.pulsar.client.api.transaction.Transaction +import org.apache.pulsar.client.api.{MessageId, Producer, ProducerStats, Schema, TypedMessageBuilder} +import org.apache.pulsar.common.schema.{SchemaInfo, SchemaType} +import pulsar_auth.RequestContext + +import java.util.concurrent.{CompletableFuture, ConcurrentLinkedQueue} +import scala.concurrent.Future +import scala.jdk.CollectionConverters.* + +/** Service-level coverage for `ProducerServiceImpl.send` - the method itself, not its helpers. + * + * `awaitSendsTest` pins the tail helper, but nothing proved that `send` still CALLS it, nor what + * `send` does to the broker when the batch it was handed is only partly valid. Both are properties + * of the method, so they are asserted by invoking the real `send`. + * + * The server test tier has no broker (CI runs these before Pulsar is even started), so the Pulsar + * client interfaces are implemented directly here rather than with a mocking library. `Recording` + * is deliberately dumb: it appends every `sendAsync` payload to a queue and hands back a + * caller-supplied future. That queue is the broker oracle - "what actually got published" - and the + * future is the broker's verdict, controllable by hand so "answered before the ack" is observable + * instead of timing-dependent. Every method `send` does not use is left `???`, so an unexpected + * interaction fails loudly rather than passing silently. + * + * Each test builds its own service instance and producer, so parallel execution is safe. + */ +object ProducerSendTest extends ZIOSpecDefault { + + private final class Recording(nextFuture: () => CompletableFuture[MessageId]) extends Producer[Array[Byte]]: + /** Every payload handed to the broker, in call order. */ + private val sent = ConcurrentLinkedQueue[String]() + + def published: List[String] = sent.asScala.toList + + private def record(value: Array[Byte]): CompletableFuture[MessageId] = + // `nextFuture` first: it may THROW, which is how a real builder/`sendAsync` reports a + // synchronous failure (producer closed, payload over the max message size), and in that + // case nothing was submitted, so nothing may be recorded as published either. + val future = nextFuture() + sent.add(String(value, "UTF-8")) + future + + override def newMessage(): TypedMessageBuilder[Array[Byte]] = Builder(this) + override def getTopic: String = "persistent://public/default/send-test" + + override def newMessage[V](schema: Schema[V]): TypedMessageBuilder[V] = ??? + override def newMessage(txn: Transaction): TypedMessageBuilder[Array[Byte]] = ??? + override def getProducerName: String = ??? + override def send(message: Array[Byte]): MessageId = ??? + override def sendAsync(message: Array[Byte]): CompletableFuture[MessageId] = ??? + override def flush(): Unit = ??? + override def flushAsync(): CompletableFuture[Void] = ??? + override def getLastSequenceId: Long = ??? + override def getStats: ProducerStats = ??? + override def close(): Unit = ??? + override def closeAsync(): CompletableFuture[Void] = ??? + override def isConnected: Boolean = ??? + override def getLastDisconnectedTimestamp: Long = ??? + override def getNumOfPartitions: Int = ??? + + private final class Builder(producer: Recording) extends TypedMessageBuilder[Array[Byte]]: + private var payload: Array[Byte] = Array.empty + + override def value(value: Array[Byte]): TypedMessageBuilder[Array[Byte]] = + payload = value + this + override def properties(properties: java.util.Map[String, String]): TypedMessageBuilder[Array[Byte]] = this + override def key(key: String): TypedMessageBuilder[Array[Byte]] = this + override def eventTime(timestamp: Long): TypedMessageBuilder[Array[Byte]] = this + override def sendAsync(): CompletableFuture[MessageId] = producer.record(payload) + + override def send(): MessageId = ??? + override def keyBytes(key: Array[Byte]): TypedMessageBuilder[Array[Byte]] = ??? + override def orderingKey(orderingKey: Array[Byte]): TypedMessageBuilder[Array[Byte]] = ??? + override def property(name: String, value: String): TypedMessageBuilder[Array[Byte]] = ??? + override def sequenceId(sequenceId: Long): TypedMessageBuilder[Array[Byte]] = ??? + override def replicationClusters(clusters: java.util.List[String]): TypedMessageBuilder[Array[Byte]] = ??? + override def disableReplication(): TypedMessageBuilder[Array[Byte]] = ??? + override def deliverAt(timestamp: Long): TypedMessageBuilder[Array[Byte]] = ??? + override def deliverAfter(delay: Long, unit: java.util.concurrent.TimeUnit): TypedMessageBuilder[Array[Byte]] = ??? + override def loadConf(config: java.util.Map[String, Object]): TypedMessageBuilder[Array[Byte]] = ??? + + /** A PulsarAdmin that answers exactly one question: "what schema does this topic carry?". */ + private final class SchemaOnlyAdmin(schemaInfo: SchemaInfo) extends PulsarAdmin: + override def schemas(): Schemas = SchemaLookup(schemaInfo) + + override def clusters(): org.apache.pulsar.client.admin.Clusters = ??? + override def brokers(): org.apache.pulsar.client.admin.Brokers = ??? + override def tenants(): org.apache.pulsar.client.admin.Tenants = ??? + override def resourcegroups(): org.apache.pulsar.client.admin.ResourceGroups = ??? + override def properties(): org.apache.pulsar.client.admin.Properties = ??? + override def namespaces(): org.apache.pulsar.client.admin.Namespaces = ??? + override def topics(): org.apache.pulsar.client.admin.Topics = ??? + override def topicPolicies(): org.apache.pulsar.client.admin.TopicPolicies = ??? + override def topicPolicies(isGlobal: Boolean): org.apache.pulsar.client.admin.TopicPolicies = ??? + override def bookies(): org.apache.pulsar.client.admin.Bookies = ??? + override def nonPersistentTopics(): org.apache.pulsar.client.admin.NonPersistentTopics = ??? + override def resourceQuotas(): org.apache.pulsar.client.admin.ResourceQuotas = ??? + override def lookups(): org.apache.pulsar.client.admin.Lookup = ??? + override def functions(): org.apache.pulsar.client.admin.Functions = ??? + override def source(): org.apache.pulsar.client.admin.Source = ??? + override def sources(): org.apache.pulsar.client.admin.Sources = ??? + override def sink(): org.apache.pulsar.client.admin.Sink = ??? + override def sinks(): org.apache.pulsar.client.admin.Sinks = ??? + override def worker(): org.apache.pulsar.client.admin.Worker = ??? + override def brokerStats(): org.apache.pulsar.client.admin.BrokerStats = ??? + override def proxyStats(): org.apache.pulsar.client.admin.ProxyStats = ??? + override def getServiceUrl: String = ??? + override def packages(): org.apache.pulsar.client.admin.Packages = ??? + override def transactions(): org.apache.pulsar.client.admin.Transactions = ??? + override def close(): Unit = ??? + + private final class SchemaLookup(schemaInfo: SchemaInfo) extends Schemas: + override def getSchemaInfo(topic: String): SchemaInfo = schemaInfo + + override def getSchemaInfoAsync(topic: String): CompletableFuture[SchemaInfo] = ??? + override def getSchemaInfoWithVersion(topic: String): org.apache.pulsar.common.schema.SchemaInfoWithVersion = ??? + override def getSchemaInfoWithVersionAsync(topic: String): CompletableFuture[org.apache.pulsar.common.schema.SchemaInfoWithVersion] = ??? + override def getSchemaInfo(topic: String, version: Long): SchemaInfo = ??? + override def getSchemaInfoAsync(topic: String, version: Long): CompletableFuture[SchemaInfo] = ??? + override def deleteSchema(topic: String): Unit = ??? + override def deleteSchemaAsync(topic: String): CompletableFuture[Void] = ??? + override def deleteSchema(topic: String, force: Boolean): Unit = ??? + override def deleteSchemaAsync(topic: String, force: Boolean): CompletableFuture[Void] = ??? + override def createSchema(topic: String, schemaInfo: SchemaInfo): Unit = ??? + override def createSchemaAsync(topic: String, schemaInfo: SchemaInfo): CompletableFuture[Void] = ??? + override def createSchema(topic: String, payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload): Unit = ??? + override def createSchemaAsync( + topic: String, + payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload + ): CompletableFuture[Void] = ??? + override def testCompatibility( + topic: String, + payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload + ): org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse = ??? + override def testCompatibilityAsync( + topic: String, + payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload + ): CompletableFuture[org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse] = ??? + override def getVersionBySchema(topic: String, payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload): java.lang.Long = ??? + override def getVersionBySchemaAsync( + topic: String, + payload: org.apache.pulsar.common.protocol.schema.PostSchemaPayload + ): CompletableFuture[java.lang.Long] = ??? + override def testCompatibility(topic: String, schemaInfo: SchemaInfo): org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse = ??? + override def testCompatibilityAsync( + topic: String, + schemaInfo: SchemaInfo + ): CompletableFuture[org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse] = ??? + override def getVersionBySchema(topic: String, schemaInfo: SchemaInfo): java.lang.Long = ??? + override def getVersionBySchemaAsync(topic: String, schemaInfo: SchemaInfo): CompletableFuture[java.lang.Long] = ??? + override def getAllSchemas(topic: String): java.util.List[SchemaInfo] = ??? + override def getAllSchemasAsync(topic: String): CompletableFuture[java.util.List[SchemaInfo]] = ??? + + /** A JSON-schema'd topic: `jsonToValue` accepts well-formed JSON and rejects everything else, so + * it is the shortest route to a batch that is valid up to item N and invalid at item N+1. */ + private val jsonSchema: SchemaInfo = + SchemaInfo.builder().name("send-test").`type`(SchemaType.JSON).schema(Array.emptyByteArray).build() + + private def msg(value: String): ProducerMessage = + ProducerMessage(value = ByteString.copyFromUtf8(value)) + + private def sendRequest(format: MessageFormat, values: String*): SendRequest = + SendRequest(producerName = "p", format = format, messages = values.map(msg)) + + /** Invoke the real `send` with the admin the interceptor would have installed. */ + private def callSend(producer: Producer[Array[Byte]], request: SendRequest): Future[SendResponse] = + val service = ProducerServiceImpl() + service.producers.put("p", producer) + io.grpc.Context + .current() + .withValue(RequestContext.pulsarAdmin, SchemaOnlyAdmin(jsonSchema)) + .call(() => service.send(request)) + + private def acked(): CompletableFuture[MessageId] = CompletableFuture.completedFuture(MessageId.earliest) + + def spec = suite(this.getClass.toString)( + // ---- finding 7: an invalid item must abort the batch BEFORE anything is published ---- + test("a batch whose second item is invalid publishes nothing") { + // REGRESSION - `send` converted every item first but then INTERLEAVED validation with + // `sendAsync`, so a valid item preceding an invalid one was already on the topic by the + // time the call answered INVALID_ARGUMENT. The caller sees a wholly failed batch and + // retries it, duplicating the item that did land. Validation must cover the whole batch + // before the first publish. + val producer = Recording(() => acked()) + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_JSON, """{"ok":1}""", "not json")) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.INVALID_ARGUMENT.value, + producer.published.isEmpty + ) ?? s"status=${r.getStatus} published=${producer.published}" + }, + test("a batch whose first item is invalid publishes nothing either") { + // The symmetric case, which the interleaved version happened to get right - kept so the + // pair pins "rejection is position-independent" rather than one lucky ordering. + val producer = Recording(() => acked()) + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_JSON, "not json", """{"ok":1}""")) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.INVALID_ARGUMENT.value, + producer.published.isEmpty + ) ?? s"status=${r.getStatus} published=${producer.published}" + }, + test("one invalid item anywhere in a longer batch still publishes nothing") { + val producer = Recording(() => acked()) + val values = Seq("""{"a":1}""", """{"b":2}""", """{"c":3}""", "}{", """{"d":4}""") + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_JSON, values*)) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.INVALID_ARGUMENT.value, + producer.published.isEmpty + ) ?? s"status=${r.getStatus} published=${producer.published}" + }, + test("a wholly valid batch is published in full, in request order") { + // The control for the three above: whole-batch validation must not start rejecting or + // dropping items that are fine. + val producer = Recording(() => acked()) + val values = Seq("""{"a":1}""", """{"b":2}""", """{"c":3}""") + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_JSON, values*)) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.OK.value, + producer.published == values.toList + ) ?? s"status=${r.getStatus} published=${producer.published}" + }, + // ---- finding 24: `send` itself must wait for the broker, not just the helper ---- + test("send does not answer while the broker has not acked") { + // REGRESSION - `send` used to discard the `sendAsync` futures and return Code.OK at once, + // reporting a successful publish for messages the broker had not accepted yet. Asserted + // through `send` rather than through `awaitSends`: restoring the immediate OK inside + // `send` leaves every helper test green. + val brokerVerdict = CompletableFuture[MessageId]() + val producer = Recording(() => brokerVerdict) + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_BYTES, "a", "b")) + + val answeredBeforeAck = response.isCompleted + brokerVerdict.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredBeforeAck, + r.getStatus.code == Code.OK.value, + producer.published == List("a", "b") + ) ?? s"answeredBeforeAck=$answeredBeforeAck status=${r.getStatus}" + }, + test("a broker rejection arriving after the call turns send's response non-OK") { + val brokerVerdict = CompletableFuture[MessageId]() + val producer = Recording(() => brokerVerdict) + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_BYTES, "a")) + + brokerVerdict.completeExceptionally(RuntimeException("Producer send timeout")) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer send timeout") + ) ?? s"status=${r.getStatus}" + }, + // ---- finding 22: no verdict while any submitted send is still travelling ---- + test("a synchronous send failure on a later item waits for the items already submitted") { + // REGRESSION - publication is submitted one item at a time, and a builder/`sendAsync` + // that throws on item N used to break straight out of `send` with FAILED_PRECONDITION. + // Items 1..N-1 were already in flight to the broker and their futures were dropped on + // the floor, so the RPC answered "failed" while part of its own batch was still on its + // way to the topic. The caller retries, and whatever landed is duplicated. Publication + // cannot be made atomic after the fact, but the verdict must not exist until every + // submitted send has settled. + val submitted = java.util.concurrent.atomic.AtomicInteger(0) + val inFlight = CompletableFuture[MessageId]() + val producer = Recording { () => + if submitted.incrementAndGet() == 3 then throw RuntimeException("Producer is closed") + inFlight + } + + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_BYTES, "a", "b", "c")) + + val answeredWhileInFlight = response.isCompleted + inFlight.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredWhileInFlight, + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer is closed"), + // the third item never reached the broker; the first two did and cannot be recalled + producer.published == List("a", "b") + ) ?? s"answeredWhileInFlight=$answeredWhileInFlight status=${r.getStatus} published=${producer.published}" + }, + test("a broker rejection on one item does not answer while a sibling is still in flight") { + // The same guarantee for the asynchronous half: the broker refuses item 1 while item 2 + // is still being decided. Asserted through `send` because `awaitSends` being correct + // proves nothing if `send` stops calling it with the whole batch. + val rejected = CompletableFuture[MessageId]() + val stillInFlight = CompletableFuture[MessageId]() + val nth = java.util.concurrent.atomic.AtomicInteger(0) + val producer = Recording(() => if nth.incrementAndGet() == 1 then rejected else stillInFlight) + + val response = callSend(producer, sendRequest(MessageFormat.MESSAGE_FORMAT_BYTES, "a", "b")) + rejected.completeExceptionally(RuntimeException("Producer fenced")) + + val answeredWhileSiblingInFlight = response.isCompleted + stillInFlight.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredWhileSiblingInFlight, + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer fenced") + ) ?? s"answeredWhileSiblingInFlight=$answeredWhileSiblingInFlight status=${r.getStatus}" + } + ) +} diff --git a/server/src/test/scala/producer/awaitSendsTest.scala b/server/src/test/scala/producer/awaitSendsTest.scala new file mode 100644 index 000000000..96b88ec92 --- /dev/null +++ b/server/src/test/scala/producer/awaitSendsTest.scala @@ -0,0 +1,168 @@ +package producer + +import zio.* +import zio.test.* + +import com.google.rpc.code.Code +import org.apache.pulsar.client.api.MessageId + +import java.util.concurrent.CompletableFuture + +/** `producer.awaitSends` is the tail of `ProducerServiceImpl.send`: it turns the in-flight + * `sendAsync` futures into the gRPC response. + * + * Regression context: `send` called `newMessage.sendAsync` and DISCARDED every future, then + * answered `Code.OK` unconditionally. A Pulsar `sendAsync` future completes only when the broker + * has acknowledged (or rejected) the message, so every asynchronous rejection - schema + * incompatibility, producer fenced, exceeded quota, terminated topic, send timeout - was reported + * to the UI as a successful publish for a message that never landed. + * + * The second half of the same defect: once the futures WERE awaited, `Future.sequence` awaited them + * fail-fast, so one rejection completed the RPC while its siblings were still travelling to the + * broker. Those siblings landed after the caller had been told the batch failed, and the obvious + * retry duplicated them. Publication is not atomic - `send` submits one item at a time - so what + * this helper owes the caller is that the batch has stopped moving by the time its verdict exists. + * + * These use real `CompletableFuture`s (exactly what the Pulsar client hands back) rather than a + * mock producer, and complete them by hand so the "answered before the ack" case is observable + * instead of timing-dependent. `awaitSends` holds no shared state, so parallel execution is safe. + */ +object awaitSendsTest extends ZIOSpecDefault { + + private def acked(): CompletableFuture[MessageId] = + CompletableFuture.completedFuture(MessageId.earliest) + + private def pending(): CompletableFuture[MessageId] = + new CompletableFuture[MessageId]() + + def spec = suite(this.getClass.toString)( + test("no response is produced while a send is still in flight") { + val inFlight = pending() + val response = awaitSends(Seq(acked(), inFlight)) + + val answeredBeforeAck = response.isCompleted + inFlight.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue(!answeredBeforeAck, r.getStatus.code == Code.OK.value) ?? + "the response must not exist until the broker has acked every message" + }, + test("a rejection that arrives after the call still turns the response non-OK") { + // The exact production shape: the response is being built while the broker is still + // deciding, and it decides "no". + val inFlight = pending() + val response = awaitSends(Seq(acked(), inFlight)) + + inFlight.completeExceptionally(new RuntimeException("Producer send timeout")) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code != Code.OK.value, + r.getStatus.message.contains("Producer send timeout") + ) + }, + test("a rejection already present when the response is built is reported") { + val rejected = pending() + rejected.completeExceptionally(new RuntimeException("Topic terminated")) + + for r <- ZIO.fromFuture(_ => awaitSends(Seq(rejected, acked()))) + yield assertTrue( + r.getStatus.code != Code.OK.value, + r.getStatus.message.contains("Topic terminated") + ) + }, + test("a rejection does not answer the batch while a sibling send is still in flight") { + // REGRESSION - `Future.sequence` is FAIL-FAST: the first rejected future completed the + // whole response while its siblings were still on their way to the broker. The caller is + // told the batch failed, the siblings land afterwards, and the natural retry duplicates + // them. Every submitted send has to SETTLE before the verdict exists. + val rejected = pending() + rejected.completeExceptionally(new RuntimeException("Producer send timeout")) + val stillInFlight = pending() + + val response = awaitSends(Seq(rejected, stillInFlight)) + + val answeredWhileSiblingInFlight = response.isCompleted + stillInFlight.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredWhileSiblingInFlight, + r.getStatus.code != Code.OK.value, + r.getStatus.message.contains("Producer send timeout") + ) ?? s"answeredWhileSiblingInFlight=$answeredWhileSiblingInFlight status=${r.getStatus}" + }, + test("a rejection arriving mid-flight still waits for the sibling that is left") { + // Same defect, reached the other way round: both sends are pending when the response is + // built and one is rejected afterwards, so the fail-fast short circuit fires on the + // completion thread rather than on the calling thread. + val first = pending() + val second = pending() + + val response = awaitSends(Seq(first, second)) + first.completeExceptionally(new RuntimeException("Topic terminated")) + + val answeredWhileSiblingInFlight = response.isCompleted + second.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredWhileSiblingInFlight, + r.getStatus.code != Code.OK.value, + r.getStatus.message.contains("Topic terminated") + ) ?? s"answeredWhileSiblingInFlight=$answeredWhileSiblingInFlight status=${r.getStatus}" + }, + test("two rejections report one of them rather than losing the verdict") { + val first = pending() + val second = pending() + val response = awaitSends(Seq(first, second)) + + first.completeExceptionally(new RuntimeException("Producer fenced")) + second.completeExceptionally(new RuntimeException("Topic terminated")) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer fenced") + ) ?? s"status=${r.getStatus}" + }, + test("a submit failure is reported only after every send already submitted has settled") { + // `send` publishes one item at a time, so a builder/`sendAsync` that throws on item N + // leaves items 1..N-1 in flight. The failure the caller sees is the submit failure, but + // it may not be produced until those siblings have settled - otherwise the RPC answers + // while part of its own batch is still travelling to the topic. + val inFlight = pending() + val submitFailure = new RuntimeException("Producer is closed") + + val response = awaitSends(Seq(inFlight), Some(submitFailure)) + + val answeredWhileSiblingInFlight = response.isCompleted + inFlight.complete(MessageId.latest) + + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + !answeredWhileSiblingInFlight, + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer is closed") + ) ?? s"answeredWhileSiblingInFlight=$answeredWhileSiblingInFlight status=${r.getStatus}" + }, + test("a submit failure with nothing in flight is reported immediately") { + val response = awaitSends(Seq.empty, Some(new RuntimeException("Producer is closed"))) + for r <- ZIO.fromFuture(_ => response) + yield assertTrue( + response.isCompleted, + r.getStatus.code == Code.FAILED_PRECONDITION.value, + r.getStatus.message.contains("Producer is closed") + ) + }, + test("a fully acked batch is OK") { + for r <- ZIO.fromFuture(_ => awaitSends(Seq(acked(), acked(), acked()))) + yield assertTrue(r.getStatus.code == Code.OK.value, r.getStatus.message.isEmpty) + }, + test("an empty batch answers OK immediately") { + val response = awaitSends(Seq.empty) + for r <- ZIO.fromFuture(_ => response) + yield assertTrue(response.isCompleted, r.getStatus.code == Code.OK.value) + } + ) +} diff --git a/server/src/test/scala/producer/jsonToValueTest.scala b/server/src/test/scala/producer/jsonToValueTest.scala new file mode 100644 index 000000000..c713db544 --- /dev/null +++ b/server/src/test/scala/producer/jsonToValueTest.scala @@ -0,0 +1,502 @@ +package producer + +import zio.* +import zio.test.* +import zio.test.Assertion.* + +import org.apache.pulsar.common.schema.{SchemaInfo, SchemaType} +import _root_.conversions.primitiveConv +import _root_.schema.avro + +import java.nio.charset.StandardCharsets +import io.circe.parser.parse as parseJson + +/* Tests for `producer.jsonToValue` - the WRITE path that turns a JSON payload from the + * producer UI into the wire bytes for a topic's schema. It is the mirror of the READ path + * covered by `conversions.primitiveConvTest`, so wherever possible the corresponding + * `primitiveConv` decoder is used as the oracle for a round-trip. + * + * `jsonToValue` is a pure function over its arguments (no shared/singleton state), so the + * default parallel execution of ZIO Test suites is safe here - no `TestAspect.sequential`. + */ +object jsonToValueTest extends ZIOSpecDefault { + + private def schemaInfoOf(schemaType: SchemaType, definition: Array[Byte] = Array.emptyByteArray): SchemaInfo = + SchemaInfo.builder + .name(s"test-$schemaType") + .`type`(schemaType) + .schema(definition) + .build + + private def encode(schemaType: SchemaType, json: String): Either[Throwable, Array[Byte]] = + jsonToValue(schemaInfoOf(schemaType), json.getBytes(StandardCharsets.UTF_8)) + + private def passesThrough(schemaType: SchemaType, payload: String): Boolean = + encode(schemaType, payload) match + case Right(bytes) => bytes.sameElements(payload.getBytes(StandardCharsets.UTF_8)) + case Left(_) => false + + def spec = suite(this.getClass.toString)( + // ---------------------------------------------------------------- INT8 + test("INT8 encodes one big-endian byte that round-trips through primitiveConv.bytesToInt8") { + case class TestCase(json: String, expected: Byte) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.INT8, testCase.json) match + case Right(bytes) => bytes.length == 1 && primitiveConv.bytesToInt8(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0), + TestCase("1", 1), + TestCase("-1", -1), + TestCase("42", 42), + TestCase("-42", -42), + TestCase("127", Byte.MaxValue), + TestCase("-128", Byte.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("INT8 rejects out-of-range and unparseable input") { + // Out-of-range is caught by the explicit Byte.MinValue/MaxValue guard + // (ProducerServiceImpl.scala:190) - this one is a real, non-vacuous check. + val rejected = List("128", "-128000", "-129", "1.5", "0x2a", "abc", "true", " 42", "42 ", "+42", "") + + val failures = rejected.filterNot(json => encode(SchemaType.INT8, json).isLeft) + assertTrue(failures.isEmpty) + }, + // --------------------------------------------------------------- INT16 + test("INT16 encodes two big-endian bytes that round-trip through primitiveConv.bytesToInt16") { + case class TestCase(json: String, expected: Short) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.INT16, testCase.json) match + case Right(bytes) => bytes.length == 2 && primitiveConv.bytesToInt16(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0), + TestCase("1", 1), + TestCase("-1", -1), + TestCase("42", 42), + TestCase("-42", -42), + TestCase("32767", Short.MaxValue), + TestCase("-32768", Short.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("INT16 rejects out-of-range and unparseable input") { + // Guarded by the explicit Short.MinValue/MaxValue check (ProducerServiceImpl.scala:201). + val rejected = List("32768", "-32769", "2147483647", "1.5", "abc", "") + + val failures = rejected.filterNot(json => encode(SchemaType.INT16, json).isLeft) + assertTrue(failures.isEmpty) + }, + // --------------------------------------------------------------- INT32 + test("INT32 encodes four big-endian bytes that round-trip through primitiveConv.bytesToInt32") { + case class TestCase(json: String, expected: Int) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.INT32, testCase.json) match + case Right(bytes) => bytes.length == 4 && primitiveConv.bytesToInt32(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0), + TestCase("1", 1), + TestCase("-1", -1), + TestCase("42", 42), + TestCase("-42", -42), + TestCase("2147483647", Int.MaxValue), + TestCase("-2147483648", Int.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("INT32 rejects out-of-range and unparseable input") { + // NOTE: the explicit range guard at ProducerServiceImpl.scala:211 is VACUOUS - + // `n` is already an Int, so `n > Int.MaxValue || n < Int.MinValue` can never hold. + // Out-of-range input is nonetheless rejected, because Guava's Ints.tryParse returns + // null on overflow. The observable contract is therefore still fail-closed; only the + // error MESSAGE is wrong ("Unable to parse" instead of "out of range"). Asserting the + // rejection, not the message. + val rejected = List("2147483648", "-2147483649", "9223372036854775807", "1.5", "abc", "") + + val failures = rejected.filterNot(json => encode(SchemaType.INT32, json).isLeft) + assertTrue(failures.isEmpty) + }, + // --------------------------------------------------------------- INT64 + test("INT64 encodes eight big-endian bytes that round-trip through primitiveConv.bytesToInt64") { + case class TestCase(json: String, expected: Long) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.INT64, testCase.json) match + case Right(bytes) => bytes.length == 8 && primitiveConv.bytesToInt64(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0L), + TestCase("1", 1L), + TestCase("-1", -1L), + TestCase("42", 42L), + TestCase("-42", -42L), + TestCase("2147483648", 2147483648L), + TestCase("9223372036854775807", Long.MaxValue), + TestCase("-9223372036854775808", Long.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("INT64 rejects out-of-range and unparseable input") { + // Same shape as INT32: the guard at ProducerServiceImpl.scala:221 is vacuous + // (`n` is already a Long); Longs.tryParse returning null is what actually rejects. + val rejected = List("9223372036854775808", "-9223372036854775809", "1.5", "abc", "") + + val failures = rejected.filterNot(json => encode(SchemaType.INT64, json).isLeft) + assertTrue(failures.isEmpty) + }, + // ----------------------------------------------- INT8/16/32/64, shared + // REGRESSION - the four integer branches handed the payload straight to Guava's + // `Ints/Longs.tryParse`, which accepts JAVA integer literal syntax rather than JSON: `01`, + // `00` and `-01` all parsed and were encoded onto the topic. A leading zero is not a valid + // JSON number, and this is the JSON message format - FLOAT/DOUBLE on the same switch have + // gated on JSON number syntax since 2026-07-25, the integer widths had not. + test("integer schemas reject leading-zero literals that are not valid JSON numbers") { + val rejected = List("01", "00", "007", "-01", "-00", "0123") + val widths = List(SchemaType.INT8, SchemaType.INT16, SchemaType.INT32, SchemaType.INT64) + + val accepted = + for + width <- widths + json <- rejected + if encode(width, json).isRight + yield s"$width accepted $json" + + assertTrue(accepted.isEmpty) ?? s"accepted non-JSON integer literals: ${accepted.mkString(", ")}" + }, + test("a single zero and ordinary integers still encode for every integer width") { + // The control for the case above: the JSON-syntax gate must reject the leading-zero + // forms WITHOUT also rejecting `0` itself, or negative and multi-digit values. + val accepted = List("0", "-0", "7", "-7", "42", "-42") + val widths = List(SchemaType.INT8, SchemaType.INT16, SchemaType.INT32, SchemaType.INT64) + + val rejected = + for + width <- widths + json <- accepted + if encode(width, json).isLeft + yield s"$width rejected $json" + + assertTrue(rejected.isEmpty) ?? s"rejected valid JSON integers: ${rejected.mkString(", ")}" + }, + // --------------------------------------------------------------- FLOAT + test("FLOAT encodes four bytes that round-trip through primitiveConv.bytesToFloat32") { + case class TestCase(json: String, expected: Float) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.FLOAT, testCase.json) match + case Right(bytes) => bytes.length == 4 && primitiveConv.bytesToFloat32(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0.0f), + TestCase("0.0", 0.0f), + TestCase("1", 1.0f), + TestCase("-1", -1.0f), + TestCase("1.5", 1.5f), + TestCase("-1.5", -1.5f), + TestCase("42", 42.0f), + TestCase("3.4028235E38", Float.MaxValue), + TestCase("-3.4028235E38", Float.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("FLOAT rejects infinities and values that overflow the float range") { + // The bound check at ProducerServiceImpl.scala:231 is a symmetric magnitude bound + // (Scala's Float.MinValue is -Float.MaxValue), so the only values it can reject are + // the infinities - including inputs that parse INTO an infinity, e.g. "1e39". + val rejected = List("Infinity", "-Infinity", "1e39", "-1e39") + + val failures = rejected.filterNot(json => encode(SchemaType.FLOAT, json).isLeft) + assertTrue(failures.isEmpty) + }, + // REGRESSION (fixed 2026-07-25) - FLOAT used to accept NaN. The guard at ProducerServiceImpl.scala:231 + // (`n > MaxValue || n < MinValue`) is always false for NaN, so NaN slips past the very + // check that rejects both infinities and is encoded as 0x7fc00000 onto the topic - + // even though `NaN` is not valid JSON and this is the JSON message format. + test("FLOAT rejects NaN") { + val rejected = List("NaN", "-NaN") + + val failures = rejected.filterNot(json => encode(SchemaType.FLOAT, json).isLeft) + assertTrue(failures.isEmpty) + }, + // REGRESSION (fixed 2026-07-25) - the FLOAT/DOUBLE branches delegated parsing to Guava, whose + // FLOATING_POINT_PATTERN is Java literal syntax, not JSON: `+1`, `01`, `.5`, `1.`, a hex + // float literal and a trailing `f`/`d` suffix all parsed and were silently encoded onto the + // topic. This is the JSON message format - the STRING branch on the same switch has always + // required real JSON - so the payload has to be a JSON number. + test("FLOAT rejects Java float literals that are not valid JSON numbers") { + val rejected = List("+1", "01", ".5", "1.", "0x1p3", "1f", "1d", "1.5F") + + val failures = rejected.filterNot(json => encode(SchemaType.FLOAT, json).isLeft) + assertTrue(failures.isEmpty) ?? s"accepted non-JSON numeric forms: ${failures.mkString(", ")}" + }, + // -------------------------------------------------------------- DOUBLE + test("DOUBLE encodes eight bytes that round-trip through primitiveConv.bytesToFloat64") { + case class TestCase(json: String, expected: Double) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.DOUBLE, testCase.json) match + case Right(bytes) => bytes.length == 8 && primitiveConv.bytesToFloat64(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("0", 0.0d), + TestCase("0.0", 0.0d), + TestCase("1", 1.0d), + TestCase("-1", -1.0d), + TestCase("1.5", 1.5d), + TestCase("-1.5", -1.5d), + TestCase("42", 42.0d), + TestCase("1e39", 1e39d), + TestCase("1.7976931348623157E308", Double.MaxValue), + TestCase("-1.7976931348623157E308", Double.MinValue) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("DOUBLE rejects infinities and values that overflow the double range") { + val rejected = List("Infinity", "-Infinity", "1e309", "-1e309") + + val failures = rejected.filterNot(json => encode(SchemaType.DOUBLE, json).isLeft) + assertTrue(failures.isEmpty) + }, + // REGRESSION (fixed 2026-07-25) - DOUBLE used to accept NaN, same reason as FLOAT: the guard at + // ProducerServiceImpl.scala:241 compares against NaN and is therefore always false, + // so NaN is encoded as 0x7ff8000000000000 while both infinities are rejected. + test("DOUBLE rejects NaN") { + val rejected = List("NaN", "-NaN") + + val failures = rejected.filterNot(json => encode(SchemaType.DOUBLE, json).isLeft) + assertTrue(failures.isEmpty) + }, + // See the FLOAT case above - same Guava parser, same non-JSON forms. + test("DOUBLE rejects Java double literals that are not valid JSON numbers") { + val rejected = List("+1", "01", ".5", "1.", "0x1p3", "1f", "1d", "1.5D") + + val failures = rejected.filterNot(json => encode(SchemaType.DOUBLE, json).isLeft) + assertTrue(failures.isEmpty) ?? s"accepted non-JSON numeric forms: ${failures.mkString(", ")}" + }, + // ------------------------------------------------------------- BOOLEAN + test("BOOLEAN encodes one byte that round-trips through primitiveConv.bytesToBoolean") { + case class TestCase(json: String, expected: Boolean) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.BOOLEAN, testCase.json) match + case Right(bytes) => bytes.length == 1 && primitiveConv.bytesToBoolean(bytes) == Right(testCase.expected) + case Left(_) => false + + val testCases = List( + TestCase("true", true), + TestCase("false", false) + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("BOOLEAN accepts only the two bare lowercase literals, byte-for-byte") { + // Documents the ACTUAL contract: BOOLEAN does a raw string compare of the whole + // payload (ProducerServiceImpl.scala:177-181) rather than parsing JSON like STRING + // does. It happens to accept exactly the canonical JSON boolean literals, but it is + // stricter than JSON: " true" and "true\n" are valid JSON booleans and are rejected. + // Fail-closed, so documented rather than flagged as a defect - but the two branches + // of the same endpoint disagree about what "JSON" means. + val rejected = List("True", "TRUE", "\"true\"", " true", "true ", "true\n", "1", "0", "yes", "on", "null", "") + + val failures = rejected.filterNot(json => encode(SchemaType.BOOLEAN, json).isLeft) + assertTrue(failures.isEmpty) + }, + // -------------------------------------------------------------- STRING + test("STRING requires a quoted JSON string literal and round-trips through primitiveConv.bytesToString") { + case class TestCase(json: String, expected: String) + + def runTestCase(testCase: TestCase): Boolean = + encode(SchemaType.STRING, testCase.json) match + case Right(bytes) => primitiveConv.bytesToString(bytes) == testCase.expected + case Left(_) => false + + val testCases = List( + TestCase("\"\"", ""), + TestCase("\"hello\"", "hello"), + TestCase("\"123\"", "123"), + TestCase("\"a\\nb\"", "a\nb"), + TestCase("\"qu\\\"ote\\\"s\"", """qu"ote"s"""), + // Non-ASCII guards the encoding of the write path: the payload must come back out + // as UTF-8, which is what the read path (primitiveConv.bytesToString) assumes. + TestCase("\"Gruß\"", "Gruß"), + TestCase("\"世界\"", "世界") + ) + + val failures = testCases.filterNot(runTestCase).map(_.json) + assertTrue(failures.isEmpty) + }, + test("STRING rejects unquoted, non-string and malformed JSON") { + val rejected = List( + "hello", // bare token: not valid JSON at all + "123", // valid JSON, but a number + "true", + "null", + "{}", + """{"a":1}""", + "[]", + """["a"]""", + "\"unterminated", + "" // empty payload + ) + + val failures = rejected.filterNot(json => encode(SchemaType.STRING, json).isLeft) + assertTrue(failures.isEmpty) + }, + // ---------------------------------------------------------- JSON, NONE + test("JSON forwards a syntactically valid payload byte-for-byte") { + // The bytes must reach the topic unchanged - validation must not reformat the payload. + val payloads = List( + """{"a":1}""", + """{"a":{"b":[1,2]},"c":null}""", + "[1,2,3]", + "null", + "true", + "123", + "\"a string\"", + """ {"a":1} """, // surrounding whitespace is legal JSON and is preserved + """{"a":"Grüß 世界"}""" + ) + + val failures = payloads.filterNot(p => passesThrough(SchemaType.JSON, p)) + assertTrue(failures.isEmpty) + }, + // REGRESSION (fixed 2026-07-25) - `case SchemaType.JSON => Right(jsonAsBytes)` did no parsing at all, so a + // topic with a JSON schema accepted arbitrary bytes: the producer reported success and the + // broken payload landed on the topic, where the READ path then fails to deserialize it. The + // sibling STRING branch on the same switch has always parsed strict JSON. + test("JSON rejects a payload that is not valid JSON") { + val rejected = List( + "not json at all", + """{"a":}""", + "{", + "[1,2", + "{'a':1}", // single quotes are not JSON + """{"a":1} trailing""", + """{"a":1}{"b":2}""", // two documents, not one + "undefined", + "NaN", + "" // empty payload + ) + + val failures = rejected.filterNot(json => encode(SchemaType.JSON, json).isLeft) + assertTrue(failures.isEmpty) + }, + test("NONE stays permissive and forwards raw bytes byte-for-byte") { + // Deliberately NOT tightened: SchemaType.NONE means "no schema", so the payload is + // opaque bytes that may legitimately not be JSON at all. + val payloads = List("""{"a":1}""", "[1,2,3]", "null", "", "not json at all", """{"a":}""", "\u0000\u0001raw") + + val failures = payloads.filterNot(p => passesThrough(SchemaType.NONE, p)) + assertTrue(failures.isEmpty) + }, + // ---------------------------------------------------------------- AVRO + test("AVRO encodes against the writer schema and rejects payloads that do not fit it") { + val avroSchemaDefinition = + """ + |{ + | "type": "record", + | "name": "User", + | "fields": [ + | {"name": "name", "type": "string"}, + | {"name": "favorite_number", "type": "int"} + | ] + |} + """.stripMargin + + val schemaInfo = schemaInfoOf(SchemaType.AVRO, avroSchemaDefinition.getBytes(StandardCharsets.UTF_8)) + + val jsonToEncode = """{"name":"Alyssa","favorite_number":256}""" + val encoded = jsonToValue(schemaInfo, jsonToEncode.getBytes(StandardCharsets.UTF_8)) + + // Round-trip back through the matching decoder used by the consumer path. + val decoded = encoded.flatMap(bytes => avro.converters.toJson(avroSchemaDefinition.getBytes(StandardCharsets.UTF_8), bytes)) + val roundTripped = decoded.map(bytes => parseJson(String(bytes, StandardCharsets.UTF_8))) + + val rejected = List( + """{"name":"Alyssa"}""", // missing required field + """{"name":"Alyssa","favorite_number":"not a number"}""", + "not json at all", + "" + ) + val rejectFailures = rejected.filterNot(json => jsonToValue(schemaInfo, json.getBytes(StandardCharsets.UTF_8)).isLeft) + + assertTrue( + encoded.isRight, + roundTripped == Right(parseJson(jsonToEncode)), + rejectFailures.isEmpty + ) + }, + // ---------------------------------------------------- PROTOBUF_NATIVE + test("PROTOBUF_NATIVE rejects a payload it cannot encode against the schema descriptor") { + // A valid descriptor needs a compiled .proto (covered in schema.protobufnative tests); + // here we only pin that the branch surfaces the converter's failure as a Left rather + // than throwing out of jsonToValue. + val schemaInfo = schemaInfoOf(SchemaType.PROTOBUF_NATIVE, "not a descriptor".getBytes(StandardCharsets.UTF_8)) + + assertTrue(jsonToValue(schemaInfo, """{"a":1}""".getBytes(StandardCharsets.UTF_8)).isLeft) + }, + // --------------------------------------------------- unsupported types + test("unsupported schema types are rejected") { + val unsupported = List( + SchemaType.BYTES, + SchemaType.DATE, + SchemaType.TIME, + SchemaType.TIMESTAMP, + SchemaType.INSTANT, + SchemaType.LOCAL_DATE, + SchemaType.LOCAL_TIME, + SchemaType.LOCAL_DATE_TIME, + SchemaType.KEY_VALUE, + SchemaType.PROTOBUF + ) + + def rejectsWithUnsupportedMessage(schemaType: SchemaType): Boolean = + encode(schemaType, """{"a":1}""") match + case Left(err) => err.getMessage.startsWith("Unsupported schema type") + case Right(_) => false + + val failures = unsupported.filterNot(rejectsWithUnsupportedMessage).map(_.toString) + assertTrue(failures.isEmpty) + }, + // --------------------------------------------------------- empty input + test("an empty payload is rejected for every primitive schema type") { + val primitives = List( + SchemaType.INT8, + SchemaType.INT16, + SchemaType.INT32, + SchemaType.INT64, + SchemaType.FLOAT, + SchemaType.DOUBLE, + SchemaType.BOOLEAN, + SchemaType.STRING + ) + + val failures = primitives.filterNot(t => jsonToValue(schemaInfoOf(t), Array.emptyByteArray).isLeft).map(_.toString) + assertTrue(failures.isEmpty) + } + ) +} diff --git a/server/src/test/scala/pulsar_auth/pulsarAuthCookieTest.scala b/server/src/test/scala/pulsar_auth/pulsarAuthCookieTest.scala new file mode 100644 index 000000000..b11cdee4b --- /dev/null +++ b/server/src/test/scala/pulsar_auth/pulsarAuthCookieTest.scala @@ -0,0 +1,425 @@ +package pulsar_auth + +import zio.test.* +import ch.qos.logback.classic.{Logger as LogbackLogger} +import ch.qos.logback.classic.spi.ILoggingEvent +import org.slf4j.LoggerFactory +import scala.jdk.CollectionConverters.* + +/** Cookie construction + parsing. This carries the session credential (tokens, OAuth2 private keys, + * auth-param strings), so both the hardening attributes and the encode/decode round-trip matter. + * + * Regression context: `Secure` was computed and then never interpolated into the header (the + * SameSite fragment was interpolated twice instead), so the cookie could never be marked Secure. + * Later: the JSON value was interpolated raw into the header, so a `;` inside any raw field + * (authPluginClassName, credential names) truncated the stored credential and injected cookie + * attributes - the value is whole-encoded now, and the oracle below parses the full header so an + * injected attribute can never hide from the assertions again. + */ +object pulsarAuthCookieTest extends ZIOSpecDefault: + + private def auth(creds: (String, Credentials)*): PulsarAuth = + PulsarAuth(current = Some(creds.headOption.map(_._1).getOrElse("Default")), credentials = creds.toMap) + + private val empty = auth("Default" -> EmptyCredentials(`type` = "empty")) + + /** Parse the Set-Cookie header the way a browser tokenizes it: everything before the first `;` + * is `name=value`, every further `;`-separated token is an attribute. The old oracle extracted + * the value with `takeWhile(_ != ';')` and looked no further - structurally blind to an + * injected `;`, which simply truncated what it looked at. Splitting the WHOLE header and + * asserting the attribute list EXACTLY is what makes a smuggled attribute visible. */ + private def parseSetCookie(header: String): (String, List[String]) = + val parts = header.split(";", -1).toList.map(_.trim) + (parts.head, parts.tail.filter(_.nonEmpty)) + + /** The value a browser would store and send back. */ + private def valueOf(cookie: String): String = + parseSetCookie(cookie)._1.stripPrefix("pulsar_auth=") + + /** The attribute tokens after the value - everything the browser enforces. */ + private def attributeListOf(cookie: String): List[String] = + parseSetCookie(cookie)._2 + + // ---- log capture (for the no-credential-in-logs test). The "pulsar-auth" logger is shared + // with parallel suites, so assertions key on content UNIQUE to the test, never on counts. ---- + + /** A thread-safe capture appender - parallel suites append to the same logger while we + * snapshot, and logback's own ListAppender backs onto a plain ArrayList. */ + private final class QueueAppender extends ch.qos.logback.core.AppenderBase[ILoggingEvent]: + val events = new java.util.concurrent.ConcurrentLinkedQueue[ILoggingEvent]() + override def append(event: ILoggingEvent): Unit = + events.add(event) + () + + // SLF4J hands a SubstituteLogger to callers that arrive while the backend is still + // initializing; re-fetch until the real logback binding is in place (milliseconds). + private def pulsarAuthLogbackLogger(): LogbackLogger = + var logger = LoggerFactory.getLogger("pulsar-auth") + var attempts = 0 + while !logger.isInstanceOf[LogbackLogger] && attempts < 500 do + Thread.sleep(2) + logger = LoggerFactory.getLogger("pulsar-auth") + attempts += 1 + logger match + case l: LogbackLogger => l + case other => throw new IllegalStateException(s"Expected a logback logger, got ${other.getClass.getName}") + + private def withCapturedPulsarAuthLogs[A](body: => A): (A, List[String]) = + val logbackLogger = pulsarAuthLogbackLogger() + val appender = new QueueAppender() + appender.start() + logbackLogger.addAppender(appender) + try + val result = body + (result, appender.events.asScala.toList.map(_.getFormattedMessage)) + finally + logbackLogger.detachAppender(appender) + appender.stop() + + def spec = suite(this.getClass.toString)( + test("Secure is emitted when cookieSecure is true") { + val cookie = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = None) + assertTrue(attributeListOf(cookie).contains("Secure")) + }, + test("Secure is absent when cookieSecure is false or unset") { + val off = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(false), cookieSameSite = None) + val unset = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + assertTrue( + !attributeListOf(off).contains("Secure"), + !attributeListOf(unset).contains("Secure") + ) + }, + test("SameSite appears exactly once") { + // It used to be interpolated twice, which is also how the Secure fragment went missing. + val cookie = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some("strict")) + val occurrences = "SameSite".r.findAllIn(cookie).size + assertTrue(occurrences == 1) ?? s"SameSite occurred $occurrences times in: $cookie" + }, + test("Secure and SameSite are emitted together when both are configured") { + val cookie = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some("lax")) + val attrs = attributeListOf(cookie) + assertTrue(attrs.contains("Secure"), attrs.contains("SameSite=Lax")) + }, + test("SameSite=None is only emitted when the cookie is also Secure") { + // Browsers reject SameSite=None without Secure, so emitting it alone would break auth. + val secure = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some("none")) + val insecure = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(false), cookieSameSite = Some("none")) + assertTrue( + attributeListOf(secure).contains("SameSite=None"), + !attributeListOf(insecure).exists(_.contains("SameSite")) + ) + }, + test("lax and strict are emitted regardless of Secure") { + val lax = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = Some("lax")) + val strict = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = Some("strict")) + assertTrue( + attributeListOf(lax).contains("SameSite=Lax"), + attributeListOf(strict).contains("SameSite=Strict") + ) + }, + test("the cookie is always HttpOnly and long-lived") { + val attrs = attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None)) + assertTrue(attrs.contains("HttpOnly"), attrs.contains("Max-Age=31536000")) + }, + test("Path is derived from publicBaseUrl, defaulting to /") { + val default = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val rooted = pulsarAuthToCookie(empty, publicBaseUrl = Some("http://host:8090"), cookieSecure = None, cookieSameSite = None) + val subPath = pulsarAuthToCookie(empty, publicBaseUrl = Some("http://host:8090/dekaf"), cookieSecure = None, cookieSameSite = None) + assertTrue( + attributeListOf(default).contains("Path=/"), + attributeListOf(rooted).contains("Path=/"), + attributeListOf(subPath).contains("Path=/dekaf") + ) + }, + test("the attribute list is exactly the configured set - nothing more, nothing less") { + // Presence checks alone cannot catch an EXTRA attribute riding along, and an injected + // attribute is precisely an extra one. Pin the exact list for both config extremes. + val minimal = attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None)) + val full = attributeListOf( + pulsarAuthToCookie(empty, publicBaseUrl = Some("http://host:8090/dekaf"), cookieSecure = Some(true), cookieSameSite = Some("lax")) + ) + assertTrue( + minimal == List("Path=/", "HttpOnly", "Max-Age=31536000"), + full == List("Path=/dekaf", "HttpOnly", "Max-Age=31536000", "Secure", "SameSite=Lax") + ) ?? s"minimal=$minimal full=$full" + }, + test("cookieSameSite is matched case-insensitively and trimmed") { + // REGRESSION (fixed 2026-07-25) - the match required exact lowercase, so `Lax`/`STRICT` + // (and a stray trailing space from YAML) fell through to "" and emitted NO SameSite + // attribute. The operator saw a configured value; the browser got its default. Silently + // dropping a CSRF control on a capitalisation difference is the bug this pins. + val cases = List( + "lax" -> "SameSite=Lax", + "Lax" -> "SameSite=Lax", + "LAX" -> "SameSite=Lax", + " lax " -> "SameSite=Lax", + "strict" -> "SameSite=Strict", + "Strict" -> "SameSite=Strict", + "STRICT" -> "SameSite=Strict", + "\tStRiCt\n" -> "SameSite=Strict" + ) + val wrong = cases.filterNot: (configured, expected) => + attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = Some(configured))).contains(expected) + assertTrue(wrong.isEmpty) ?? s"these did not produce their attribute: ${wrong.map(_._1).mkString("|")}" + }, + test("cookieSameSite=none is matched case-insensitively too, and still requires Secure") { + val variants = List("none", "None", "NONE", " none ") + val emitted = variants.filter: v => + attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some(v))).contains("SameSite=None") + val leakedWithoutSecure = variants.filter: v => + attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(false), cookieSameSite = Some(v))).exists(_.contains("SameSite")) + assertTrue( + emitted == variants, + leakedWithoutSecure.isEmpty + ) ?? s"emitted=$emitted leakedWithoutSecure=$leakedWithoutSecure" + }, + test("an unknown or blank cookieSameSite value emits no SameSite attribute") { + // Unrecognised values stay omitted (a warning is logged) - the fix widened what counts + // as recognised, it did not start emitting arbitrary operator input into the header. + val ignored = List("bogus", "", " ", "lax; Domain=evil.example", "same-site=lax") + val emitted = ignored.filter: v => + attributeListOf(pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some(v))).exists(_.contains("SameSite")) + assertTrue(emitted.isEmpty) ?? s"these emitted a SameSite attribute: ${emitted.mkString("|")}" + }, + test("a configured cookieSameSite never injects extra cookie attributes") { + // The value is interpolated straight into the Set-Cookie header, so an operator value + // carrying `;` must not be able to append attributes of its own. + val cookie = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some("lax; Domain=evil.example")) + assertTrue(!cookie.contains("evil.example")) + }, + test("a hostile authPluginClassName cannot truncate the credential or inject cookie attributes") { + // REGRESSION - the per-field encoding scheme covered authParams and the OAuth2 fields + // but wrote authPluginClassName (free text from the /pulsar-auth/add BODY - only the + // name path segment is charset-validated) raw into the header, and JSON does not escape + // `;`. The browser stores a cookie only up to the first `;`, so the stored credential + // was truncated JSON (the next request failed to parse - silent auth corruption) and + // `Domain=evil.example` became a REAL attribute of a server-emitted cookie. Worse, + // setCookieAndSuccess re-injects the operator-configured Default credentials into + // EVERY response, so one bad configured value rewrote the header for every user. + val hostile = auth( + "prod" -> AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = "org.x.Auth; Domain=evil.example", + authParams = "token:secret; Path=/pwned" + ) + ) + val cookie = pulsarAuthToCookie(hostile, publicBaseUrl = None, cookieSecure = Some(true), cookieSameSite = Some("strict")) + assertTrue( + // what the browser stores parses back to exactly what was written - no truncation + parsePulsarAuthCookie(Some(valueOf(cookie))) == Right(hostile), + // and the attribute list is exactly the server's own - no smuggled Domain/Path + attributeListOf(cookie) == List("Path=/", "HttpOnly", "Max-Age=31536000", "Secure", "SameSite=Strict") + ) ?? s"cookie=$cookie" + }, + test("hostile credential names cannot truncate the credential or inject cookie attributes") { + // Credential names are charset-validated on /pulsar-auth/add, but parsePulsarAuthCookie + // performs NO name validation - a hand-crafted cookie can carry any map keys and any + // `current`, and setCookieAndSuccess echoes them straight back into Set-Cookie. The + // writer must therefore never trust the names either. + val hostileName = "evil; Domain=evil.example" + val hostile = PulsarAuth( + current = Some(hostileName), + credentials = Map( + hostileName -> JwtCredentials(`type` = "jwt", token = "aaa.bbb.ccc"), + "Default" -> EmptyCredentials(`type` = "empty") + ) + ) + val cookie = pulsarAuthToCookie(hostile, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + assertTrue( + parsePulsarAuthCookie(Some(valueOf(cookie))) == Right(hostile), + attributeListOf(cookie) == List("Path=/", "HttpOnly", "Max-Age=31536000") + ) ?? s"cookie=$cookie" + }, + // ---- round-trip: what is written must read back identically ---- + test("an empty credential round-trips through the cookie") { + val cookie = pulsarAuthToCookie(empty, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(empty)) + }, + test("a jwt credential round-trips through the cookie") { + val jwt = auth("Default" -> JwtCredentials(`type` = "jwt", token = "aaa.bbb.ccc")) + val cookie = pulsarAuthToCookie(jwt, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(jwt)) + }, + test("an authParamsString credential round-trips, including + and % in authParams") { + // REGRESSION (fixed 2026-07-25) - pulsarAuthToCookie used to URL-encode ONLY the OAuth2 fields, but + // parsePulsarAuthCookie URL-DECODES the whole cookie value. So authParams (which carries + // passwords/tokens) is written raw and read decoded: `+` becomes a space and `%xx` is + // eaten. Asserting the correct behavior - the round trip must be lossless. + val creds = AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = "org.apache.pulsar.client.impl.auth.AuthenticationToken", + authParams = "token:a+b%2Fc" + ) + val a = auth("Default" -> creds) + val cookie = pulsarAuthToCookie(a, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(a)) + }, + test("an oauth2 credential round-trips, including a private key with + / and = in it") { + // The four oauth2 fields are URL-encoded individually while the READ decodes the whole + // cookie once, so this is the pairing most likely to drift. A base64 private key is the + // realistic payload: it is full of the exact characters URL coding rewrites. + val creds = OAuth2Credentials( + `type` = "oauth2", + issuerUrl = "https://issuer.example.com/oauth2/token?tenant=a b", + privateKey = "MIIEvQIBADAN+Bg/kqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQ==", + audience = Some("urn:dekaf:audience with spaces"), + scope = Some("read write+admin") + ) + val a = auth("Default" -> creds) + val cookie = pulsarAuthToCookie(a, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(a)) + }, + test("optional oauth2 fields round-trip as None") { + val creds = OAuth2Credentials( + `type` = "oauth2", + issuerUrl = "https://issuer.example.com", + privateKey = "key", + audience = None, + scope = None + ) + val a = auth("Default" -> creds) + val cookie = pulsarAuthToCookie(a, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(a)) + }, + test("several credentials round-trip together and `current` selects among them") { + // Every real deployment holds more than one entry; the map and the selected name have to + // survive together, or switching credentials silently reverts. + val a = PulsarAuth( + current = Some("staging"), + credentials = Map( + "Default" -> EmptyCredentials(`type` = "empty"), + "staging" -> JwtCredentials(`type` = "jwt", token = "aaa.bbb.ccc"), + "prod" -> AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = "org.apache.pulsar.client.impl.auth.AuthenticationToken", + authParams = "token:x+y%2Fz" + ) + ) + ) + val cookie = pulsarAuthToCookie(a, publicBaseUrl = None, cookieSecure = None, cookieSameSite = None) + val value = valueOf(cookie) + assertTrue(parsePulsarAuthCookie(Some(value)) == Right(a)) + }, + test("a credential whose type does not match its shape is rejected") { + // credentialsDecoder tries each decoder in turn and takes the first success, so a + // malformed entry must fail rather than silently decode as a weaker credential type. + val badJwt = """{"current":"Default","credentials":{"Default":{"type":"jwt","token":"not-a-jwt"}}}""" + val badIssuer = """{"current":"Default","credentials":{"Default":{"type":"oauth2","issuerUrl":"ftp://x","privateKey":"k"}}}""" + assertTrue( + parsePulsarAuthCookie(Some(badJwt)).isLeft, + parsePulsarAuthCookie(Some(badIssuer)).isLeft + ) + }, + test("a missing cookie yields the default auth rather than an error") { + assertTrue(parsePulsarAuthCookie(None).isRight) + }, + test("a malformed cookie yields a Left, not an exception") { + assertTrue(parsePulsarAuthCookie(Some("not json at all")).isLeft) + }, + test("malformed percent-encoding yields a Left, not a thrown IllegalArgumentException") { + // URLDecoder.decode throws on these; it used to run outside the Either, turning a + // hand-edited cookie into a server error instead of a 400. + val malformed = List("%", "%ZZ", "a%2", "%E0%A4%A") + val escaped = malformed.filter(c => scala.util.Try(parsePulsarAuthCookie(Some(c))).isFailure) + assertTrue(escaped.isEmpty) ?? s"these threw instead of returning Left: ${escaped.mkString(", ")}" + }, + test("every malformed cookie form is reported as a parse failure") { + val malformed = List("%", "%ZZ", "a%2", "not json at all", "{\"current\":}") + val notLeft = malformed.filterNot(c => scala.util.Try(parsePulsarAuthCookie(Some(c))).toOption.exists(_.isLeft)) + assertTrue(notLeft.isEmpty) ?? s"these did not yield Left: ${notLeft.mkString(", ")}" + }, + test("a legacy cookie (raw JSON, per-field-encoded values) still parses unchanged") { + // Until 2026-08-08 the writer emitted the JSON RAW and URL-encoded only the four OAuth2 + // fields and authParams; browsers hold such cookies for up to a year (Max-Age=31536000). + // The read path URL-decodes the whole value once and is deliberately UNCHANGED by the + // whole-value-encoding fix, so a legacy value - whose only %xx/+ sequences sit inside + // the fields the old writer encoded - must keep decoding to exactly what it held. + // The legacy value is constructed here BY HAND (replicating the old writer) because + // pulsarAuthToCookie itself now writes the new format. + import java.net.URLEncoder + import java.nio.charset.StandardCharsets.UTF_8 + import io.circe.syntax.* + def enc(s: String): String = URLEncoder.encode(s, UTF_8) + + val original = PulsarAuth( + current = Some("prod"), + credentials = Map( + "Default" -> EmptyCredentials(`type` = "empty"), + "jwt1" -> JwtCredentials(`type` = "jwt", token = "aaa.bbb.ccc"), + "prod" -> AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = "org.apache.pulsar.client.impl.auth.AuthenticationToken", + authParams = "token:a+b%2Fc" + ), + "oa" -> OAuth2Credentials( + `type` = "oauth2", + issuerUrl = "https://issuer.example.com/oauth2/token?tenant=a b", + privateKey = "MIIEvQIBADAN+Bg/kqhkiG9w0BAQ==", + audience = Some("urn:dekaf:audience with spaces"), + scope = Some("read write+admin") + ) + ) + ) + // the OLD writer: URL-encode selected fields, then serialize the JSON raw + val legacyValue = original + .copy(credentials = original.credentials.map((name, credentials) => + credentials match + case cr: OAuth2Credentials => ( + name, + cr.copy( + issuerUrl = enc(cr.issuerUrl), + privateKey = enc(cr.privateKey), + audience = cr.audience.map(enc), + scope = cr.scope.map(enc) + ) + ) + case cr: AuthParamsStringCredentials => (name, cr.copy(authParams = enc(cr.authParams))) + case other => (name, other) + )) + .asJson.noSpaces + assertTrue( + legacyValue.contains("""{"current":"prod""""), // it really is the raw-JSON legacy shape + parsePulsarAuthCookie(Some(legacyValue)) == Right(original) + ) + }, + test("a parse failure never logs any fragment of the cookie value") { + // REGRESSION - the WARN on a parse failure logged circe's message, and for malformed + // JSON that message (jawn's) embeds a quoted fragment of the raw INPUT at the failure + // offset: `expected : got '"sk3cr...' (line 1, column 69)` (verified against jawn + // 1.4, which quotes the next handful of characters). The input is the credential + // cookie, so a cookie corrupted inside a token (exactly what the old `;`-truncation + // produced) wrote a slice of that token into ordinary operational logs at WARN. + // The secret is gibberish ON PURPOSE: the leak oracle scans for 4-char windows of it, + // and a dictionary word (like "token") could match a legitimate foreign log line from + // a parallel suite sharing the "pulsar-auth" logger. + val secret = "sk3cr3t4t0k3nv4lu3x9y8z7w6v5" + // corruption point INSIDE the credential: the `:` after "token" is missing, so the + // parse fails right where the secret string begins and jawn quotes it + val cutInsideToken = s"""{"current":"Default","credentials":{"Default":{"type":"jwt","token" "$secret"}}}""" + // decode failures embed cookie content too (the cursor PATH carries the credential + // names, and circe 0.14 can embed the offending value) - cover that shape as well + val wrongType = s"""{"current":"Default","credentials":{"Default":{"type":"jwt","token":{"nested":"$secret"}}}}""" + + val (results, logged) = withCapturedPulsarAuthLogs { + (parsePulsarAuthCookie(Some(cutInsideToken)), parsePulsarAuthCookie(Some(wrongType))) + } + // any 4-char window of the secret counts as a leak - jawn embeds only a bounded + // fragment of the input, never the whole token + val fragments = (0 to secret.length - 4).map(i => secret.substring(i, i + 4)) + val leaked = logged.filter(msg => fragments.exists(msg.contains)) + assertTrue( + results._1.isLeft, + results._2.isLeft, + // control: the parse failure DID log (logging is synchronous on this thread, so our + // event is guaranteed captured; the shared logger may add foreign events, which the + // unique secret keeps out of the leak check) + logged.exists(_.contains("Unable to parse cookie")), + leaked.isEmpty + ) ?? s"leaked=${leaked.mkString("|")}" + } + ) diff --git a/server/src/test/scala/pulsar_auth/pulsarAuthRoutesHttpTest.scala b/server/src/test/scala/pulsar_auth/pulsarAuthRoutesHttpTest.scala new file mode 100644 index 000000000..fae1e391e --- /dev/null +++ b/server/src/test/scala/pulsar_auth/pulsarAuthRoutesHttpTest.scala @@ -0,0 +1,244 @@ +package pulsar_auth + +import zio.* +import zio.test.* + +import io.javalin.Javalin + +import java.net.URI +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import scala.jdk.OptionConverters.* + +/** The add/use/delete routes under `/pulsar-auth`, over real HTTP. + * + * `pulsarAuthCookieTest` covers `pulsarAuthToCookie`/`parsePulsarAuthCookie` as functions, but the + * routes are what a browser actually talks to, and everything between the function and the wire was + * untested: whether the configured hardening attributes reach `Set-Cookie` at all, whether a cookie + * value carrying `+`/`%` in `authParams` survives being handed to a client and sent back (the value + * is URL-encoded on write and URL-decoded on read, and it travels through Jetty's cookie parser in + * between), and whether a rejected request leaves the caller's current credential alone. + * + * Real Javalin on an ephemeral port and a real `java.net.http` client - no servlet stubs. The + * cookie config is passed to `routesWith` because the process-level `config` val loads once per JVM + * and cannot be varied. + */ +object pulsarAuthRoutesHttpTest extends ZIOSpecDefault: + + /** `+` and `%` are the two characters the encode/decode pairing gets wrong when it drifts, and + * `authParams` is where a token or password lives. */ + private val authParams = "token:a+b%2Fc" + + private val credentials = AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = "org.apache.pulsar.client.impl.auth.AuthenticationToken", + authParams = authParams + ) + + private val credentialsJson = + s"""{"type":"authParamsString","authPluginClassName":"${credentials.authPluginClassName}","authParams":"$authParams"}""" + + private val client = HttpClient.newHttpClient() + + private case class Response(status: Int, body: String, setCookie: Option[String]): + /** The cookie value a browser would store and send back. */ + def cookieHeader: String = "pulsar_auth=" + setCookie.getOrElse("").stripPrefix("pulsar_auth=").takeWhile(_ != ';') + + /** The attribute segment - everything the browser enforces. */ + def attributes: String = setCookie.getOrElse("").dropWhile(_ != ';') + + /** The attribute TOKENS, from splitting the whole header the way a browser does. Unlike + * `attributes.contains(...)`, asserting this list exactly surfaces an injected `;` in the + * value - the smuggled attribute (and the JSON tail) show up as extra elements. */ + def attributeList: List[String] = + setCookie.getOrElse("").split(";", -1).toList.map(_.trim).drop(1).filter(_.nonEmpty) + + def parsedAuth: Either[Throwable, PulsarAuth] = + parsePulsarAuthCookie(Some(cookieHeader.stripPrefix("pulsar_auth="))) + + private def post(base: String, path: String, body: String = "", cookie: Option[String] = None): Task[Response] = + ZIO.attemptBlocking { + val builder = HttpRequest.newBuilder(URI.create(s"$base$path")).POST(HttpRequest.BodyPublishers.ofString(body)) + cookie.foreach(c => builder.header("Cookie", c)) + val response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString()) + Response(response.statusCode, response.body, response.headers.firstValue("Set-Cookie").toScala) + } + + private def serve[A]( + publicBaseUrl: Option[String] = None, + cookieSecure: Option[Boolean] = None, + cookieSameSite: Option[String] = None + )(use: String => Task[A]): Task[A] = + ZIO.acquireReleaseWith( + ZIO.attemptBlocking { + Javalin + .create(config => config.showJavalinBanner = false) + .routes(PulsarAuthRoutes.routesWith(publicBaseUrl, cookieSecure, cookieSameSite)) + .start(0) + } + )(app => ZIO.attemptBlocking(app.stop()).ignore)(app => use(s"http://localhost:${app.port()}")) + + def spec = suite(this.getClass.toString)( + test("the configured Secure/SameSite attributes reach the Set-Cookie header") { + // The routes render the cookie themselves. A correct `pulsarAuthToCookie` proves nothing + // if the route calls it with the wrong inputs (or drops the header), and this credential + // is the session's Pulsar authentication - Secure/SameSite/HttpOnly are what stop it + // travelling in clear text or riding along on a cross-site request. + serve(publicBaseUrl = Some("http://gateway.example/dekaf"), cookieSecure = Some(true), cookieSameSite = Some("strict")) { base => + for response <- post(base, "/pulsar-auth/add/prod", credentialsJson) + yield assertTrue( + response.status == 200, + response.setCookie.isDefined, + response.attributes.contains("Secure"), + response.attributes.contains("SameSite=Strict"), + response.attributes.contains("HttpOnly"), + response.attributes.contains("Path=/dekaf"), + response.attributes.contains("Max-Age=31536000") + ) ?? s"Set-Cookie: ${response.setCookie}" + } + }, + test("SameSite=None is withheld over plain HTTP, where a browser would reject the whole cookie") { + serve(cookieSecure = Some(false), cookieSameSite = Some("none")) { base => + for response <- post(base, "/pulsar-auth/add/prod", credentialsJson) + yield assertTrue( + response.status == 200, + !response.attributes.contains("SameSite"), + !response.attributes.contains("Secure") + ) ?? s"Set-Cookie: ${response.setCookie}" + } + }, + test("a hostile authPluginClassName in the add body cannot inject Set-Cookie attributes or corrupt the credential") { + // REGRESSION (HIGH) - `authPluginClassName` arrives as free text in the add BODY (only + // the {credentialsName} path segment is charset-validated) and was interpolated raw + // into the Set-Cookie header; JSON does not escape `;`. So this body truncated the + // stored cookie at the `;` (every later request failed to parse - silent credential + // corruption) and `Domain=evil.example` became a real attribute of a server-emitted + // cookie. The whole JSON value is URL-encoded now; this drives the actual attack + // surface end to end: hostile body in, header out, cookie back in through real Jetty. + val hostileClassName = "org.x.Auth; Domain=evil.example" + val hostileJson = s"""{"type":"authParamsString","authPluginClassName":"$hostileClassName","authParams":"token:secret"}""" + serve() { base => + for + added <- post(base, "/pulsar-auth/add/prod", hostileJson) + reused <- post(base, "/pulsar-auth/use/prod", cookie = Some(added.cookieHeader)) + yield assertTrue( + added.status == 200, + // exactly the server's own attributes - in particular no Domain + added.attributeList == List("Path=/", "HttpOnly", "Max-Age=31536000"), + // the credential survives the full round trip un-truncated + reused.status == 200, + reused.parsedAuth.map(_.credentials.get("prod")) == Right(Some(AuthParamsStringCredentials( + `type` = "authParamsString", + authPluginClassName = hostileClassName, + authParams = "token:secret" + ))) + ) ?? s"added=${added.setCookie} reused=${reused.setCookie}" + } + }, + test("a credential with + and % in authParams survives a full HTTP cookie round trip") { + // Write path: the route URL-encodes authParams into the cookie. Read path: the whole + // cookie value is URL-decoded. If those two ever disagree again, `+` becomes a space and + // `%xx` is eaten - and the failure is a broken Pulsar connection, not a parse error. + // Sending the cookie back to /use/{name} is the tight oracle: that route answers 200 + // only if the server re-read the credential map out of the cookie it just issued. + serve() { base => + for + added <- post(base, "/pulsar-auth/add/prod", credentialsJson) + reused <- post(base, "/pulsar-auth/use/prod", cookie = Some(added.cookieHeader)) + yield assertTrue( + added.status == 200, + reused.status == 200, + added.parsedAuth.map(_.credentials.get("prod")) == Right(Some(credentials)), + reused.parsedAuth.map(_.credentials.get("prod")) == Right(Some(credentials)), + reused.parsedAuth.map(_.current) == Right(Some("prod")) + ) ?? s"added=${added.setCookie} reused=${reused.setCookie}" + } + }, + test("an unknown /use/{name} is a 404 that does not touch the current cookie") { + // REGRESSION - selecting a name that is not in the map used to succeed and write it into + // the cookie, after which every client construction failed and the interceptor answered + // UNAUTHENTICATED for every call: one request bricked the session. The route must both + // refuse AND leave the caller's cookie alone, so no Set-Cookie may be written at all. + serve() { base => + for + added <- post(base, "/pulsar-auth/add/prod", credentialsJson) + // control: a name that DOES exist still switches, so the assertions below are not vacuous + known <- post(base, "/pulsar-auth/use/Default", cookie = Some(added.cookieHeader)) + unknown <- post(base, "/pulsar-auth/use/nope", cookie = Some(added.cookieHeader)) + yield assertTrue( + known.status == 200, + known.parsedAuth.map(_.current) == Right(Some("Default")), + unknown.status == 404, + unknown.setCookie.isEmpty, + unknown.body.contains("nope") + ) ?? s"known=${known.status}/${known.setCookie} unknown=${unknown.status}/${unknown.setCookie}" + } + }, + test("deleting a credential that is not the current one leaves the selection alone") { + // REGRESSION - delete unconditionally reassigned + // `current = newCredentials.keys.headOption.orElse(Some("Default"))`, so removing an + // unrelated credential silently repointed the session at whatever key happened to come + // first in map iteration order. The user deletes a stale entry and their next Pulsar + // call runs under different (often Default/empty) credentials - an authorization change + // nobody asked for and nothing reports. + serve() { base => + for + aaa <- post(base, "/pulsar-auth/add/aaa", credentialsJson) + bbb <- post(base, "/pulsar-auth/add/bbb", credentialsJson, cookie = Some(aaa.cookieHeader)) + ccc <- post(base, "/pulsar-auth/add/ccc", credentialsJson, cookie = Some(bbb.cookieHeader)) + deleted <- post(base, "/pulsar-auth/delete/aaa", cookie = Some(ccc.cookieHeader)) + // oracle: the deleted name is really gone from the cookie the delete handed back + reuseDeleted <- post(base, "/pulsar-auth/use/aaa", cookie = Some(deleted.cookieHeader)) + reuseSurvivor <- post(base, "/pulsar-auth/use/bbb", cookie = Some(deleted.cookieHeader)) + yield assertTrue( + ccc.parsedAuth.map(_.current) == Right(Some("ccc")), + deleted.status == 200, + deleted.parsedAuth.map(_.current) == Right(Some("ccc")), + deleted.parsedAuth.map(_.credentials.keySet) == Right(Set("Default", "bbb", "ccc")), + reuseDeleted.status == 404, + reuseSurvivor.status == 200 + ) ?? s"beforeDelete=${ccc.setCookie} afterDelete=${deleted.setCookie}" + } + }, + test("deleting the current credential falls back to Default, never to a name that is gone") { + // The other half of the contract: `current` may only change when it names the credential + // actually removed, and it must then land on a credential that still exists. Default is + // always present (setCookieAndSuccess re-adds it on every response). + serve() { base => + for + aaa <- post(base, "/pulsar-auth/add/aaa", credentialsJson) + bbb <- post(base, "/pulsar-auth/add/bbb", credentialsJson, cookie = Some(aaa.cookieHeader)) + deleted <- post(base, "/pulsar-auth/delete/bbb", cookie = Some(bbb.cookieHeader)) + reuseDeleted <- post(base, "/pulsar-auth/use/bbb", cookie = Some(deleted.cookieHeader)) + yield assertTrue( + bbb.parsedAuth.map(_.current) == Right(Some("bbb")), // it really was current + deleted.status == 200, + deleted.parsedAuth.map(_.current) == Right(Some("Default")), + deleted.parsedAuth.map(_.credentials.keySet) == Right(Set("Default", "aaa")), + reuseDeleted.status == 404 + ) ?? s"beforeDelete=${bbb.setCookie} afterDelete=${deleted.setCookie}" + } + }, + test("deleting an unknown credential is a 404 that does not touch the current cookie") { + // Deleting `/nope` answered 200 AND rewrote `current` - a typo could switch the session's + // authentication. Refuse, and write no cookie at all. Deleting Default is a separate, + // pre-existing 400 and is pinned here so the new 404 branch cannot swallow it. + serve() { base => + for + aaa <- post(base, "/pulsar-auth/add/aaa", credentialsJson) + bbb <- post(base, "/pulsar-auth/add/bbb", credentialsJson, cookie = Some(aaa.cookieHeader)) + unknown <- post(base, "/pulsar-auth/delete/nope", cookie = Some(bbb.cookieHeader)) + default <- post(base, "/pulsar-auth/delete/Default", cookie = Some(bbb.cookieHeader)) + // control: a name that DOES exist is still deletable, so the refusals are not vacuous + known <- post(base, "/pulsar-auth/delete/aaa", cookie = Some(bbb.cookieHeader)) + yield assertTrue( + unknown.status == 404, + unknown.setCookie.isEmpty, + unknown.body.contains("nope"), + default.status == 400, + default.setCookie.isEmpty, + known.status == 200, + known.parsedAuth.map(_.credentials.keySet) == Right(Set("Default", "bbb")) + ) ?? s"unknown=${unknown.status}/${unknown.setCookie} default=${default.status} known=${known.status}" + } + } + ) @@ TestAspect.sequential diff --git a/server/src/test/scala/schema/protobufnative/compilerTest.scala b/server/src/test/scala/schema/protobufnative/compilerTest.scala index f12325ebb..545f4b444 100644 --- a/server/src/test/scala/schema/protobufnative/compilerTest.scala +++ b/server/src/test/scala/schema/protobufnative/compilerTest.scala @@ -23,11 +23,41 @@ object compilerTest extends ZIOSpecDefault: val fileEntry = FileEntry(relativePath, fileEntryContent) val compiledFiles = compiler.compileFiles(Seq(fileEntry)) - val file = compiledFiles.files.getOrElse(relativePath, Left("No such file")) match - case Right(f) => f - file.schemas.get("Person") match - case Some(schema) => - assertTrue(!schema.rawSchema.isEmpty) - assertTrue(schema.humanReadableSchema.contains("Person")) + // Every arm must FAIL the test rather than throw MatchError, and both conditions must + // be in ONE assertTrue - a second `assertTrue` statement is evaluated and discarded, + // so the rawSchema check used to be dead. + compiledFiles.files.getOrElse(relativePath, Left("No such file")) match + case Left(err) => assertNever(s"compilation failed for $relativePath: $err") + case Right(file) => + file.schemas.get("Person") match + case None => assertNever(s"no 'Person' schema; got: ${file.schemas.keys.mkString(", ")}") + case Some(schema) => + assertTrue( + !schema.rawSchema.isEmpty, + schema.humanReadableSchema.contains("Person"), + schema.humanReadableSchema.contains("person_name"), + schema.humanReadableSchema.contains("person_age") + ) + }, + test("reports a compilation error for a malformed proto file") { + val relativePath = "bad/file" + val compiledFiles = compiler.compileFiles(Seq(FileEntry(relativePath, "this is not a proto file"))) + + assertTrue(compiledFiles.files.get(relativePath).exists(_.isLeft)) + }, + test("a message absent from the file does not resolve") { + val relativePath = "a/b/c" + val fileEntryContent = + """syntax = "proto3"; + | + |message Person { + | string person_name = 1; + |} + |""".stripMargin + val compiledFiles = compiler.compileFiles(Seq(FileEntry(relativePath, fileEntryContent))) + + compiledFiles.files.getOrElse(relativePath, Left("No such file")) match + case Left(err) => assertNever(s"compilation failed: $err") + case Right(file) => assertTrue(file.schemas.get("Absent").isEmpty) } ) diff --git a/server/src/test/scala/server/grpc/statusCodeTest.scala b/server/src/test/scala/server/grpc/statusCodeTest.scala new file mode 100644 index 000000000..466f240cf --- /dev/null +++ b/server/src/test/scala/server/grpc/statusCodeTest.scala @@ -0,0 +1,69 @@ +package server.grpc + +import com.google.rpc.code.Code +import zio.test.* + +/** The number that actually goes on the wire for a `google.rpc.Status`. + * + * `google/rpc/code.proto` DECLARES ITS ENUM OUT OF NUMERIC ORDER - `UNAUTHENTICATED = 16` sits + * ninth, between `PERMISSION_DENIED = 7` and `RESOURCE_EXHAUSTED = 8`. ScalaPB's `.index` is the + * DECLARATION POSITION and `.value` is the proto number, so the two agree only for the first eight + * codes and diverge for every one after them: + * + * {{{ + * position: 8 UNAUTHENTICATED(16) 9 RESOURCE_EXHAUSTED(8) 10 FAILED_PRECONDITION(9) ... + * }}} + * + * Sending `.index` therefore mislabels every status from `UNAUTHENTICATED` onwards - a + * FAILED_PRECONDITION goes out as 10, which is ABORTED, and an INTERNAL goes out as 14, which is + * UNAVAILABLE. It stayed invisible because clients that only ask "is this OK?" get the right answer + * either way; it surfaces the moment one tests for a SPECIFIC non-OK code, as the Topic Positions + * tab does when it distinguishes "the session has not been started" from a real fault. + */ +object statusCodeTest extends ZIOSpecDefault: + + def spec = suite("google.rpc.Status codes")( + test("value is the PROTO NUMBER, which is what a client compares against") { + assertTrue(Code.OK.value == 0) && + assertTrue(Code.INVALID_ARGUMENT.value == 3) && + assertTrue(Code.NOT_FOUND.value == 5) && + assertTrue(Code.FAILED_PRECONDITION.value == 9) && + assertTrue(Code.INTERNAL.value == 13) && + assertTrue(Code.UNAUTHENTICATED.value == 16) + }, + test("index is the DECLARATION POSITION and DIVERGES past the first eight codes") { + // This is the trap, written down. If a future ScalaPB or a re-ordered code.proto ever + // makes these agree, this test fails and the warning above can be retired. + assertTrue(Code.FAILED_PRECONDITION.index == 10) && + assertTrue(Code.FAILED_PRECONDITION.index != Code.FAILED_PRECONDITION.value) && + assertTrue(Code.INTERNAL.index != Code.INTERNAL.value) && + assertTrue(Code.UNAUTHENTICATED.index != Code.UNAUTHENTICATED.value) && + // ...and agree for the first eight, which is why `.index` looked correct for years. + assertTrue(Code.OK.index == Code.OK.value) && + assertTrue(Code.INVALID_ARGUMENT.index == Code.INVALID_ARGUMENT.value) + }, + test("no service builds a Status out of .index any more") { + // The whole-tree guard: `.value` is correct for every code, `.index` only for the first + // eight, so the safe rule is that `.index` never reaches a Status at all. + import scala.jdk.CollectionConverters.* + val sources = java.nio.file.Files + .walk(java.nio.file.Paths.get("src/main/scala")) + .iterator + .asScala + .filter(p => p.toString.endsWith(".scala")) + .toVector + + val offenders = sources.flatMap { path => + java.nio.file.Files + .readAllLines(path) + .asScala + .zipWithIndex + .filter((line, _) => line.contains("Code.") && line.contains(".index")) + // A commented-out line is not code. + .filterNot((line, _) => line.trim.startsWith("//")) + .map((line, i) => s"${path.getFileName}:${i + 1}: ${line.trim}") + } + + assertTrue(offenders.isEmpty) || assertTrue(offenders.mkString("\n").isEmpty) + } + ) diff --git a/ui/build.js b/ui/build.js index ccf4586c9..bf70ae2c6 100644 --- a/ui/build.js +++ b/ui/build.js @@ -1,3 +1,4 @@ +const fs = require("fs"); const path = require("path"); const cssModulesPlugin = require("esbuild-css-modules-plugin"); @@ -16,6 +17,15 @@ const outdir = path.resolve( const isDevelopment = process.argv.includes("--dev"); const isWatch = process.argv.includes("--watch"); +// Monaco is not part of the bundle: @monaco-editor/react loads it at RUNTIME with its own AMD +// loader, whose default base is cdn.jsdelivr.net. That makes every code editor in Dekaf depend on +// the public internet - it never mounts at all without it. Ship the same files next to the bundle +// and point the loader at them (see loader.config in components/ui/CodeEditor/CodeEditor.tsx). +const copyMonaco = () => { + const from = path.resolve(__dirname, "node_modules", "monaco-editor", "min", "vs"); + fs.cpSync(from, path.join(outdir, "vs"), { recursive: true }); +}; + require("esbuild") .build({ target: ["chrome100"], @@ -43,4 +53,11 @@ require("esbuild") watch: isWatch, logLevel: "info", }) - .catch(() => process.exit(1)); + .then(copyMonaco) + .catch((err) => { + // Log before exiting: a bare `process.exit(1)` turns a bundle error or a missing + // `monaco-editor/min/vs` (the copyMonaco cpSync) into a silent failed exit with nothing to + // diagnose from. + console.error(err); + process.exit(1); + }); diff --git a/ui/components/TopicPage/TopicPage.test.tsx b/ui/components/TopicPage/TopicPage.test.tsx new file mode 100644 index 000000000..583e84240 --- /dev/null +++ b/ui/components/TopicPage/TopicPage.test.tsx @@ -0,0 +1,125 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * Which topic the page's children belong to. + * + * Pulsar allows `persistent://t/n/x` and `non-persistent://t/n/x` to exist at the same time: two + * different topics whose tenant, namespace and name are identical. The route carries the scheme, so + * navigating from one to the other changes NOTHING else about this page - and a child keyed only by + * tenant/namespace/name is not remounted, so it keeps consuming the topic the user just left. + * + * Only the transport is replaced; the page, its router and the consumer session are real. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); + +const mockClients = { current: undefined as unknown }; +jest.mock('../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +import React from 'react'; +import { act, render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import { HelmetProvider } from 'react-helmet-async'; +import { SWRConfig } from 'swr'; +import TopicPage from './TopicPage'; +import { Status } from '../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../grpc-web/google/rpc/code_pb'; + +const okStatus = () => { + const s = new Status(); + s.setCode(Code.OK); + s.setMessage(''); + return s; +}; + +const installClients = () => { + mockClients.current = { + topicServiceClient: { + getIsPartitionedTopic: () => + Promise.resolve({ + getStatus: () => okStatus(), + getIsPartitioned: () => false, + getPartitionsCount: () => undefined, + getActivePartitionsCount: () => undefined, + }), + }, + consumerServiceClient: { + createConsumer: () => new Promise(() => undefined), + resume: () => ({ on: () => undefined, removeListener: () => undefined, cancel: () => undefined }), + pause: () => new Promise(() => undefined), + deleteConsumer: () => Promise.resolve({ getStatus: () => okStatus() }), + resolveTopicSelector: () => Promise.reject(new Error('not used by these tests')), + }, + producerServiceClient: { + createProducer: () => Promise.resolve({ getStatus: () => okStatus() }), + deleteProducer: () => Promise.resolve({ getStatus: () => okStatus() }), + send: () => Promise.resolve({ getStatus: () => okStatus() }), + }, + libraryServiceClient: { + listLibraryItems: () => Promise.reject(new Error('no library in these tests')), + getLibraryItem: () => Promise.reject(new Error('no library in these tests')), + }, + }; +}; + +const page = (topicPersistency: 'persistent' | 'non-persistent') => ( + + + + + + + +); + +describe('navigating between the two topics that differ only in their scheme', () => { + it('starts a new consumer session rather than keeping the previous topic\'s one', async () => { + installClients(); + + let rerender: any; + await act(async () => { + ({ rerender } = render(page('persistent'))); + }); + + const before = screen.getByTestId('cs-session'); + // The session identifies itself; a remount produces a different element. + (before as any).__markedByThisTest = true; + + await act(async () => { + rerender(page('non-persistent')); + }); + + const after = screen.getByTestId('cs-session'); + expect((after as any).__markedByThisTest).toBeUndefined(); + }); + + it('keeps the same session while the topic does not change', async () => { + installClients(); + + let rerender: any; + await act(async () => { + ({ rerender } = render(page('persistent'))); + }); + + const before = screen.getByTestId('cs-session'); + (before as any).__markedByThisTest = true; + + await act(async () => { + rerender(page('persistent')); + }); + + expect((screen.getByTestId('cs-session') as any).__markedByThisTest).toBe(true); + }); +}); diff --git a/ui/components/TopicPage/TopicPage.tsx b/ui/components/TopicPage/TopicPage.tsx index ddd3e2fb6..33271a953 100644 --- a/ui/components/TopicPage/TopicPage.tsx +++ b/ui/components/TopicPage/TopicPage.tsx @@ -125,7 +125,10 @@ const TopicPage: React.FC = (props) => { extraCrumbs = extraCrumbs.concat([{ type: 'link', id: 'subscriptions', value: 'Subscriptions' }]); } - const key = `${props.tenant}-${props.namespace}-${props.topic}`; + // The persistency belongs in here: `persistent://t/n/x` and `non-persistent://t/n/x` can both + // exist, and they are different topics. Without the scheme, navigating between them left every + // child of this page mounted - the consumer session kept consuming the topic just left. + const key = `${props.topicPersistency}-${props.tenant}-${props.namespace}-${props.topic}`; let buttons: ToolbarButtonProps[] = [ { diff --git a/ui/components/app/contexts/Notifications.module.css b/ui/components/app/contexts/Notifications.module.css index 0c7e5514e..81009cb7f 100644 --- a/ui/components/app/contexts/Notifications.module.css +++ b/ui/components/app/contexts/Notifications.module.css @@ -1,4 +1,5 @@ .ToastContainer { + width: auto !important; max-width: calc(100vw - 24rem) !important; padding: 8rem !important; position: relative; @@ -14,6 +15,11 @@ } .Toast { + /* The container is width:auto, so every toast must declare its own column - a bare min-width + let long messages run the full viewport width. */ + width: 320rem !important; + max-width: calc(100vw - 48rem) !important; + margin-left: auto !important; font-family: "Inter" !important; border-radius: 8rem !important; box-shadow: 0rem 2rem 4rem rgb(0 0 0 / 27%) !important; @@ -33,4 +39,4 @@ bottom: 4rem; right: 2rem; display: inline-flex; -} \ No newline at end of file +} diff --git a/ui/components/app/contexts/Notifications.test.tsx b/ui/components/app/contexts/Notifications.test.tsx new file mode 100644 index 000000000..2d73f2bce --- /dev/null +++ b/ui/components/app/contexts/Notifications.test.tsx @@ -0,0 +1,28 @@ +/** + * @jest-environment jsdom + */ +import { defaultValue } from './Notifications'; + +// Four call sites do `const res = await call().catch(err => notifyError(...))` and then branch on +// `res === undefined`. That guard is only sound if the notifier genuinely returns undefined. +// +// These were once bare arrows returning `toast.*(...)`, i.e. react-toastify's Id. TypeScript allows a +// value-returning function where `=> void` is declared, so the compiler could not catch it, and the +// consequence showed up far away: `res` held a toast id, the undefined check passed straight through, +// and the next line threw "res.getStatus is not a function" - crashing the component instead of +// showing the error it was trying to report. +// +// Asserted directly because the end-to-end symptom is timing-dependent and an unreliable detector: +// with the bug reintroduced a full jest run still reported 217 passed while logging the TypeError. +describe('notification helpers return void', () => { + const notifiers = [ + ['notifySuccess', defaultValue.notifySuccess], + ['notifyInfo', defaultValue.notifyInfo], + ['notifyWarn', defaultValue.notifyWarn], + ['notifyError', defaultValue.notifyError], + ] as const; + + it.each(notifiers)('%s returns undefined so `res === undefined` guards hold', (_name, notify) => { + expect(notify('a message')).toBeUndefined(); + }); +}); diff --git a/ui/components/app/contexts/Notifications.tsx b/ui/components/app/contexts/Notifications.tsx index 2f3fee9d1..f3271d13b 100644 --- a/ui/components/app/contexts/Notifications.tsx +++ b/ui/components/app/contexts/Notifications.tsx @@ -8,6 +8,12 @@ import { copyToClipboard, copyFailureMessage } from '../clipboard'; export const toastContainerId = '__dekaf__toast-container'; +// These four are NEWS: they say a thing happened and get out of the way. A persistent, +// user-closed panel holding controls does not belong here - react-toastify's removal is +// animation-mediated and a toast created under a still-exiting id is silently dropped, so any +// dismiss-then-re-announce choreography either stacks two panels or swallows the new one (both +// happened to the consumer session's caught-up panel, 2026-08-11/12, e2e CS-DM-R2 and R3B). +// State a component owns is rendered by that component; see ReplayCaughtUpBanner's dock. export type Value = { notifySuccess: (content: ReactNode, notificationId?: string, isShort?: boolean) => void, notifyInfo: (content: ReactNode, notificationId?: string, isShort?: boolean) => void, @@ -40,11 +46,18 @@ const withCopyButton = (content: ReactNode) => { } const isShortTimeout = 100; -const defaultValue: Value = { - notifySuccess: (content, notificationId, isShort) => toast.success(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }), - notifyInfo: (content, notificationId, isShort) => toast.info(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }), - notifyWarn: (content, notificationId, isShort) => toast.warn(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }), - notifyError: (content, notificationId, isShort) => toast.error(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }), + +// Each body is braced so it genuinely returns undefined, matching the `=> void` above. +// As bare expressions these returned react-toastify's Id, and TypeScript permits a +// value-returning function where `void` is declared - so nothing caught it. Callers do +// `const res = await call().catch(err => notifyError(...))` and then test `res === undefined`; +// with an Id coming back that guard silently failed and the next line threw +// "res.getStatus is not a function", crashing the component instead of showing the error. +export const defaultValue: Value = { + notifySuccess: (content, notificationId, isShort) => { toast.success(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }); }, + notifyInfo: (content, notificationId, isShort) => { toast.info(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }); }, + notifyWarn: (content, notificationId, isShort) => { toast.warn(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }); }, + notifyError: (content, notificationId, isShort) => { toast.error(withCopyButton(content), { containerId: toastContainerId, toastId: notificationId || content?.toString(), autoClose: isShort ? isShortTimeout : undefined }); }, }; const Context = React.createContext(defaultValue); @@ -56,7 +69,11 @@ export const DefaultProvider = ({ children }: { children: ReactElement }) => { enableMultiContainer containerId={toastContainerId} position="top-right" - autoClose={5000} + // Doubled from 5s (2026-08-11): several of these carry a sentence worth of remediation - + // a broker setting to change, a mode to pick - and 5s was not long enough to finish + // reading one before it went. Hovering still pauses it, and the persistent kind (which + // never auto-closes) is unaffected. + autoClose={10000} newestOnTop={true} hideProgressBar={true} closeOnClick={true} diff --git a/ui/components/app/pulsar-auth/Editor/Editor.test.tsx b/ui/components/app/pulsar-auth/Editor/Editor.test.tsx new file mode 100644 index 000000000..93659526d --- /dev/null +++ b/ui/components/app/pulsar-auth/Editor/Editor.test.tsx @@ -0,0 +1,123 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The credential list's "Set as current" and "Delete" actions POST to /pulsar-auth/use|delete. The + * server now answers 404 for a name it does not know - e.g. a row the browser still shows after the + * store changed under it. Those handlers only guarded a REJECTED fetch (`.catch`); a resolved but + * non-OK response slipped through, so the click did nothing and said nothing. A non-OK status must + * surface, the way the Add flow (CredentialsEditor) already does. + */ +const mockClients = { current: undefined as unknown }; +jest.mock('../../contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +const mockNotifications = { current: undefined as unknown }; +jest.mock('../../../app/contexts/Notifications', () => ({ + useContext: () => mockNotifications.current, +})); + +const mockAppContext = { current: undefined as unknown }; +jest.mock('../../../app/contexts/AppContext', () => ({ + useContext: () => mockAppContext.current, +})); + +import React from 'react'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import Editor from './Editor'; +import { + GetMaskedCredentialsResponse, + GetCurrentCredentialsResponse, + MaskedCredentials, + CredentialsType, +} from '../../../../grpc-web/tools/teal/pulsar/ui/api/v1/pulsar_auth_pb'; +import { StringValue } from 'google-protobuf/google/protobuf/wrappers_pb'; + +const listResponse = () => { + const res = new GetMaskedCredentialsResponse(); + const cred = new MaskedCredentials(); + cred.setName('cred-a'); + cred.setType(CredentialsType.CREDENTIALS_TYPE_JWT); + res.setCredentialsList([cred]); + return res; +}; + +const currentResponse = () => { + const res = new GetCurrentCredentialsResponse(); + res.setName(new StringValue().setValue('Default')); + return res; +}; + +const notFound = () => + Promise.resolve({ ok: false, status: 404, text: () => Promise.resolve('unknown credentials name') } as unknown as Response); +const okResponse = () => + Promise.resolve({ ok: true, status: 200, text: () => Promise.resolve('') } as unknown as Response); + +const makeHarness = (fetchImpl: () => Promise) => { + const notifyError = jest.fn(); + mockClients.current = { + pulsarAuthServiceClient: { + getMaskedCredentials: () => Promise.resolve(listResponse()), + getCurrentCredentials: () => Promise.resolve(currentResponse()), + }, + }; + mockNotifications.current = { notifyError, notifySuccess: jest.fn(), notifyInfo: jest.fn(), notifyWarn: jest.fn() }; + mockAppContext.current = { config: { publicBaseUrl: '' } }; + (global as any).fetch = jest.fn(fetchImpl); + return { notifyError }; +}; + +const renderEditor = async () => { + await act(async () => { + render( + // A fresh cache per render, so the global `mutate` in the handlers never bleeds between tests. + new Map(), shouldRetryOnError: false, dedupingInterval: 0, revalidateOnFocus: false }}> + + + ); + }); +}; + +describe('a resolved non-OK response to a use/delete action is surfaced', () => { + it('surfaces a 404 from "Set as current"', async () => { + const { notifyError } = makeHarness(notFound); + await renderEditor(); + + const button = await screen.findByTestId('credentials-set-current'); + await act(async () => { + fireEvent.click(button); + }); + + await waitFor(() => expect(notifyError).toHaveBeenCalled()); + expect(String(notifyError.mock.calls[0][0])).toContain('404'); + }); + + it('surfaces a 404 from "Delete"', async () => { + const { notifyError } = makeHarness(notFound); + await renderEditor(); + + const button = await screen.findByTestId('credentials-delete'); + await act(async () => { + fireEvent.click(button); + }); + + await waitFor(() => expect(notifyError).toHaveBeenCalled()); + expect(String(notifyError.mock.calls[0][0])).toContain('404'); + }); + + it('says nothing when the action succeeds - the toast is for failures only', async () => { + // The counterpart: "surface non-OK" is trivially satisfiable by shouting on every click. + const { notifyError } = makeHarness(okResponse); + await renderEditor(); + + const button = await screen.findByTestId('credentials-set-current'); + await act(async () => { + fireEvent.click(button); + await Promise.resolve(); + }); + + expect(notifyError).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/components/app/pulsar-auth/Editor/Editor.tsx b/ui/components/app/pulsar-auth/Editor/Editor.tsx index ac15ea778..f3f6be9f4 100644 --- a/ui/components/app/pulsar-auth/Editor/Editor.tsx +++ b/ui/components/app/pulsar-auth/Editor/Editor.tsx @@ -96,8 +96,16 @@ const Editor: React.FC = (props) => { testId="credentials-set-current" type='regular' onClick={async () => { - await fetch(`${config.publicBaseUrl}/pulsar-auth/use/${encodeURIComponent(item.name)}`, { method: 'POST' }) - .catch((err) => notifyError(`Unable to set current credentials: ${err}`)); + const res = await fetch(`${config.publicBaseUrl}/pulsar-auth/use/${encodeURIComponent(item.name)}`, { method: 'POST' }) + .catch((err) => { + notifyError(`Unable to set current credentials: ${err}`); + return undefined; + }); + // A resolved response can still be an error (e.g. 404 for a name the + // server no longer knows); the .catch above only handles a rejected call. + if (res !== undefined && !res.ok) { + notifyError(`Unable to set current credentials. ${res.status}: ${await res.text()}`); + } await mutate(swrKeys.pulsar.auth.credentials._()); await mutate(swrKeys.pulsar.auth.credentials.current._()); }} @@ -107,8 +115,16 @@ const Editor: React.FC = (props) => { testId="credentials-delete" type='danger' onClick={async () => { - await fetch(`${config.publicBaseUrl}/pulsar-auth/delete/${encodeURIComponent(item.name)}`, { method: 'POST' }) - .catch((err) => notifyError(`Unable to delete credentials: ${err}`)); + const res = await fetch(`${config.publicBaseUrl}/pulsar-auth/delete/${encodeURIComponent(item.name)}`, { method: 'POST' }) + .catch((err) => { + notifyError(`Unable to delete credentials: ${err}`); + return undefined; + }); + // A resolved response can still be an error (e.g. 404 for a name the + // server no longer knows); the .catch above only handles a rejected call. + if (res !== undefined && !res.ok) { + notifyError(`Unable to delete credentials. ${res.status}: ${await res.text()}`); + } await mutate(swrKeys.pulsar.auth.credentials._()); await mutate(swrKeys.pulsar.auth.credentials.current._()); }} diff --git a/ui/components/conversions/conversions.spec.ts b/ui/components/conversions/conversions.spec.ts new file mode 100644 index 000000000..43728fc1e --- /dev/null +++ b/ui/components/conversions/conversions.spec.ts @@ -0,0 +1,75 @@ +import { hexStringFromByteArray, hexStringToByteArray } from "./conversions"; + +/** + * Regression: the shared hex parser used for binary/hex message input (message ids, "Start + * from" positions, the producer's `bytes-hex` value) accepted malformed input and silently produced + * bytes for it, because it fed every 2-char slice through `parseInt(_, 16)` and assigned the result + * into a `Uint8Array` - `NaN` becomes 0, a negative becomes its two's complement, and `parseInt` + * happily stops at the first non-hex character instead of failing. + * + * The `invalidCases` table below records the exact bytes each input produced BEFORE the fix. + */ +describe("hexStringToByteArray", () => { + const validCases: { name: string; input: string; bytes: number[] }[] = [ + { name: "empty string", input: "", bytes: [] }, + { name: "whitespace only", input: " ", bytes: [] }, + { name: "single byte", input: "a1", bytes: [0xa1] }, + { name: "packed bytes ('hex-no-space' rendering)", input: "a1b2d3", bytes: [0xa1, 0xb2, 0xd3] }, + { name: "space separated bytes ('hex-with-space' rendering)", input: "a1 b2 d3", bytes: [0xa1, 0xb2, 0xd3] }, + { name: "upper case digits", input: "A1B2", bytes: [0xa1, 0xb2] }, + { name: "surrounding whitespace", input: " a1b2 ", bytes: [0xa1, 0xb2] }, + { name: "newline separated bytes", input: "a1\nb2", bytes: [0xa1, 0xb2] }, + ]; + + it.each(validCases)("accepts $name", ({ input, bytes }) => { + expect(hexStringToByteArray(input)).toEqual(Uint8Array.from(bytes)); + }); + + it("round-trips both rendering styles produced by hexStringFromByteArray", () => { + const bytes = Uint8Array.from([0x00, 0x0f, 0xa1, 0xff]); + expect(hexStringToByteArray(hexStringFromByteArray(bytes, "hex-no-space"))).toEqual(bytes); + expect(hexStringToByteArray(hexStringFromByteArray(bytes, "hex-with-space"))).toEqual(bytes); + }); + + // `producedBefore` documents the corrupt output of the pre-fix parser for that exact input. + const invalidCases: { input: string; producedBefore: string }[] = [ + { input: "zz", producedBefore: "[0]" }, + { input: "gg", producedBefore: "[0]" }, + { input: "1g", producedBefore: "[1]" }, + { input: "g1", producedBefore: "[0]" }, + { input: "z1z2", producedBefore: "[0, 0]" }, + { input: "a1b2!!", producedBefore: "[161, 178, 0]" }, + { input: "0x", producedBefore: "[0]" }, + { input: "0xff", producedBefore: "[0, 255]" }, + { input: "-1", producedBefore: "[255]" }, + { input: "+1", producedBefore: "[1]" }, + { input: "Infinity", producedBefore: "[0, 15, 0, 0]" }, + // Whitespace was stripped everywhere, so a pair split across a space was silently regrouped. + { input: "a 1b 2", producedBefore: "[161, 178]" }, + ]; + + it.each(invalidCases)("rejects $input instead of silently producing $producedBefore", ({ input }) => { + expect(() => hexStringToByteArray(input)).toThrow(Error); + }); + + // Odd-length input was already rejected, but by throwing a bare string literal - so callers doing + // `catch (err) { err.message }` or `err instanceof Error` got nothing usable. + const oddLengthCases = ["a", "abc", "a1b"]; + + it.each(oddLengthCases)("rejects odd-length %s with a real Error", (input) => { + expect(() => hexStringToByteArray(input)).toThrow(Error); + }); + + it("throws Error instances, never bare strings", () => { + for (const input of [...invalidCases.map((c) => c.input), ...oddLengthCases]) { + let thrown: unknown = undefined; + try { + hexStringToByteArray(input); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(Error); + expect(String((thrown as Error).message)).not.toHaveLength(0); + } + }); +}); diff --git a/ui/components/conversions/conversions.tsx b/ui/components/conversions/conversions.tsx index 31b1cbc35..35601061b 100644 --- a/ui/components/conversions/conversions.tsx +++ b/ui/components/conversions/conversions.tsx @@ -1,8 +1,23 @@ +const hexGroupRegExp = /^[0-9a-fA-F]+$/; + export function hexStringToByteArray(hexString: string): Uint8Array { - const normalizedHexString = hexString.replace(/\s/g, ''); - if (normalizedHexString.length % 2 !== 0) { - throw "Must have an even number of hex digits to convert to bytes"; + // Byte pairs may be separated by whitespace - that is how hexStringFromByteArray renders + // 'hex-with-space'. A pair itself must not be split though, so validate group by group instead of + // stripping all whitespace up front: otherwise "a 1b 2" silently regroups into different bytes. + const groups = hexString.split(/\s+/).filter(group => group.length > 0); + + for (const group of groups) { + if (!hexGroupRegExp.test(group)) { + // parseInt() would otherwise stop at the first non-hex character or yield NaN, and the value + // assigned into a Uint8Array would silently become some other byte. + throw new Error(`Invalid hex string: "${group}" is not a hex number.`); + } + if (group.length % 2 !== 0) { + throw new Error(`Invalid hex string: "${group}" must have an even number of hex digits to convert to bytes.`); + } } + + const normalizedHexString = groups.join(''); var numBytes = normalizedHexString.length / 2; var byteArray = new Uint8Array(numBytes); for (var i = 0; i < numBytes; i++) { diff --git a/ui/components/local-storage-keys.ts b/ui/components/local-storage-keys.ts index 272f70b77..51ff45de0 100644 --- a/ui/components/local-storage-keys.ts +++ b/ui/components/local-storage-keys.ts @@ -2,5 +2,25 @@ export const localStorageKeys = { messageExportConfig: "messageExportConfig", autoRefresh: "autoRefresh", defaultMessageFilterType: "defaultMessageFilterType", - isHidePartitionedTopics: "isHidePartitionedTopics" + isHidePartitionedTopics: "isHidePartitionedTopics", + /** + * Cap on messages per second a consumer session DELIVERS, browser-wide. 0 = unlimited. + * + * Rides each Resume request (like `include_consumer_stats`), NEVER the session config: the + * number belongs to the browser doing the watching, so it must not travel with a saved session + * into a library item. Applied when a session starts or resumes. + */ + consumerSessionRateLimit: "consumerSessionRateLimit", + /** + * Auto-pause the consumer session each time this many MORE messages have loaded. 0 = off. + * + * Purely client-side - the same Pause the toolbar button sends, triggered by the loaded + * counter - and browser-wide for the same reason as the rate limit above. Re-arms on every + * resume, so Play works as "load the next n". + */ + consumerSessionPauseAfterLoaded: "consumerSessionPauseAfterLoaded", + /** Whether the browser-wide consumer-session More tools panel is open. */ + consumerSessionToolsOpen: "consumerSessionToolsOpen", + /** The last selected More tools tab. Runtime validation handles tabs removed by newer builds. */ + consumerSessionToolsTab: "consumerSessionToolsTab" } as const; diff --git a/ui/components/ui/CodeEditor/CodeEditor.tsx b/ui/components/ui/CodeEditor/CodeEditor.tsx index 550c46f4c..77f52b7a2 100644 --- a/ui/components/ui/CodeEditor/CodeEditor.tsx +++ b/ui/components/ui/CodeEditor/CodeEditor.tsx @@ -5,6 +5,15 @@ import { IRange } from 'monaco-editor'; import s from './CodeEditor.module.css'; +// @monaco-editor/react fetches Monaco at runtime through its own AMD loader, and its default base +// is cdn.jsdelivr.net - so every code editor here depended on the public internet: seconds of +// third-party network before the first editor mounts, and no editor at all when jsdelivr is +// unreachable (offline, air-gapped, or blocked). Dekaf serves the identical files itself +// (ui/build.js copies monaco-editor/min/vs next to the bundle), so resolve them from our own +// origin. Absolute-ised against document.baseURI because the loader also builds worker URLs from +// this value, and those are not resolved against the page's . +loader.config({ paths: { vs: new URL('ui/static/dist/vs', document.baseURI).toString() } }); + export type Dependencies = { label: string, documentation: string, diff --git a/ui/components/ui/ConsumerSession/Console/Console.module.css b/ui/components/ui/ConsumerSession/Console/Console.module.css index de971912c..01c291591 100644 --- a/ui/components/ui/ConsumerSession/Console/Console.module.css +++ b/ui/components/ui/ConsumerSession/Console/Console.module.css @@ -2,8 +2,27 @@ display: flex; flex-direction: column; background-color: #fff; - overflow: auto; - border-top: 4px solid var(--border-color); + overflow: hidden; + border-top: none; + position: relative; +} + +/* The full handle stays in flow and hit-testable. Its centre line uses the shared handle's + thickness and replaces the old top border. */ +.Console .ResizeHandle::after { + background-color: var(--border-color); +} + +.Console .ResizeHandle:hover::after, +.Console .ResizeHandle:active::after { + background-color: var(--accent-color-blue); +} + +.CloseConsole { + position: absolute; + top: 9rem; + right: 8rem; + z-index: 60; } .SubscriptionsCursors { diff --git a/ui/components/ui/ConsumerSession/Console/Console.test.tsx b/ui/components/ui/ConsumerSession/Console/Console.test.tsx new file mode 100644 index 000000000..a2622322b --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/Console.test.tsx @@ -0,0 +1,121 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + */ +jest.mock('./Producer/Producer', () => ({ + __esModule: true, + default: () =>
Produce content
+})); +jest.mock('./TopicPositions/TopicPositions', () => ({ + __esModule: true, + default: (props: { isVisible: boolean }) => props.isVisible ?
: null +})); +jest.mock('./ContextRepl/ContextRepl', () => ({ + __esModule: true, + default: (props: { isVisible: boolean }) => props.isVisible ?
: null +})); +jest.mock('./ContextLogs/ContextLogs', () => ({ + __esModule: true, + default: (props: { isVisible: boolean }) => props.isVisible ?
: null +})); + +import React from 'react'; +import '@testing-library/jest-dom'; +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import Console from './Console'; +import { localStorageKeys } from '../../../local-storage-keys'; + +const topicContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'events' + } +}; + +const namespaceContext = { + pulsarResource: { + type: 'namespace' as const, + tenant: 'public', + namespace: 'default' + } +}; + +const props = (overrides: Record = {}) => ({ + isShow: true, + onClose: jest.fn(), + sessionKey: 0, + sessionSubscriptionName: 'subscription', + sessionConfig: {} as never, + sessionState: 'new' as const, + onSessionStateChange: jest.fn(), + messages: [], + consumerName: 'consumer', + currentTopic: 'persistent://public/default/events', + libraryContext: topicContext, + onResizeStart: jest.fn(), + ...overrides +}); + +describe('More tools interactions and selected-tab persistence', () => { + beforeEach(() => window.localStorage.clear()); + afterEach(() => { + cleanup(); + window.localStorage.clear(); + }); + + it('opens on Topic Positions, remembers an interactive tab selection, and restores it on remount', () => { + const first = render(); + expect(screen.getByTestId('mock-topic-positions')).toBeVisible(); + + fireEvent.click(screen.getByTestId('console-tab-logs')); + expect(screen.getByTestId('mock-logs')).toBeVisible(); + expect(window.localStorage.getItem(localStorageKeys.consumerSessionToolsTab)).toBe('"context-logs"'); + + first.unmount(); + render(); + expect(screen.getByTestId('mock-logs')).toBeVisible(); + }); + + it('falls back safely and heals storage when a saved tab no longer exists', async () => { + window.localStorage.setItem(localStorageKeys.consumerSessionToolsTab, JSON.stringify('removed-in-a-new-version')); + + render(); + + expect(screen.getByTestId('mock-topic-positions')).toBeVisible(); + await waitFor(() => { + expect(window.localStorage.getItem(localStorageKeys.consumerSessionToolsTab)).toBe('"topic-positions"'); + }); + }); + + it('temporarily falls back from Produce where that tab is unavailable without erasing the preference', () => { + window.localStorage.setItem(localStorageKeys.consumerSessionToolsTab, JSON.stringify('producer')); + + const first = render(); + expect(screen.getByTestId('mock-topic-positions')).toBeVisible(); + expect(window.localStorage.getItem(localStorageKeys.consumerSessionToolsTab)).toBe('"producer"'); + + first.unmount(); + render(); + expect(screen.getByTestId('mock-producer')).toBeVisible(); + }); + + it('closes from the top-right cross', () => { + const onClose = jest.fn(); + render(); + + fireEvent.click(screen.getByTestId('cs-tools-close')); + + expect(onClose).toHaveBeenCalledTimes(1); + }); + + it('does not expose the close or active debug content while the panel is closed', () => { + window.localStorage.setItem(localStorageKeys.consumerSessionToolsTab, JSON.stringify('context-logs')); + render(); + + expect(screen.queryByTestId('cs-tools-close')).toBeNull(); + expect(screen.queryByTestId('mock-logs')).toBeNull(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Console/Console.tsx b/ui/components/ui/ConsumerSession/Console/Console.tsx index 37ae22c76..f84674b8e 100644 --- a/ui/components/ui/ConsumerSession/Console/Console.tsx +++ b/ui/components/ui/ConsumerSession/Console/Console.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import useLocalStorage from 'use-local-storage-state'; import Producer from './Producer/Producer'; import { MessageDescriptor, ConsumerSessionConfig, SessionState } from '../types'; @@ -7,7 +8,11 @@ import Tabs, { Tab } from '../../Tabs/Tabs'; import s from './Console.module.css' import DebugLogs from './ContextLogs/ContextLogs'; import ExpressionInspector from './ContextRepl/ContextRepl'; +import TopicPositions from './TopicPositions/TopicPositions'; import { LibraryContext } from '../../LibraryBrowser/model/library-context'; +import PaneResizeHandle, { PaneResizeHandleKeyboardTarget } from '../../resizable/PaneResizeHandle'; +import ActionButton from '../../ActionButton/ActionButton'; +import { localStorageKeys } from '../../../local-storage-keys'; export type ConsoleProps = { isShow: boolean; @@ -21,20 +26,53 @@ export type ConsoleProps = { consumerName: string; currentTopic: string | undefined; libraryContext: LibraryContext; + onResizeStart: (startClientY: number, renderedHeight?: number) => void; + /** The panel's pane, for the resize handle's keyboard half (arrows/Home/End + ARIA value). */ + resizePane?: PaneResizeHandleKeyboardTarget; }; -type TabKey = 'producer' | 'visualize' | 'context-logs' | 'context-repl' | 'export'; +type TabKey = 'producer' | 'context-logs' | 'context-repl' | 'topic-positions'; -const Console: React.FC = (props) => { - const [activeTab, setActiveTab] = React.useState('producer'); +const defaultTab: TabKey = 'topic-positions'; - if (props.sessionConfig === undefined) { - return null; - } +const Console: React.FC = (props) => { + // Topic Positions leads and is the default: it is the tab that answers "where am I?", which + // is the first question on an open session - and unlike 'producer' it exists on EVERY page + // (the Produce tab only renders on topic pages, so a producer default pointed at a missing + // tab everywhere else). + // Read as `unknown` on purpose: localStorage outlives releases, so a tab removed by a future + // build can leave an arbitrary old key behind. Unknown keys fall back safely and are healed. + const [storedActiveTab, setStoredActiveTab] = useLocalStorage(localStorageKeys.consumerSessionToolsTab, { + defaultValue: defaultTab + }); + const hasProducerTab = props.libraryContext.pulsarResource.type === 'topic'; + // Assigned after the actual tab list is built below. Render callbacks close over this binding + // and run only after the assignment. + let activeTab: TabKey = defaultTab; - let tabs: Tab[] = []; + let tabs: Tab[] = [ + { + key: 'topic-positions', + title: 'Topic Positions', + testId: 'console-tab-topic-positions', + // Rendered always, like its siblings - but it is handed `isVisible` and polls nothing + // while hidden, which is what keeps a debug view off the broker for everyone not looking + // at it. + isRenderAlways: true, + render: () => ( + + ) + } + ]; - if (props.libraryContext.pulsarResource.type === 'topic') { + if (hasProducerTab) { tabs = tabs.concat([{ key: 'producer', title: 'Produce', @@ -58,6 +96,7 @@ const Console: React.FC = (props) => { } tabs = tabs.concat([ + { key: 'context-repl', title: 'Context REPL', @@ -67,7 +106,7 @@ const Console: React.FC = (props) => { ) }, @@ -80,17 +119,58 @@ const Console: React.FC = (props) => { ) } ]); + // Validate against the tabs that THIS BUILD actually rendered, not a second hard-coded list. + // Removing a tab therefore makes an old saved key fall back automatically. Produce is the one + // contextual exception: it is absent on namespace pages but may return on the next topic page, + // so keep that preference instead of erasing it there. + const storedTab = typeof storedActiveTab === 'string' ? storedActiveTab : undefined; + const availableStoredTab = tabs.find((tab) => tab.key === storedTab); + const isStoredTabAvailable = availableStoredTab !== undefined; + const isTemporarilyUnavailableProducer = storedTab === 'producer' && !hasProducerTab; + activeTab = availableStoredTab?.key ?? defaultTab; + + React.useEffect(() => { + if (!isStoredTabAvailable && !isTemporarilyUnavailableProducer) { + setStoredActiveTab(defaultTab); + } + }, [storedActiveTab, isStoredTabAvailable, isTemporarilyUnavailableProducer, setStoredActiveTab]); + + if (props.sessionConfig === undefined) { + return null; + } + return ( -
+
+ {props.isShow && ( + + )} + {props.isShow && ( +
+ +
+ )} activeTab={activeTab} - onActiveTabChange={setActiveTab} + onActiveTabChange={setStoredActiveTab} tabs={tabs} />
diff --git a/ui/components/ui/ConsumerSession/Console/ContextRepl/ContextRepl.tsx b/ui/components/ui/ConsumerSession/Console/ContextRepl/ContextRepl.tsx index 317cee038..9e6c264ea 100644 --- a/ui/components/ui/ConsumerSession/Console/ContextRepl/ContextRepl.tsx +++ b/ui/components/ui/ConsumerSession/Console/ContextRepl/ContextRepl.tsx @@ -78,7 +78,8 @@ const ExpressionInspector: React.FC = (props) => { >
- Run any JavaScript expression in the context of the session. Try libs 2 + 2 or lastMessage. + Run any JavaScript expression in the session context. Try libs, 2 + 2, or lastMessage + {' '}(the most recently processed message).
{!isConsumerCreated && (
diff --git a/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.spec.ts b/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.spec.ts index c56070ad7..3f0bf21c4 100644 --- a/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.spec.ts +++ b/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.spec.ts @@ -45,4 +45,38 @@ describe("valueToBytes", () => { ); } ); + + // The hex parser REFUSES malformed input rather than silently assigning some other byte - it + // throws. This function advertises an Either, and its caller checks `isRight` inside an async + // click handler with no try around it, so a throw here rejects the handler: no toast, no message + // sent, nothing on screen to say why. + const invalidHexTestCases: { hex: string; why: string }[] = [ + { hex: "zz", why: "not hex digits at all" }, + { hex: "a1 zz", why: "one bad group among good ones" }, + { hex: "a1b", why: "an odd number of digits is half a byte" }, + { hex: "0xa1", why: "a JavaScript literal is not a hex byte string" }, + { hex: "a1,b2", why: "only whitespace separates bytes" }, + ]; + + it.each(invalidHexTestCases)( + "should return an error, not throw, when the hex string is invalid ($why)", + ({ hex }) => { + expect(() => valueToBytes(hex, "bytes-hex")).not.toThrow(); + expect(Either.isLeft(valueToBytes(hex, "bytes-hex"))).toBe(true); + } + ); + + it("explains what was wrong with the hex it refused", () => { + const got = valueToBytes("a1b", "bytes-hex"); + + pipe( + got, + Either.match( + (err) => expect(String(err.message)).toMatch(/hex/i), + () => { + throw new Error("should return an error, but bytes where returned"); + } + ) + ); + }); }); diff --git a/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.ts b/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.ts index 439639eb0..f9756abd8 100644 --- a/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.ts +++ b/ui/components/ui/ConsumerSession/Console/Producer/lib/lib.ts @@ -21,8 +21,15 @@ export function valueToBytes(value: string, valueType: ValueType): Either.Either return Either.right(bytes); }; case 'bytes-hex': { - const bytes = hexStringToByteArray(value); - return Either.right(bytes); + // The parser REFUSES malformed hex by throwing rather than quietly writing some other byte. + // This function advertises an Either and its caller inspects it inside an async click + // handler with no try of its own, so an escaping throw rejected that handler: no message + // sent, and no toast either - the click simply did nothing. + try { + return Either.right(hexStringToByteArray(value)); + } catch (err) { + return Either.left(err as Error); + } }; } } diff --git a/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.module.css b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.module.css new file mode 100644 index 000000000..422d9424d --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.module.css @@ -0,0 +1,38 @@ +.TopicPositions { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; +} + +/* The table itself sits flush with the panel; only the text states carry padding. */ +.Empty, +.Error { + color: var(--text-color-secondary, #666); + padding: 8rem 12rem; +} + +.Error { + color: var(--error-color, #b00020); +} + +/* The table is wide by nature - fourteen columns of ids, timestamps, and progress - so it scrolls inside its + own box rather than making the console scroll sideways. The shared Table sizes itself with + flex (its scroll container is flex: 1), so this wrap must be a flex column with a definite + height - as a plain block the scroll container collapses to 0 and no rows render. */ +.TableWrap { + flex: 1; + min-height: 0; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.Id { + font-family: monospace; +} + +.Unavailable { + color: var(--text-color-secondary, #666); + font-style: italic; +} diff --git a/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.test.tsx b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.test.tsx new file mode 100644 index 000000000..7c03155a9 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.test.tsx @@ -0,0 +1,663 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The Topic Positions tab as a component, now built on the shared Table: what it polls, when it + * polls at all, and what it renders for each answer. The loader's TRANSITIONS (keep-last-good, + * error recovery, vanished session) are pinned in topic-positions.spec.ts against the extracted + * loader; here the wiring is pinned - gating by tab visibility and session state, the deadline + * on the RPC, the aggregate row reaching the screen. + */ +const mockClients = { current: undefined as unknown }; +jest.mock('../../../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current +})); + +const mockNotifications = { + current: { + notifySuccess: jest.fn(), + notifyInfo: jest.fn(), + notifyWarn: jest.fn(), + notifyError: jest.fn() + } +}; +jest.mock('../../../../app/contexts/Notifications', () => ({ + useContext: () => mockNotifications.current +})); + +const mockClipboardWriteText = jest.fn, [string]>(); + +// The production table virtualizes rows. jsdom has no layout, so render the same headers/cells in +// a plain table here; this lets the component suite pin which values reach the screen without +// pretending at viewport measurements. +jest.mock('react-virtuoso', () => { + const ReactRuntime = require('react'); + return { + TableVirtuoso: (props: any) => + ReactRuntime.createElement( + 'table', + null, + ReactRuntime.createElement('thead', null, props.fixedHeaderContent()), + ReactRuntime.createElement( + 'tbody', + null, + props.data.map((entry: any, index: number) => ReactRuntime.createElement('tr', { key: index }, props.itemContent(index, entry))) + ) + ) + }; +}); + +// react-virtuoso (inside the shared Table) measures itself with ResizeObserver, which jsdom does +// not provide. A no-op stand-in is enough: nothing here asserts on measured sizes. +class ResizeObserverStub { + observe(): void {} + unobserve(): void {} + disconnect(): void {} +} +(globalThis as { ResizeObserver?: unknown }).ResizeObserver = + (globalThis as { ResizeObserver?: unknown }).ResizeObserver ?? ResizeObserverStub; + +import React from 'react'; +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import TopicPositions from './TopicPositions'; +import { Code } from '../../../../../grpc-web/google/rpc/code_pb'; + +// With jest.mock in the file, esbuild-jest runs babel's hoisting over untyped JS, so imported +// bindings must not appear in type annotations here (the lifecycle suite documents the same +// constraint) - hence the inlined literal union instead of the imported SessionState. +type SessionStateLiteral = 'new' | 'initializing' | 'running' | 'pausing' | 'paused'; + +const statusOf = (code: number, message = '') => ({ + getCode: () => code, + getMessage: () => message +}); + +type FakePosition = { + getTopicFqn: () => string; + getFirstMessageId: () => { getValue_asU8: () => Uint8Array } | undefined; + getFirstPublishTime: () => { getValue: () => number } | undefined; + getLastMessageId: () => { getValue_asU8: () => Uint8Array } | undefined; + getLastPublishTime: () => { getValue: () => number } | undefined; + getFirstConsumedMessageId: () => { getValue_asU8: () => Uint8Array } | undefined; + getFirstConsumedPublishTime: () => { getValue: () => number } | undefined; + getCursorMessageId: () => { getValue_asU8: () => Uint8Array } | undefined; + getCursorPublishTime: () => { getValue: () => number } | undefined; + getCursorTimeFraction: () => { getValue: () => number } | undefined; + getCursorEntryFraction: () => { getValue: () => number } | undefined; + getCursorEntryOrdinal: () => { getValue: () => number } | undefined; + getRetainedEntries: () => { getValue: () => number } | undefined; + getUnavailableReason: () => { getValue: () => string } | undefined; +}; + +type PositionOverrides = { + firstId?: number[]; + first?: number; + lastId?: number[]; + last?: number; + firstConsumed?: number; + lastConsumed?: number; + firstConsumedId?: number[]; + lastConsumedId?: number[]; + ordinal?: number; + retained?: number; + timeFraction?: number; + entryFraction?: number; + unavailableReason?: string; +}; + +const position = (fqn: string, over: PositionOverrides = {}): FakePosition => ({ + getTopicFqn: () => fqn, + getFirstMessageId: () => + over.firstId === undefined ? undefined : { getValue_asU8: () => new Uint8Array(over.firstId as number[]) }, + getFirstPublishTime: () => (over.first === undefined ? undefined : { getValue: () => over.first as number }), + getLastMessageId: () => + over.lastId === undefined ? undefined : { getValue_asU8: () => new Uint8Array(over.lastId as number[]) }, + getLastPublishTime: () => (over.last === undefined ? undefined : { getValue: () => over.last as number }), + getFirstConsumedMessageId: () => + over.firstConsumedId === undefined ? undefined : { getValue_asU8: () => new Uint8Array(over.firstConsumedId as number[]) }, + getFirstConsumedPublishTime: () => + over.firstConsumed === undefined ? undefined : { getValue: () => over.firstConsumed as number }, + getCursorMessageId: () => + over.lastConsumedId === undefined ? undefined : { getValue_asU8: () => new Uint8Array(over.lastConsumedId as number[]) }, + getCursorPublishTime: () => + over.lastConsumed === undefined ? undefined : { getValue: () => over.lastConsumed as number }, + getCursorTimeFraction: () => + over.timeFraction === undefined ? undefined : { getValue: () => over.timeFraction as number }, + getCursorEntryFraction: () => + over.entryFraction === undefined ? undefined : { getValue: () => over.entryFraction as number }, + getCursorEntryOrdinal: () => (over.ordinal === undefined ? undefined : { getValue: () => over.ordinal as number }), + getRetainedEntries: () => (over.retained === undefined ? undefined : { getValue: () => over.retained as number }), + getUnavailableReason: () => + over.unavailableReason === undefined ? undefined : { getValue: () => over.unavailableReason as string } +}); + +const answer = (code: number, message: string, positions: FakePosition[]) => ({ + getStatus: () => statusOf(code, message), + getPositionsList: () => positions +}); + +const withClient = (getTopicPositions: jest.Mock) => { + mockClients.current = { consumerServiceClient: { getTopicPositions } }; + return getTopicPositions; +}; + +beforeAll(() => { + Object.defineProperty(window, 'isSecureContext', { value: true, configurable: true }); + Object.defineProperty(window.navigator, 'clipboard', { + configurable: true, + value: { writeText: (text: string) => mockClipboardWriteText(text) } + }); + Object.defineProperty(document, 'execCommand', { configurable: true, value: jest.fn(() => false) }); +}); + +beforeEach(() => { + mockClipboardWriteText.mockReset().mockResolvedValue(undefined); + Object.values(mockNotifications.current).forEach((notify) => notify.mockReset()); +}); + +afterEach(() => { + cleanup(); + window.localStorage.clear(); +}); + +const renderTab = async (sessionState: SessionStateLiteral, isVisible = true) => { + await act(async () => { + render( + // A FRESH SWR cache per test: the Table polls through useSWR, and a shared cache would leak + // one test's rows into the next. + new Map(), dedupingInterval: 0, revalidateOnFocus: false }}> + + + ); + }); +}; + +const renderedTopicOrder = (): string[] => + Array.from(document.querySelectorAll("[data-testid='topic-positions'] tbody tr td:first-child")) + .map((cell) => cell.textContent?.trim() ?? ''); + +const header = (columnKey: string): HTMLElement => + document.querySelector(`[data-testid='table-th'][data-column-key='${columnKey}']`) as HTMLElement; + +describe('when the tab may not poll', () => { + it('session not started: no RPC at all - nothing exists to ask about', async () => { + // The poll is simply not armed before Play, which is cheaper than polling into a server + // refusal and cannot leak an internal session name. + const rpc = withClient(jest.fn()); + await renderTab('new'); + + expect(screen.getByTestId('topic-positions-not-started')).toBeTruthy(); + expect(rpc).not.toHaveBeenCalled(); + }); + + it('hidden tab: no polling - only the tab on screen pays the broker cost', async () => { + const rpc = withClient(jest.fn()); + await renderTab('running', false); + + expect(rpc).not.toHaveBeenCalled(); + }); + + it('stops periodic requests while hidden and resumes only when visible again', async () => { + const rpc = withClient(jest.fn().mockResolvedValue(answer(Code.OK, '', [position('persistent://t/n/a')]))); + const cache = new Map(); + const swr = { provider: () => cache, dedupingInterval: 0, revalidateOnFocus: false }; + const tab = (isVisible: boolean) => ( + + + + ); + + let rerenderTab: (isVisible: boolean) => void = () => undefined; + await act(async () => { + const view = render(tab(true)); + rerenderTab = (isVisible) => view.rerender(tab(isVisible)); + }); + await waitFor(() => expect(rpc).toHaveBeenCalledTimes(1)); + + await act(async () => { + rerenderTab(false); + }); + const callsWhenHidden = rpc.mock.calls.length; + + // More than two refresh periods: an accidentally live interval would reliably fire here. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 2200)); + }); + expect(rpc).toHaveBeenCalledTimes(callsWhenHidden); + + await act(async () => { + rerenderTab(true); + }); + await waitFor(() => expect(rpc).toHaveBeenCalledTimes(callsWhenHidden + 1)); + }); +}); + +describe('a polling tab', () => { + it('renders every topic plus the ALL TOPICS aggregate, and asks with a DEADLINE', async () => { + const rpc = withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position('persistent://t/n/a', { first: 1000, last: 2000, lastConsumed: 1500, ordinal: 5, retained: 10 }), + position('persistent://t/n/b', { first: 1200, last: 2400, lastConsumed: 1300, ordinal: 2, retained: 10 }) + ]) + ) + ); + await renderTab('running'); + + // The small Virtuoso stand-in above renders the rows without layout. The accounting gives us + // one compact assertion that both topic rows plus the aggregate reached the shared Table. + // The count is split across elements, so match on the flattened text. + await waitFor(() => expect(document.body.textContent).toContain('3 of 3 topics')); + + // The deadline is what keeps one hung RPC from wedging the poll forever. + const opts = rpc.mock.calls[0][1]; + expect(opts?.deadline).toBeDefined(); + }); + + it('renders every cell in the All topics aggregate row in bold', async () => { + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position('persistent://t/n/a', { first: 1000, last: 2000, lastConsumed: 1500, ordinal: 5, retained: 10 }), + position('persistent://t/n/b', { first: 1200, last: 2400, lastConsumed: 1300, ordinal: 2, retained: 10 }) + ]) + ) + ); + await renderTab('running'); + + await waitFor(() => expect(renderedTopicOrder()[0]).toBe('All topics')); + const rows = Array.from(document.querySelectorAll("[data-testid='topic-positions'] tbody tr")); + const aggregateCells = Array.from(rows[0].querySelectorAll('td')); + expect(aggregateCells).toHaveLength(14); + aggregateCells.forEach((cell) => { + const content = cell.querySelector("[data-testid='topic-positions-aggregate-cell']"); + expect(content).not.toBeNull(); + expect(content?.style.fontWeight).toBe('var(--font-weight-bold)'); + }); + + expect(rows[1].querySelector("[data-testid='topic-positions-aggregate-cell']")).toBeNull(); + }); + + it('renders consumed bounds even when broker endpoint inspection is unavailable', async () => { + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position('non-persistent://t/n/live', { + firstConsumed: 1000, + lastConsumed: 2000, + firstConsumedId: [1, 2], + lastConsumedId: [3, 4], + unavailableReason: 'Endpoint inspection is unavailable' + }) + ]) + ) + ); + await renderTab('running'); + + await waitFor(() => expect(document.body.textContent).toContain('01 02')); + expect(document.body.textContent).toContain('03 04'); + expect(document.body.textContent).toContain(new Date(1000).toLocaleString()); + expect(document.body.textContent).toContain(new Date(2000).toLocaleString()); + expect(document.body.textContent).toContain('Endpoint inspection is unavailable'); + }); + + it('a session that vanished mid-run ends the view without a crash or a stale-data banner', async () => { + withClient(jest.fn().mockResolvedValue(answer(Code.FAILED_PRECONDITION, 'no such session', []))); + await renderTab('running'); + + // Not the stale-data banner: nothing on screen is stale, the session is simply over. What it + // stops doing - polling - is pinned under fake timers below. + await waitFor(() => expect(screen.getByTestId('topic-positions-session-gone')).toBeTruthy()); + expect(screen.queryByTestId('topic-positions-error')).toBeNull(); + expect(document.body.textContent).toContain('no such session'); + }); + + it('a transport failure surfaces as the stale-data banner, not a throw', async () => { + withClient(jest.fn().mockRejectedValue(new Error('connection refused'))); + await renderTab('running'); + + await waitFor(() => expect(screen.getByTestId('topic-positions-error')).toBeTruthy()); + expect(screen.getByTestId('topic-positions-error').textContent).toContain('connection refused'); + expect(screen.getByTestId('topic-positions-error').textContent).toContain('showing the last data'); + }); +}); + +/** + * A session the server no longer has is TERMINAL. The server answers FAILED_PRECONDITION for a + * deleted/ended session precisely so the browser stops asking; treating it as "no rows" left the + * tab scanning a session that does not exist, once a second, forever, behind a blank table that + * looked like a session with nothing to report. + * + * These use FAKE TIMERS and walk SEVERAL refresh periods: a single tick cannot tell a stopped + * poll from a slow one. The CONTROL case below is what makes the rest meaningful - it proves this + * harness really does drive the 1-second auto-refresh, so "no further calls" is evidence. + */ +describe('a session that vanished mid-run', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + cleanup(); + jest.useRealTimers(); + }); + + const renderPolling = async (rpc: jest.Mock) => { + withClient(rpc); + await act(async () => { + render( + new Map(), dedupingInterval: 0, revalidateOnFocus: false }}> + + + ); + }); + }; + + /** One auto-refresh period, plus the promise turns SWR needs to settle the answer it triggers. */ + const tick = async () => { + await act(async () => { + jest.advanceTimersByTime(1000); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + }; + + const gone = (message: string) => answer(Code.FAILED_PRECONDITION, message, []); + const ok = (...fqns: string[]) => answer(Code.OK, '', fqns.map((fqn) => position(fqn))); + + it('CONTROL: a live session is polled once per refresh period, tick after tick', async () => { + const rpc = jest.fn().mockResolvedValue(ok('persistent://t/n/a')); + await renderPolling(rpc); + + expect(rpc).toHaveBeenCalledTimes(1); + await tick(); + expect(rpc).toHaveBeenCalledTimes(2); + await tick(); + expect(rpc).toHaveBeenCalledTimes(3); + await tick(); + expect(rpc).toHaveBeenCalledTimes(4); + }); + + it('stops polling for good the moment the server says the session is gone', async () => { + const rpc = jest.fn().mockResolvedValue(gone('There is no consumer session named __dekaf_test')); + await renderPolling(rpc); + + expect(rpc).toHaveBeenCalledTimes(1); + for (let period = 0; period < 6; period += 1) { + await tick(); + } + expect(rpc).toHaveBeenCalledTimes(1); + }); + + it('polls a live session, then stops at the period the session disappears', async () => { + const rpc = jest + .fn() + .mockResolvedValueOnce(ok('persistent://t/n/a', 'persistent://t/n/b')) + .mockResolvedValueOnce(ok('persistent://t/n/a', 'persistent://t/n/b')) + .mockResolvedValue(gone('The consumer session has ended')); + await renderPolling(rpc); + + await tick(); + expect(rpc).toHaveBeenCalledTimes(2); + await tick(); + expect(rpc).toHaveBeenCalledTimes(3); + + for (let period = 0; period < 5; period += 1) { + await tick(); + } + expect(rpc).toHaveBeenCalledTimes(3); + }); + + it('says the session ended instead of showing a blank table', async () => { + const rpc = jest.fn().mockResolvedValue(gone('The consumer session has ended')); + await renderPolling(rpc); + await tick(); + + const ended = screen.getByTestId('topic-positions-session-gone'); + expect(ended.textContent).toContain('ended'); + expect(ended.textContent).toContain('The consumer session has ended'); + // Not a table with nothing in it, and not the stale-data banner either - nothing is stale, + // the session is over. + expect(screen.queryByTestId('topic-positions-table')).toBeNull(); + expect(screen.queryByTestId('topic-positions-error')).toBeNull(); + }); + + it('an OK answer with no topics is an EMPTY result, not a vanished session', async () => { + const rpc = jest.fn().mockResolvedValue(ok()); + await renderPolling(rpc); + + await tick(); + await tick(); + + expect(screen.queryByTestId('topic-positions-session-gone')).toBeNull(); + expect(screen.getByTestId('topic-positions-table')).toBeTruthy(); + expect(rpc).toHaveBeenCalledTimes(3); + }); + + it('a transport failure is NOT terminal - the poll keeps trying', async () => { + const rpc = jest.fn().mockRejectedValue(new Error('connection refused')); + await renderPolling(rpc); + + await tick(); + await tick(); + + expect(screen.queryByTestId('topic-positions-session-gone')).toBeNull(); + expect(rpc).toHaveBeenCalledTimes(3); + }); + + it('a session started again after the old one ended polls again', async () => { + // ConsumerSession keeps ONE generated consumer name for its whole lifetime, so Stop/Play + // reuses it. The terminal state has to clear on the way through 'new', or Topic Positions + // would stay dead for the rest of the page's life. + const rpc = jest.fn().mockResolvedValue(gone('The consumer session has ended')); + withClient(rpc); + const tab = (sessionState: string) => ( + new Map(), dedupingInterval: 0, revalidateOnFocus: false }}> + + + ); + + let rerenderTab: (sessionState: string) => void = () => undefined; + await act(async () => { + const view = render(tab('running')); + rerenderTab = (sessionState) => view.rerender(tab(sessionState)); + }); + await tick(); + expect(screen.getByTestId('topic-positions-session-gone')).toBeTruthy(); + const callsWhileGone = rpc.mock.calls.length; + + rpc.mockResolvedValue(ok('persistent://t/n/a')); + await act(async () => { + rerenderTab('new'); + }); + await act(async () => { + rerenderTab('running'); + }); + + expect(screen.queryByTestId('topic-positions-session-gone')).toBeNull(); + await tick(); + expect(rpc.mock.calls.length).toBeGreaterThan(callsWhileGone); + }); +}); + +describe('column UX', () => { + it('uses a scan-friendly default order, precise titles, and a real tooltip on every header', async () => { + withClient(jest.fn().mockResolvedValue(answer(Code.OK, '', [position('persistent://t/n/topic')]))); + await renderTab('running'); + await waitFor(() => expect(header('topic')).toBeTruthy()); + + const headers = Array.from(document.querySelectorAll("[data-testid='table-th']")); + expect(headers.map((h) => h.dataset.columnKey)).toEqual([ + 'topic', + 'entriesRead', + 'entriesLeft', + 'entryFraction', + 'behind', + 'timeFraction', + 'firstConsumedPublished', + 'firstConsumedMessage', + 'lastConsumedPublished', + 'cursorMessage', + 'firstPublished', + 'firstMessage', + 'lastPublished', + 'lastMessage' + ]); + // Jest's SVG transform renders the active sort arrow as the literal "test-file-stub". + expect(headers.map((h) => h.textContent?.replace(/test-file-stub/g, '').trim())).toEqual([ + 'Topic / partition', + 'Stored entry position', + 'Stored entries after position', + 'Stored entry position (%)', + 'Publish-time gap', + 'Publish-time position (%)', + 'Earliest processed publish time', + 'Earliest processed message ID', + 'Furthest processed publish time', + 'Furthest processed message ID', + 'Earliest stored publish time', + 'Earliest stored message ID', + 'Latest stored publish time', + 'Latest stored message ID' + ]); + + headers.forEach((h) => { + const help = h.querySelector('[data-tooltip-html]')?.dataset.tooltipHtml; + expect(help?.trim().length).toBeGreaterThan(20); + }); + expect(header('entriesRead').querySelector('[data-tooltip-html]')?.dataset.tooltipHtml).toContain( + 'not how many entries the session read' + ); + expect(header('behind').querySelector('[data-tooltip-html]')?.dataset.tooltipHtml).toContain( + 'not subscription backlog' + ); + + // Message IDs from independent topic logs have no honest shared ordering, so those headers + // must not pretend to sort their serialized protobuf bytes. + fireEvent.click(header('firstConsumedMessage')); + expect(header('firstConsumedMessage').dataset.sortDirection).toBeUndefined(); + expect(header('topic').dataset.sortDirection).toBe('asc'); + }); + + it('sorts topic partition suffixes naturally', async () => { + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position('persistent://t/n/topic-partition-10'), + position('persistent://t/n/topic-partition-2') + ]) + ) + ); + await renderTab('running'); + + await waitFor(() => + expect(renderedTopicOrder()).toEqual([ + 'All topics', + 'persistent://t/n/topic-partition-2', + 'persistent://t/n/topic-partition-10' + ]) + ); + }); + + it('keeps missing timestamps last in both directions, preserves equal rows, and pins the aggregate', async () => { + const p = (suffix: string) => `persistent://t/n/topic-partition-${suffix}`; + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position(p('10')), + position(p('2'), { lastConsumed: 2000 }), + position(p('3'), { lastConsumed: 2000 }), + position(p('1'), { lastConsumed: 1000 }) + ]) + ) + ); + await renderTab('running'); + await waitFor(() => expect(renderedTopicOrder()).toEqual(['All topics', p('1'), p('2'), p('3'), p('10')])); + + fireEvent.click(header('lastConsumedPublished')); + await waitFor(() => expect(renderedTopicOrder()).toEqual(['All topics', p('1'), p('2'), p('3'), p('10')])); + + fireEvent.click(header('lastConsumedPublished')); + await waitFor(() => expect(renderedTopicOrder()).toEqual(['All topics', p('2'), p('3'), p('1'), p('10')])); + }); + + it('treats an incomplete retained-entry pair as missing in both sort directions', async () => { + const p = (suffix: string) => `persistent://t/n/topic-partition-${suffix}`; + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position(p('1'), { ordinal: 9 }), + position(p('2'), { ordinal: 2, retained: 10 }), + position(p('3'), { ordinal: 1, retained: 10 }) + ]) + ) + ); + await renderTab('running'); + await waitFor(() => expect(renderedTopicOrder()).toHaveLength(4)); + + fireEvent.click(header('entriesRead')); + await waitFor(() => expect(renderedTopicOrder()).toEqual(['All topics', p('3'), p('2'), p('1')])); + + fireEvent.click(header('entriesRead')); + await waitFor(() => expect(renderedTopicOrder()).toEqual(['All topics', p('2'), p('3'), p('1')])); + }); + + it('copies exact raw values while displaying readable values, and never makes a dash copyable', async () => { + const topicFqn = 'persistent://t/n/copy-me'; + withClient( + jest.fn().mockResolvedValue( + answer(Code.OK, '', [ + position(topicFqn, { + firstConsumed: 1000, + lastConsumed: 2000, + firstConsumedId: [1, 10], + lastConsumedId: [2, 11], + ordinal: 4, + retained: 10, + entryFraction: 0.4 + }) + ]) + ) + ); + await renderTab('running'); + await waitFor(() => expect(screen.getByRole('button', { name: 'Copy Earliest processed message ID' })).toBeTruthy()); + + const idCell = screen.getByRole('button', { name: 'Copy Earliest processed message ID' }); + expect(idCell.textContent).toBe('01 0a'); + expect(idCell.getAttribute('title')).toBe('010a'); + fireEvent.click(idCell); + + const timeCell = screen.getByRole('button', { name: 'Copy Earliest processed publish time' }); + expect(timeCell.textContent).toBe(new Date(1000).toLocaleString()); + expect(timeCell.getAttribute('title')).toBe(new Date(1000).toISOString()); + fireEvent.click(timeCell); + + fireEvent.click(screen.getByRole('button', { name: 'Copy Stored entry position' })); + fireEvent.click(screen.getByRole('button', { name: 'Copy Stored entry position (%)' })); + + await waitFor(() => + expect(mockClipboardWriteText.mock.calls.map(([value]) => value)).toEqual([ + '010a', + new Date(1000).toISOString(), + '4 / 10', + '0.4' + ]) + ); + expect(mockNotifications.current.notifySuccess).toHaveBeenCalledTimes(4); + expect(screen.queryByRole('button', { name: 'Copy Earliest stored message ID' })).toBeNull(); + }); + + it('reports a clipboard failure instead of claiming the value was copied', async () => { + withClient(jest.fn().mockResolvedValue(answer(Code.OK, '', [position('persistent://t/n/copy-failure')]))); + await renderTab('running'); + await waitFor(() => expect(screen.getByRole('button', { name: 'Copy Topic / partition' })).toBeTruthy()); + + mockClipboardWriteText.mockRejectedValueOnce(new Error('permission denied')); + fireEvent.click(screen.getByRole('button', { name: 'Copy Topic / partition' })); + + await waitFor(() => expect(mockNotifications.current.notifyWarn).toHaveBeenCalledTimes(1)); + expect(mockNotifications.current.notifySuccess).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.tsx b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.tsx new file mode 100644 index 000000000..748009546 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/TopicPositions/TopicPositions.tsx @@ -0,0 +1,499 @@ +import React from 'react'; + +import * as GrpcClient from '../../../../app/contexts/GrpcClient/GrpcClient'; +import * as pb from '../../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; +import { Code } from '../../../../../grpc-web/google/rpc/code_pb'; +import { createDeadline } from '../../../../../proto-utils/proto-utils'; +import { SessionState } from '../../types'; +import Table, { Columns, ColumnsConfig } from '../../../Table/Table'; +import Field from '../../Message/Field/Field'; +import NothingToShow from '../../../NothingToShow/NothingToShow'; +import { + TopicPositionRow, + behindMsOf, + entriesAfterCursorOf, + formatDurationMs, + formatEntryCount, + formatFraction, + formatTimestamp, + makePositionsLoader, + timestampIso, + topicPositionFromPb +} from './topic-positions'; +import s from './TopicPositions.module.css'; + +export type TopicPositionsProps = { + consumerName: string; + sessionState: SessionState; + /** The tab is mounted while hidden, so it has to be told - polling only runs on screen. */ + isVisible: boolean; +}; + +type ColumnKey = + | 'topic' + | 'firstMessage' + | 'firstPublished' + | 'lastMessage' + | 'lastPublished' + | 'firstConsumedMessage' + | 'firstConsumedPublished' + // Keep the original persisted column key: this cursor is the last-consumed high-water mark. + | 'cursorMessage' + | 'lastConsumedPublished' + | 'behind' + | 'timeFraction' + | 'entryFraction' + | 'entriesRead' + | 'entriesLeft'; + +/** Comparators here are plain and direction-blind. The Table applies direction, missing-last, and + * aggregate pinning as separate ordering rules so none of them can accidentally invert another. */ +const byRow = ( + compare: (a: TopicPositionRow, b: TopicPositionRow) => number +) => (a: { data: TopicPositionRow }, b: { data: TopicPositionRow }): number => compare(a.data, b.data); + +const compareNumbers = (a: number | undefined, b: number | undefined): number => { + if (a === undefined && b === undefined) { + return 0; + } + // Missing values are partitioned by Column.isSortValueMissing before this comparator runs. The + // fallback still makes the function total if it is reused without that predicate. + if (a === undefined) { + return 1; + } + if (b === undefined) { + return -1; + } + return a - b; +}; + +const isAggregateRow = (row: TopicPositionRow): boolean => row.isAggregate === true; + +const naturalTopicCompare = (a: string, b: string): number => + a.localeCompare(b, 'en', { numeric: true, sensitivity: 'base' }) || a.localeCompare(b); + +const isKnownNumber = (value: number | undefined): value is number => + value !== undefined && Number.isFinite(value); + +const missingNumber = ( + valueOf: (row: TopicPositionRow) => number | undefined +) => (entry: { data: TopicPositionRow }): boolean => !isKnownNumber(valueOf(entry.data)); + +type CellValue = string | React.ReactElement | undefined; + +const aggregateCellStyle: React.CSSProperties = { + fontWeight: 'var(--font-weight-bold)' +}; + +/** Reuse the consumer message table's copy interaction and raw/display conventions. Topic + * Positions refreshes every second, so its cells deliberately use the native raw-value title + * instead of registering thousands of changing react-tooltip anchors. */ +const copyableCell = ( + isAggregate: boolean, + title: string, + value: CellValue, + rawValue: string | undefined, + copyLabel?: string +): React.ReactElement => { + const field = ( + + ); + + return isAggregate ? ( +
+ {field} +
+ ) : field; +}; + +const topicCell = (row: TopicPositionRow): React.ReactElement => + copyableCell( + isAggregateRow(row), + 'Topic / partition', + row.topicFqn, + row.topicFqn, + // The column heading is "Topic / partition", but what lands on the clipboard is one fully + // qualified name - so the toast names that, not the heading. + 'Topic FQN' + ); + +const messageIdCell = (isAggregate: boolean, title: string, value: string | undefined): React.ReactElement => + copyableCell(isAggregate, title, value === undefined ? undefined : {value}, value?.replace(/\s/g, '')); + +const timestampCell = (isAggregate: boolean, title: string, value: number | undefined): React.ReactElement => + copyableCell(isAggregate, title, isKnownNumber(value) ? formatTimestamp(value) : undefined, timestampIso(value)); + +const numberCell = (isAggregate: boolean, title: string, display: string, value: number | undefined): React.ReactElement => + copyableCell(isAggregate, title, isKnownNumber(value) ? display : undefined, isKnownNumber(value) ? String(value) : undefined); + +const TopicPositions: React.FC = (props) => { + const { consumerServiceClient } = GrpcClient.useContext(); + // The loader reports here instead of throwing (the Table would toast on every render). When + // rows are still shown, they are the LAST GOOD ones - the banner says so. + const [loadError, setLoadError] = React.useState(undefined); + + // Polling is gated by exactly two things: the TAB being the one on screen, and the session + // existing at all. Opening the tab IS the request - the per-partition broker cost only runs + // while somebody is looking, and the Table's own auto-refresh toggle is the explicit freeze + // for anyone who wants the numbers to hold still. (A "capture" checkbox used to gate this too; + // it predated the shared-Table rework, guarded nothing the tab-gate does not - the session + // records its read position unconditionally either way - and its name wrongly implied the + // HISTORY started when it was ticked.) + const isSessionStarted = props.sessionState !== 'new' && props.sessionState !== 'initializing'; + + // A THIRD gate, and the only terminal one: the server answered FAILED_PRECONDITION, meaning the + // session it was asked about is gone. Polling has to STOP - the old behavior read that answer as + // "no rows" and went on scanning a session that does not exist, once a second, for as long as + // the tab stayed open, behind a blank table indistinguishable from a session with no topics. + const [sessionGone, setSessionGone] = React.useState<{ message?: string } | undefined>(undefined); + + // Cleared wherever a NEW session can begin. ConsumerSession keeps one generated consumer name + // for its whole lifetime, so Stop/Play reuses it: without the reset on the way through 'new' + // (isSessionStarted false), the tab would stay dead for the rest of the page's life. + React.useEffect(() => { + setSessionGone(undefined); + }, [props.consumerName, isSessionStarted]); + + const isPolling = props.isVisible && isSessionStarted && sessionGone === undefined; + + // One loader per consumer name, so its last-good memory dies with the session it belongs to. + const loader = React.useMemo( + () => + makePositionsLoader({ + okCode: Code.OK, + failedPreconditionCode: Code.FAILED_PRECONDITION, + onError: setLoadError, + onSessionGone: (message) => setSessionGone({ message }), + fetch: async () => { + const req = new pb.GetTopicPositionsRequest(); + req.setConsumerName(props.consumerName); + // The deadline is what keeps ONE hung RPC from wedging the poll forever: without it, + // a request that never settles left `inFlight` latched and every later refresh skipped. + const res = await consumerServiceClient.getTopicPositions(req, { deadline: createDeadline(8) }); + return { + code: res.getStatus()?.getCode(), + message: res.getStatus()?.getMessage(), + rows: res.getPositionsList().map(topicPositionFromPb) + }; + } + }), + [props.consumerName] + ); + + const columns: Columns = React.useMemo(() => { + // Put progress in the first viewport, followed by the consumed bounds and retained-log + // context. Exact entry figures lead the approximate producer-clock figures. IDs sit beside + // their timestamps but after them: they are exact and copyable, yet harder to scan. Any + // non-sticky column can still be dragged elsewhere. + const defaultConfig: ColumnsConfig = [ + { columnKey: 'topic', visibility: 'visible', width: 390, stickyTo: 'left' }, + { columnKey: 'entriesRead', visibility: 'visible', width: 170 }, + { columnKey: 'entriesLeft', visibility: 'visible', width: 150 }, + { columnKey: 'entryFraction', visibility: 'visible', width: 190 }, + { columnKey: 'behind', visibility: 'visible', width: 130 }, + { columnKey: 'timeFraction', visibility: 'visible', width: 200 }, + { columnKey: 'firstConsumedPublished', visibility: 'visible', width: 190 }, + { columnKey: 'firstConsumedMessage', visibility: 'visible', width: 160 }, + { columnKey: 'lastConsumedPublished', visibility: 'visible', width: 190 }, + { columnKey: 'cursorMessage', visibility: 'visible', width: 160 }, + { columnKey: 'firstPublished', visibility: 'visible', width: 190 }, + { columnKey: 'firstMessage', visibility: 'visible', width: 160 }, + { columnKey: 'lastPublished', visibility: 'visible', width: 190 }, + { columnKey: 'lastMessage', visibility: 'visible', width: 160 } + ]; + + const retainedValue = (row: TopicPositionRow, value: number | undefined): number | undefined => + row.unavailableReason === undefined ? value : undefined; + + return { + defaultConfig, + columns: { + topic: { + title: 'Topic / partition', + render: topicCell, + sortFn: byRow((a, b) => naturalTopicCompare(a.topicFqn, b.topicFqn)) + }, + firstMessage: { + title: 'Earliest stored message ID', + // Serialized IDs do not have a meaningful cross-topic ordering. Keep this exact value + // copyable, but do not offer a misleading lexicographic sort. + render: (row) => + row.unavailableReason === undefined + ? messageIdCell(isAggregateRow(row), 'Earliest stored message ID', row.firstMessageId) + : copyableCell( + isAggregateRow(row), + 'Unavailable reason', + {row.unavailableReason}, + row.unavailableReason + ) + }, + firstPublished: { + title: 'Earliest stored publish time', + render: (row) => timestampCell(isAggregateRow(row), 'Earliest stored publish time', retainedValue(row, row.firstPublishTime)), + sortFn: byRow((a, b) => compareNumbers(retainedValue(a, a.firstPublishTime), retainedValue(b, b.firstPublishTime))), + isSortValueMissing: missingNumber((row) => retainedValue(row, row.firstPublishTime)) + }, + lastMessage: { + title: 'Latest stored message ID', + render: (row) => + messageIdCell( + isAggregateRow(row), + 'Latest stored message ID', + row.unavailableReason === undefined ? row.lastMessageId : undefined + ) + }, + lastPublished: { + title: 'Latest stored publish time', + render: (row) => timestampCell(isAggregateRow(row), 'Latest stored publish time', retainedValue(row, row.lastPublishTime)), + sortFn: byRow((a, b) => compareNumbers(retainedValue(a, a.lastPublishTime), retainedValue(b, b.lastPublishTime))), + isSortValueMissing: missingNumber((row) => retainedValue(row, row.lastPublishTime)) + }, + firstConsumedMessage: { + title: 'Earliest processed message ID', + // Processed bounds are session-local. They stay valid even when the broker cannot expose + // stored endpoints (for example, a non-persistent topic). + render: (row) => messageIdCell(isAggregateRow(row), 'Earliest processed message ID', row.firstConsumedMessageId) + }, + firstConsumedPublished: { + title: 'Earliest processed publish time', + render: (row) => timestampCell(isAggregateRow(row), 'Earliest processed publish time', row.firstConsumedPublishTime), + sortFn: byRow((a, b) => compareNumbers(a.firstConsumedPublishTime, b.firstConsumedPublishTime)), + isSortValueMissing: missingNumber((row) => row.firstConsumedPublishTime) + }, + cursorMessage: { + title: 'Furthest processed message ID', + render: (row) => messageIdCell(isAggregateRow(row), 'Furthest processed message ID', row.lastConsumedMessageId) + }, + lastConsumedPublished: { + title: 'Furthest processed publish time', + render: (row) => timestampCell(isAggregateRow(row), 'Furthest processed publish time', row.lastConsumedPublishTime), + sortFn: byRow((a, b) => compareNumbers(a.lastConsumedPublishTime, b.lastConsumedPublishTime)), + isSortValueMissing: missingNumber((row) => row.lastConsumedPublishTime) + }, + behind: { + title: 'Publish-time gap', + render: (row) => { + const value = row.unavailableReason === undefined ? behindMsOf(row) : undefined; + return numberCell(isAggregateRow(row), 'Publish-time gap', formatDurationMs(value), value); + }, + sortFn: byRow((a, b) => + compareNumbers( + a.unavailableReason === undefined ? behindMsOf(a) : undefined, + b.unavailableReason === undefined ? behindMsOf(b) : undefined + ) + ), + isSortValueMissing: missingNumber((row) => row.unavailableReason === undefined ? behindMsOf(row) : undefined) + }, + timeFraction: { + title: 'Publish-time position (%)', + render: (row) => { + const value = retainedValue(row, row.cursorTimeFraction); + return numberCell(isAggregateRow(row), 'Publish-time position (%)', formatFraction(value), value); + }, + sortFn: byRow((a, b) => + compareNumbers(retainedValue(a, a.cursorTimeFraction), retainedValue(b, b.cursorTimeFraction)) + ), + isSortValueMissing: missingNumber((row) => retainedValue(row, row.cursorTimeFraction)) + }, + entryFraction: { + title: 'Stored entry position (%)', + render: (row) => { + const value = retainedValue(row, row.cursorEntryFraction); + return numberCell(isAggregateRow(row), 'Stored entry position (%)', formatFraction(value), value); + }, + sortFn: byRow((a, b) => + compareNumbers(retainedValue(a, a.cursorEntryFraction), retainedValue(b, b.cursorEntryFraction)) + ), + isSortValueMissing: missingNumber((row) => retainedValue(row, row.cursorEntryFraction)) + }, + entriesRead: { + title: 'Stored entry position', + render: (row) => { + const ordinal = retainedValue(row, row.cursorEntryOrdinal); + const retained = retainedValue(row, row.retainedEntries); + const raw = isKnownNumber(ordinal) && isKnownNumber(retained) ? `${ordinal} / ${retained}` : undefined; + return copyableCell( + isAggregateRow(row), + 'Stored entry position', + raw === undefined ? undefined : formatEntryCount(ordinal, retained), + raw + ); + }, + sortFn: byRow((a, b) => compareNumbers(a.cursorEntryOrdinal, b.cursorEntryOrdinal)), + isSortValueMissing: ({ data: row }) => + row.unavailableReason !== undefined || !isKnownNumber(row.cursorEntryOrdinal) || !isKnownNumber(row.retainedEntries) + }, + entriesLeft: { + title: 'Stored entries after position', + render: (row) => { + const value = row.unavailableReason === undefined ? entriesAfterCursorOf(row) : undefined; + return numberCell( + isAggregateRow(row), + 'Stored entries after position', + isKnownNumber(value) ? value.toLocaleString() : '', + value + ); + }, + sortFn: byRow((a, b) => + compareNumbers( + a.unavailableReason === undefined ? entriesAfterCursorOf(a) : undefined, + b.unavailableReason === undefined ? entriesAfterCursorOf(b) : undefined + ) + ), + isSortValueMissing: missingNumber((row) => row.unavailableReason === undefined ? entriesAfterCursorOf(row) : undefined) + } + }, + help: { + topic: ( + + A non-partitioned topic or one partition of a partitioned topic. “All topics” is a Dekaf session summary. + Numeric partition suffixes sort naturally, so partition-2 comes before partition-10. + + ), + firstConsumedMessage: ( + + The lowest message-ID position processed by this Dekaf session. It includes counted start-mode discards and + messages hidden by session filters; messages bypassed by a broker seek are not observed. Older redeliveries + can move it backward. Message IDs are not ordered across topics; “All topics” has no aggregate ID. + + ), + firstConsumedPublished: ( + + Pulsar publish time on “Earliest processed message ID”. The producer client sets it; it is not event time or + the time Dekaf received the message. “All topics” shows the earliest per-topic value. + + ), + cursorMessage: ( + + The highest message-ID position processed by this Dekaf session. Older redeliveries do not move it backward. + It is not a Pulsar subscription cursor or backlog. Message IDs are not ordered across topics; “All topics” + has no aggregate ID. + + ), + lastConsumedPublished: ( + + Pulsar publish time on “Furthest processed message ID”. It need not be the latest observed timestamp when a + producer clock moves backward. “All topics” shows the latest per-topic value. + + ), + behind: ( + + Latest stored publish time minus the publish time at the furthest processed position, clamped at zero. This + approximate gap is not subscription backlog or wall-clock delay; producer clock skew and non-atomic refreshes + can distort it. “All topics” shows the worst readable topic only when all have both values. + + ), + entriesRead: ( + + The furthest processed message's 1-based entry position and the topic's current stored-entry count. This is a + storage position, not how many entries the session read or a subscription backlog. One producer batch is one + entry. On read-compacted targets it still describes the original topic. Separate snapshots can briefly make + the position exceed the total. + + ), + entriesLeft: ( + + Current stored entries after the furthest processed position, clamped at zero. This is not subscription + backlog. It is blank when the ledger containing that position is no longer stored or inspection is unavailable. + + ), + entryFraction: ( + + Stored entry position divided by the current stored-entry count. It is not the percentage consumed or a + message percentage; one producer batch is one entry. On read-compacted targets it still describes the original + topic. “All topics” uses summed positions and counts when every readable topic is known. + + ), + timeFraction: ( + + Publish time at the furthest processed position within the earliest-to-latest stored publish-time range. + Producer clocks need not be monotonic, so this is approximate and can be blank. “All topics” uses the combined + stored range and earliest session position when every readable topic has one. + + ), + firstPublished: ( + + Pulsar publish time of the earliest currently stored entry at the latest refresh. It is not necessarily the + earliest timestamp in the topic. “All topics” shows the minimum across readable topics. + + ), + firstMessage: ( + + Entry-level Pulsar message ID of the earliest currently stored entry; the batch index is omitted. Backlog, + expiry, and retention can move it forward. Dekaf does not order message IDs across topics. If inspection is + unavailable, this cell shows the broker reason. + + ), + lastPublished: ( + + Pulsar publish time of the latest currently stored entry at the latest refresh. It is not necessarily the + latest timestamp in the topic. “All topics” shows the maximum across readable topics. + + ), + lastMessage: ( + + Entry-level Pulsar message ID of the latest currently stored entry; the batch index is omitted. Dekaf does not + order message IDs across topics; use a publish-time column for cross-topic chronology. + + ) + } + }; + }, []); + + return ( +
+ {/* The same grey rounded empty-state the rest of the app uses (NothingToShow); the testids + stay on wrappers so the cells asserting these states keep their anchors. */} + {!isSessionStarted && ( +
+ +
+ )} + + {isSessionStarted && sessionGone !== undefined && ( +
+ + This consumer session has ended, so its positions stopped updating. + {sessionGone.message === undefined ? '' : ` ${sessionGone.message}.`} +  Start the session again to see them. + + } + /> +
+ )} + + {isPolling && loadError !== undefined && ( +
+ {loadError} - showing the last data that loaded. +
+ )} + + {isPolling && ( +
+ + tableId="topic-positions" + size="small" + dataLoader={{ cacheKey: [props.consumerName, 'topic-positions'], loader }} + columns={columns} + getId={(row) => row.topicFqn} + autoRefresh={{ intervalMs: 1000 }} + itemNamePlural="topics" + defaultSort={{ type: 'by-single-column', column: 'topic', direction: 'asc' }} + pinFirst={isAggregateRow} + /> +
+ )} +
+ ); +}; + +export default TopicPositions; diff --git a/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.spec.ts b/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.spec.ts new file mode 100644 index 000000000..5636188a6 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.spec.ts @@ -0,0 +1,453 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The Topic Positions row model and its formatters. + * + * The behaviour worth pinning is the one distinction the whole view rests on: ABSENT is not ZERO. A + * known publish-time position can really be 0%; a session position whose ledger has aged out is + * unknown. Rendering both as "0.0%" would invent the second, and every unset protobuf wrapper is a + * chance to do exactly that. + */ +import * as pb from '../../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; +import { + aggregateRow, + allTopicsLabel, + behindMsOf, + formatDurationMs, + entriesAfterCursorOf, + makePositionsLoader, + TopicPositionRow, + formatEntryCount, + formatFraction, + formatTimestamp, + timestampIso, + noValue, + topicPositionFromPb +} from './topic-positions'; + +describe('formatFraction', () => { + it('renders a real fraction as a percentage, keeping one decimal', () => { + // 99.9% and 100% are different answers - nearly done versus done - and whole percent merges them. + expect(formatFraction(0.999)).toBe('99.9%'); + expect(formatFraction(1)).toBe('100.0%'); + }); + + it('renders a genuine ZERO as 0.0%, for example at the start of a publish-time range', () => { + expect(formatFraction(0)).toBe('0.0%'); + }); + + it('renders UNKNOWN as the placeholder, never as zero', () => { + expect(formatFraction(undefined)).toBe(noValue); + }); + + it('refuses a non-finite fraction rather than printing NaN%', () => { + expect(formatFraction(NaN)).toBe(noValue); + expect(formatFraction(Infinity)).toBe(noValue); + }); +}); + +describe('formatTimestamp', () => { + it('renders an epoch millisecond', () => { + expect(formatTimestamp(0)).not.toBe(noValue); + expect(formatTimestamp(1785152235757)).not.toBe(noValue); + }); + + it('renders UNKNOWN as the placeholder - an empty topic has no first message', () => { + expect(formatTimestamp(undefined)).toBe(noValue); + expect(formatTimestamp(NaN)).toBe(noValue); + expect(formatTimestamp(Number.MAX_VALUE)).toBe(noValue); + }); + + it('provides ISO-8601 as the locale-independent copy value', () => { + expect(timestampIso(0)).toBe('1970-01-01T00:00:00.000Z'); + expect(timestampIso(undefined)).toBeUndefined(); + expect(timestampIso(Number.MAX_VALUE)).toBeUndefined(); + }); +}); + +describe('formatEntryCount', () => { + it('shows the arithmetic behind the percentage', () => { + expect(formatEntryCount(50, 100)).toBe('50 / 100'); + }); + + it('needs BOTH halves - an ordinal with no denominator says nothing', () => { + expect(formatEntryCount(50, undefined)).toBe(noValue); + expect(formatEntryCount(undefined, 100)).toBe(noValue); + }); +}); + +describe('entriesAfterCursorOf', () => { + it('subtracts the retained-log ordinal from the current retained count', () => { + expect(entriesAfterCursorOf({ cursorEntryOrdinal: 40, retainedEntries: 100 })).toBe(60); + }); + + it('clamps a stale denominator at zero instead of displaying a negative backlog', () => { + expect(entriesAfterCursorOf({ cursorEntryOrdinal: 150, retainedEntries: 100 })).toBe(0); + }); + + it('stays unknown unless both finite readings exist', () => { + expect(entriesAfterCursorOf({ cursorEntryOrdinal: undefined, retainedEntries: 100 })).toBeUndefined(); + expect(entriesAfterCursorOf({ cursorEntryOrdinal: 10, retainedEntries: undefined })).toBeUndefined(); + expect(entriesAfterCursorOf({ cursorEntryOrdinal: Infinity, retainedEntries: 100 })).toBeUndefined(); + }); +}); + +describe('topicPositionFromPb', () => { + it('leaves every unset wrapper undefined rather than defaulting it to zero', () => { + // What an EMPTY topic answers with: it was asked successfully and holds nothing, so there is no + // first message, no last message, and no cursor - and no reason either, because nothing failed. + const position = new pb.TopicPosition(); + position.setTopicFqn('persistent://t/n/empty'); + + const row = topicPositionFromPb(position); + + expect(row.topicFqn).toBe('persistent://t/n/empty'); + expect(row.firstMessageId).toBeUndefined(); + expect(row.firstPublishTime).toBeUndefined(); + expect(row.lastMessageId).toBeUndefined(); + expect(row.lastPublishTime).toBeUndefined(); + expect(row.firstConsumedMessageId).toBeUndefined(); + expect(row.firstConsumedPublishTime).toBeUndefined(); + expect(row.lastConsumedMessageId).toBeUndefined(); + expect(row.lastConsumedPublishTime).toBeUndefined(); + expect(row.cursorMessageId).toBeUndefined(); + expect(row.cursorPublishTime).toBeUndefined(); + expect(row.cursorTimeFraction).toBeUndefined(); + expect(row.cursorEntryFraction).toBeUndefined(); + expect(row.cursorEntryOrdinal).toBeUndefined(); + expect(row.retainedEntries).toBeUndefined(); + expect(row.unavailableReason).toBeUndefined(); + }); + + it('carries a fraction of exactly 0 through as 0, not as absent', () => { + // The mirror of the test above, and the reason the reader cannot use a falsy check anywhere in + // this path: 0 is a real reading. + const position = new pb.TopicPosition(); + position.setTopicFqn('persistent://t/n/topic'); + const fraction = new (require('google-protobuf/google/protobuf/wrappers_pb').DoubleValue)(); + fraction.setValue(0); + position.setCursorTimeFraction(fraction); + + const row = topicPositionFromPb(position); + + expect(row.cursorTimeFraction).toBe(0); + expect(formatFraction(row.cursorTimeFraction)).toBe('0.0%'); + }); + + it('reads the retained endpoints, consumed bounds and both progress figures when present', () => { + const { BytesValue, DoubleValue, Int64Value } = require('google-protobuf/google/protobuf/wrappers_pb'); + const position = new pb.TopicPosition(); + position.setTopicFqn('persistent://t/n/topic'); + + const firstId = new BytesValue(); + firstId.setValue(new Uint8Array([1, 2])); + position.setFirstMessageId(firstId); + + const firstTime = new Int64Value(); + firstTime.setValue(1000); + position.setFirstPublishTime(firstTime); + + const firstConsumedId = new BytesValue(); + firstConsumedId.setValue(new Uint8Array([3, 4])); + position.setFirstConsumedMessageId(firstConsumedId); + + const firstConsumedTime = new Int64Value(); + firstConsumedTime.setValue(1200); + position.setFirstConsumedPublishTime(firstConsumedTime); + + const lastConsumedId = new BytesValue(); + lastConsumedId.setValue(new Uint8Array([5, 6])); + position.setCursorMessageId(lastConsumedId); + + const lastConsumedTime = new Int64Value(); + lastConsumedTime.setValue(1800); + position.setCursorPublishTime(lastConsumedTime); + + const timeFraction = new DoubleValue(); + timeFraction.setValue(0.5); + position.setCursorTimeFraction(timeFraction); + + const ordinal = new Int64Value(); + ordinal.setValue(50); + position.setCursorEntryOrdinal(ordinal); + + const retained = new Int64Value(); + retained.setValue(100); + position.setRetainedEntries(retained); + + const row = topicPositionFromPb(position); + + expect(row.firstMessageId).toBeDefined(); + expect(row.firstPublishTime).toBe(1000); + expect(row.firstConsumedMessageId).toBe('03 04'); + expect(row.firstConsumedPublishTime).toBe(1200); + // The existing cursor wire fields are the furthest processed message. Keep the cursor + // aliases too: progress and lag still use the same position. + expect(row.lastConsumedMessageId).toBe('05 06'); + expect(row.lastConsumedPublishTime).toBe(1800); + expect(row.cursorMessageId).toBe('05 06'); + expect(row.cursorPublishTime).toBe(1800); + expect(row.cursorTimeFraction).toBe(0.5); + expect(formatEntryCount(row.cursorEntryOrdinal, row.retainedEntries)).toBe('50 / 100'); + }); + + it('reads the unavailable reason - a refused topic is not an empty one', () => { + const { StringValue } = require('google-protobuf/google/protobuf/wrappers_pb'); + const position = new pb.TopicPosition(); + position.setTopicFqn('non-persistent://t/n/topic'); + const reason = new StringValue(); + reason.setValue('Examine messages on a non-persistent topic is not allowed'); + position.setUnavailableReason(reason); + + const row = topicPositionFromPb(position); + + expect(row.unavailableReason).toContain('non-persistent'); + }); +}); + + +describe('behindMsOf - the consumer-lag clock', () => { + it('is last published minus cursor published', () => { + expect(behindMsOf({ lastPublishTime: 5000, cursorPublishTime: 2000 })).toBe(3000); + }); + + it('clamps at zero - a cursor past the recorded end is a stale denominator, not time travel', () => { + expect(behindMsOf({ lastPublishTime: 2000, cursorPublishTime: 5000 })).toBe(0); + }); + + it('is unknown when either side is unknown', () => { + expect(behindMsOf({ lastPublishTime: 5000, cursorPublishTime: undefined })).toBeUndefined(); + expect(behindMsOf({ lastPublishTime: undefined, cursorPublishTime: 2000 })).toBeUndefined(); + }); +}); + +describe('formatDurationMs', () => { + it('reads as humans write durations', () => { + expect(formatDurationMs(500)).toBe('<1s'); + expect(formatDurationMs(45_000)).toBe('45s'); + expect(formatDurationMs(2 * 60_000 + 5_000)).toBe('2m 05s'); + expect(formatDurationMs(2 * 3_600_000 + 5 * 60_000)).toBe('2h 05m'); + expect(formatDurationMs(3 * 86_400_000 + 4 * 3_600_000)).toBe('3d 4h'); + }); + + it('unknown and nonsense are the placeholder', () => { + expect(formatDurationMs(undefined)).toBe('-'); + expect(formatDurationMs(-5)).toBe('-'); + expect(formatDurationMs(NaN)).toBe('-'); + }); +}); + +describe('aggregateRow - the "All topics" line', () => { + const row = (over: Partial): TopicPositionRow => ({ topicFqn: 't', ...over }); + + it('takes the global time range, the WORST lag, and the summed entries', () => { + const a = row({ + topicFqn: 'a', + firstPublishTime: 1000, + lastPublishTime: 9000, + firstConsumedMessageId: '01', + firstConsumedPublishTime: 1500, + lastConsumedMessageId: '02', + lastConsumedPublishTime: 8000, + cursorPublishTime: 8000, + cursorEntryOrdinal: 90, + retainedEntries: 100 + }); + const b = row({ + topicFqn: 'b', + firstPublishTime: 2000, + lastPublishTime: 10_000, + firstConsumedMessageId: '03', + firstConsumedPublishTime: 2500, + lastConsumedMessageId: '04', + lastConsumedPublishTime: 4000, + cursorPublishTime: 4000, + cursorEntryOrdinal: 10, + retainedEntries: 100 + }); + + const all = aggregateRow([a, b]); + + expect(all?.topicFqn).toBe(allTopicsLabel); + expect(all?.isAggregate).toBe(true); + expect(all?.firstPublishTime).toBe(1000); + expect(all?.lastPublishTime).toBe(10_000); + expect(all?.firstConsumedMessageId).toBeUndefined(); + expect(all?.firstConsumedPublishTime).toBe(1500); + expect(all?.lastConsumedMessageId).toBeUndefined(); + expect(all?.lastConsumedPublishTime).toBe(8000); + // b is 6000ms behind (10000-4000), a only 1000ms (9000-8000... vs global last: a is 2000 + // behind the GLOBAL newest) - the aggregate answers with the WORST: 6000. + expect(behindMsOf(all!)).toBe(6000); + expect(all?.cursorEntryOrdinal).toBe(100); + expect(all?.retainedEntries).toBe(200); + expect(all?.cursorEntryFraction).toBe(0.5); + }); + + it('does not exist for a single topic - a summary of one is noise', () => { + expect(aggregateRow([row({ topicFqn: 'only' })])).toBeUndefined(); + }); + + it('unavailable rows contribute consumed facts, but not unavailable retained endpoints', () => { + const ok1 = row({ + topicFqn: 'a', + firstPublishTime: 1000, + lastPublishTime: 2000, + firstConsumedPublishTime: 1200, + lastConsumedPublishTime: 1800 + }); + const ok2 = row({ + topicFqn: 'b', + firstPublishTime: 1500, + lastPublishTime: 3000, + firstConsumedPublishTime: 1600, + lastConsumedPublishTime: 2500 + }); + const refused = row({ + topicFqn: 'np', + unavailableReason: 'non-persistent', + firstConsumedPublishTime: 500, + lastConsumedPublishTime: 4000 + }); + + const all = aggregateRow([ok1, ok2, refused]); + expect(all?.firstPublishTime).toBe(1000); + expect(all?.lastPublishTime).toBe(3000); + expect(all?.firstConsumedPublishTime).toBe(500); + expect(all?.lastConsumedPublishTime).toBe(4000); + }); + + it('a topic with no cursor leaves the aggregate positions unknown rather than guessed', () => { + const read = row({ topicFqn: 'a', firstPublishTime: 1000, lastPublishTime: 2000, cursorPublishTime: 1500 }); + const unread = row({ topicFqn: 'b', firstPublishTime: 1000, lastPublishTime: 2000 }); + + const all = aggregateRow([read, unread]); + expect(all?.cursorTimeFraction).toBeUndefined(); + expect(behindMsOf(all!)).toBeUndefined(); + }); + + it('does not fabricate aggregate lag from another topic when a retained endpoint is missing', () => { + const complete = row({ topicFqn: 'a', lastPublishTime: 10_000, cursorPublishTime: 9000 }); + const missingEndpoint = row({ topicFqn: 'b', cursorPublishTime: 1000 }); + + const all = aggregateRow([complete, missingEndpoint]); + + // Combining b's cursor with a's endpoint would claim 9 seconds of lag even though b's own + // endpoint is unknown. The aggregate must stay blank until every readable topic is comparable. + expect(all?.cursorPublishTime).toBeUndefined(); + expect(behindMsOf(all!)).toBeUndefined(); + }); +}); + +describe('makePositionsLoader - what the polling Table is allowed to see', () => { + const okRow = (fqn: string): TopicPositionRow => ({ topicFqn: fqn, firstPublishTime: 1, lastPublishTime: 2 }); + const OK = 0; + const FAILED_PRECONDITION = 9; + const onSessionGone = jest.fn(); + + beforeEach(() => onSessionGone.mockReset()); + + it('success replaces last-good and prepends the aggregate for multi-topic sessions', async () => { + const onError = jest.fn(); + const loader = makePositionsLoader({ + okCode: OK, + failedPreconditionCode: FAILED_PRECONDITION, + onError, + onSessionGone, + fetch: async () => ({ code: OK, message: '', rows: [okRow('a'), okRow('b')] }) + }); + + const rows = await loader(); + expect(rows.map((r) => r.topicFqn)).toEqual([allTopicsLabel, 'a', 'b']); + expect(onError).toHaveBeenLastCalledWith(undefined); + }); + + it('a transport failure KEEPS the last-good rows and reports the error - never throws', async () => { + const onError = jest.fn(); + let fail = false; + const loader = makePositionsLoader({ + okCode: OK, + failedPreconditionCode: FAILED_PRECONDITION, + onError, + onSessionGone, + fetch: async () => { + if (fail) { + throw new Error('connection refused'); + } + return { code: OK, message: '', rows: [okRow('a'), okRow('b')] }; + } + }); + + const good = await loader(); + fail = true; + const afterFailure = await loader(); + + expect(afterFailure).toEqual(good); + expect(String(onError.mock.calls[onError.mock.calls.length - 1][0])).toContain('connection refused'); + }); + + it('a non-OK answer keeps last-good too, and recovery clears the error', async () => { + const onError = jest.fn(); + let mode: 'ok' | 'broken' = 'ok'; + const loader = makePositionsLoader({ + okCode: OK, + failedPreconditionCode: FAILED_PRECONDITION, + onError, + onSessionGone, + fetch: async () => + mode === 'ok' + ? { code: OK, message: '', rows: [okRow('a'), okRow('b')] } + : { code: 2, message: 'the broker fell over', rows: [] } + }); + + const good = await loader(); + mode = 'broken'; + expect(await loader()).toEqual(good); + expect(onError).toHaveBeenLastCalledWith('the broker fell over'); + mode = 'ok'; + await loader(); + expect(onError).toHaveBeenLastCalledWith(undefined); + }); + + it('a vanished session clears the table - stale rows must not pose as current', async () => { + const onError = jest.fn(); + let gone = false; + const loader = makePositionsLoader({ + okCode: OK, + failedPreconditionCode: FAILED_PRECONDITION, + onError, + onSessionGone, + fetch: async () => + gone + ? { code: FAILED_PRECONDITION, message: 'no such session', rows: [] } + : { code: OK, message: '', rows: [okRow('a'), okRow('b')] } + }); + + await loader(); + expect(onSessionGone).not.toHaveBeenCalled(); + gone = true; + expect(await loader()).toEqual([]); + expect(onError).toHaveBeenLastCalledWith(undefined); + // TERMINAL, and said so with the server's own words. Without this signal the caller cannot + // tell "the session is over" from "the session has no topics", and polls a dead session. + expect(onSessionGone).toHaveBeenCalledWith('no such session'); + }); + + it.each([ + ['an OK answer with no topics at all', { code: OK, message: '', rows: [] as TopicPositionRow[] }], + ['a broker failure', { code: 2, message: 'the broker fell over', rows: [] as TopicPositionRow[] }] + ])('does not call it the end of the session for %s', async (_name, reply) => { + const loader = makePositionsLoader({ + okCode: OK, + failedPreconditionCode: FAILED_PRECONDITION, + onError: jest.fn(), + onSessionGone, + fetch: async () => reply + }); + + await loader(); + + expect(onSessionGone).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.ts b/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.ts new file mode 100644 index 000000000..f49a1debc --- /dev/null +++ b/ui/components/ui/ConsumerSession/Console/TopicPositions/topic-positions.ts @@ -0,0 +1,315 @@ +import * as pb from '../../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; +import { hexStringFromByteArray } from '../../../../conversions/conversions'; + +/** + * One row of the Topic Positions debug view. + * + * EVERY FIGURE IS OPTIONAL, and `undefined` means "not known" - never zero. The distinction is the + * whole point of the view: the first of N stored entries has position 1/N, while a topic whose + * session position has aged out from under retention knows nothing about where it is. A table that + * rendered the latter as "0%" would quietly invent a value. + */ +export type TopicPositionRow = { + topicFqn: string; + firstMessageId?: string; + firstPublishTime?: number; + lastMessageId?: string; + lastPublishTime?: number; + firstConsumedMessageId?: string; + firstConsumedPublishTime?: number; + lastConsumedMessageId?: string; + lastConsumedPublishTime?: number; + /** Internal high-water mark used by progress calculations; the per-topic value is furthest processed. */ + cursorMessageId?: string; + cursorPublishTime?: number; + cursorTimeFraction?: number; + cursorEntryFraction?: number; + cursorEntryOrdinal?: number; + retainedEntries?: number; + /** Set when the broker refused the topic - a non-persistent one cannot be examined at all. */ + unavailableReason?: string; + /** The synthetic "All topics" summary row - pinned first, exempt from per-topic semantics. */ + isAggregate?: boolean; +}; + +/** What the table prints where it has nothing to print. */ +export const noValue = '-'; + +const bytesToHex = (v?: { getValue_asU8: () => Uint8Array }): string | undefined => { + const bytes = v?.getValue_asU8?.(); + return bytes === undefined || bytes.length === 0 ? undefined : hexStringFromByteArray(bytes, 'hex-with-space'); +}; + +const num = (v?: { getValue: () => number }): number | undefined => (v === undefined ? undefined : v.getValue()); + +/** + * Read one row off the wire. + * + * An UNSET protobuf wrapper stays `undefined` here rather than becoming 0 - see [[TopicPositionRow]] + * for why that distinction is load-bearing rather than fussy. + */ +export function topicPositionFromPb(position: pb.TopicPosition): TopicPositionRow { + const cursorMessageId = bytesToHex(position.getCursorMessageId() as never); + const cursorPublishTime = num(position.getCursorPublishTime() as never); + return { + topicFqn: position.getTopicFqn(), + firstMessageId: bytesToHex(position.getFirstMessageId() as never), + firstPublishTime: num(position.getFirstPublishTime() as never), + lastMessageId: bytesToHex(position.getLastMessageId() as never), + lastPublishTime: num(position.getLastPublishTime() as never), + firstConsumedMessageId: bytesToHex(position.getFirstConsumedMessageId() as never), + firstConsumedPublishTime: num(position.getFirstConsumedPublishTime() as never), + // The server's cursor is the furthest/newest processed log position. Keep the cursor aliases + // for the progress arithmetic and expose the same endpoint under its user-facing name. + lastConsumedMessageId: cursorMessageId, + lastConsumedPublishTime: cursorPublishTime, + cursorMessageId, + cursorPublishTime, + cursorTimeFraction: num(position.getCursorTimeFraction() as never), + cursorEntryFraction: num(position.getCursorEntryFraction() as never), + cursorEntryOrdinal: num(position.getCursorEntryOrdinal() as never), + retainedEntries: num(position.getRetainedEntries() as never), + unavailableReason: position.getUnavailableReason()?.getValue() || undefined + }; +} + +/** + * A fraction as a percentage. + * + * One decimal place because the interesting readings are the ones near the ends - "99.9%" and "100%" + * are different answers on a session that is nearly done versus done, and rounding to whole percent + * would merge them. + */ +export function formatFraction(fraction: number | undefined): string { + if (fraction === undefined || !Number.isFinite(fraction)) { + return noValue; + } + return `${(fraction * 100).toFixed(1)}%`; +} + +/** An epoch millisecond as a local timestamp, or the placeholder when there is none. */ +export function formatTimestamp(epochMs: number | undefined): string { + const date = dateOf(epochMs); + if (date === undefined) { + return noValue; + } + return date.toLocaleString(); +} + +/** The exact, locale-independent timestamp copied out of a displayed time cell. */ +export function timestampIso(epochMs: number | undefined): string | undefined { + return dateOf(epochMs)?.toISOString(); +} + +/** `ordinal / total`, the arithmetic behind the entry percentage, shown so it can be checked. */ +export function formatEntryCount(ordinal: number | undefined, retained: number | undefined): string { + if (ordinal === undefined || retained === undefined || !Number.isFinite(ordinal) || !Number.isFinite(retained)) { + return noValue; + } + return `${ordinal.toLocaleString()} / ${retained.toLocaleString()}`; +} + +/** Entries currently stored after the session position. Separate broker snapshots can briefly make + * the position exceed the reported total; that means none are known to be after it, not a + * negative backlog. */ +export function entriesAfterCursorOf( + row: Pick +): number | undefined { + if ( + row.cursorEntryOrdinal === undefined || + row.retainedEntries === undefined || + !Number.isFinite(row.cursorEntryOrdinal) || + !Number.isFinite(row.retainedEntries) + ) { + return undefined; + } + return Math.max(0, row.retainedEntries - row.cursorEntryOrdinal); +} + +const dateOf = (epochMs: number | undefined): Date | undefined => { + if (epochMs === undefined || !Number.isFinite(epochMs)) { + return undefined; + } + const date = new Date(epochMs); + return Number.isFinite(date.getTime()) ? date : undefined; +}; + +/** + * Approximate publish-time distance between the newest retained endpoint and the session's + * furthest consumed endpoint. This is useful context for "why am I seeing old data", but it is not + * a subscription backlog: publish times come from producers and the two endpoints are refreshed + * separately. + * + * Clamped at zero: the endpoints and the cursor are separate lookups, so a message published + * between them can put the cursor "ahead" of the recorded end - stale denominator, not time travel. + */ +export function behindMsOf(row: Pick): number | undefined { + if (row.lastPublishTime === undefined || row.cursorPublishTime === undefined) { + return undefined; + } + return Math.max(0, row.lastPublishTime - row.cursorPublishTime); +} + +/** A duration as humans read one: "3d 4h", "2h 05m", "45s", "<1s". */ +export function formatDurationMs(ms: number | undefined): string { + if (ms === undefined || !Number.isFinite(ms) || ms < 0) { + return noValue; + } + if (ms < 1000) { + return '<1s'; + } + const seconds = Math.floor(ms / 1000); + const days = Math.floor(seconds / 86400); + const hours = Math.floor((seconds % 86400) / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = seconds % 60; + if (days > 0) { + return `${days}d ${hours}h`; + } + if (hours > 0) { + return `${hours}h ${String(minutes).padStart(2, '0')}m`; + } + if (minutes > 0) { + return `${minutes}m ${String(secs).padStart(2, '0')}s`; + } + return `${secs}s`; +} + +/** The label of the synthetic aggregate row. */ +export const allTopicsLabel = 'All topics'; + +/** What the loader needs from the outside world - injectable, so every transition is a unit test. */ +export type PositionsFetch = () => Promise<{ + code: number | undefined; + message: string | undefined; + rows: TopicPositionRow[]; +}>; + +/** + * Build the polling loader the shared Table drives. + * + * THE LOADER NEVER THROWS, by design: the Table toasts its data-loader errors on every render, so + * a 1-second poll that throws is a toast storm. Instead every answer is classified here - success + * replaces the last-good rows (and prepends the aggregate), a vanished session is TERMINAL and + * reported through `onSessionGone` so the caller can stop asking, and everything else KEEPS the + * last-good rows on screen and reports through `onError` - stale data with a banner beats a blank + * table with a toast. + * + * The three outcomes are genuinely different and the caller must be able to tell them apart: an OK + * answer with no topics is an EMPTY session, a transport failure is a RETRYABLE fault, and + * FAILED_PRECONDITION is the server saying the session no longer exists. Only the last one is + * final; the first two are worth polling again. + */ +export function makePositionsLoader(deps: { + fetch: PositionsFetch; + okCode: number; + failedPreconditionCode: number; + onError: (message: string | undefined) => void; + /** + * The session is gone or terminal and will never answer again. Required, not optional: the + * server returns FAILED_PRECONDITION precisely so the client stops scanning, and a caller that + * silently ignored it would poll a dead session for as long as the tab stays open. + */ + onSessionGone: (message: string | undefined) => void; +}): () => Promise { + let lastGood: TopicPositionRow[] = []; + + return async () => { + let answer; + try { + answer = await deps.fetch(); + } catch (err) { + deps.onError(`${(err as Error)?.message ?? err}`); + return lastGood; + } + + if (answer.code === deps.failedPreconditionCode) { + // The session vanished mid-run (deleted elsewhere, reaped, or stopped). Yesterday's rows + // belong to a session that no longer exists - clear rather than display them as current, + // and tell the caller this is the end rather than an empty reading. + deps.onError(undefined); + deps.onSessionGone(answer.message || undefined); + lastGood = []; + return lastGood; + } + + if (answer.code !== deps.okCode) { + deps.onError(answer.message || 'The consumer session did not answer.'); + return lastGood; + } + + deps.onError(undefined); + const aggregate = aggregateRow(answer.rows); + lastGood = aggregate === undefined ? answer.rows : [aggregate, ...answer.rows]; + return lastGood; + }; +} + +/** + * The "All topics" row: the session's whole watch, one line. + * + * ONLY WHAT AGGREGATES HONESTLY IS AGGREGATED. Retained first/last publish times take the min/max + * across broker-readable topics. Consumed publish times take the min/max across ALL topic rows, + * because those are session-local facts that remain known when retained-log inspection fails. + * The retained-entry fraction is the sum of cursor ordinals over retained entries. Publish-time + * lag is the WORST topic's value. Message ids do not aggregate - there is no global cross-topic + * message id - so all four id cells stay blank. + */ +export function aggregateRow(rows: TopicPositionRow[]): TopicPositionRow | undefined { + const members = rows.filter((r) => !r.isAggregate); + if (members.length < 2) { + return undefined; + } + const usable = members.filter((r) => r.unavailableReason === undefined); + + const defined = (values: (T | undefined)[]): T[] => values.filter((v): v is T => v !== undefined); + + const firsts = defined(usable.map((r) => r.firstPublishTime)); + const lasts = defined(usable.map((r) => r.lastPublishTime)); + const cursors = defined(usable.map((r) => r.cursorPublishTime)); + const behinds = defined(usable.map((r) => behindMsOf(r))); + const withEntries = usable.filter((r) => r.cursorEntryOrdinal !== undefined && r.retainedEntries !== undefined); + const firstConsumedTimes = defined(members.map((r) => r.firstConsumedPublishTime)); + const lastConsumedTimes = defined(members.map((r) => r.lastConsumedPublishTime)); + + const firstPublishTime = firsts.length > 0 ? Math.min(...firsts) : undefined; + const lastPublishTime = lasts.length > 0 ? Math.max(...lasts) : undefined; + // The LAGGIEST cursor stands for the session: the range up to it is what the whole session has + // certainly covered. + const cursorPublishTime = cursors.length === usable.length && cursors.length > 0 ? Math.min(...cursors) : undefined; + + const ordinalSum = withEntries.reduce((acc, r) => acc + (r.cursorEntryOrdinal as number), 0); + const retainedSum = defined(usable.map((r) => r.retainedEntries)).reduce((acc, v) => acc + v, 0); + + let cursorTimeFraction: number | undefined = undefined; + if ( + firstPublishTime !== undefined && + lastPublishTime !== undefined && + cursorPublishTime !== undefined && + lastPublishTime > firstPublishTime + ) { + cursorTimeFraction = Math.min(1, Math.max(0, (cursorPublishTime - firstPublishTime) / (lastPublishTime - firstPublishTime))); + } + + return { + topicFqn: allTopicsLabel, + isAggregate: true, + firstPublishTime, + lastPublishTime, + firstConsumedPublishTime: firstConsumedTimes.length > 0 ? Math.min(...firstConsumedTimes) : undefined, + lastConsumedPublishTime: lastConsumedTimes.length > 0 ? Math.max(...lastConsumedTimes) : undefined, + // behindMsOf(aggregate) must answer the WORST lag, so the aggregate stores the laggiest + // cursor against the global newest message; the two line up by construction. + // This internal aggregate cursor exists only to make behindMsOf return the honest worst + // per-topic lag. Falling back to the earliest known cursor would combine it with a different + // topic's global endpoint and fabricate a lag when even one readable topic lacks an endpoint. + // cursorTimeFraction was computed separately above and does not need this surrogate value. + cursorPublishTime: behinds.length === usable.length && lastPublishTime !== undefined && behinds.length > 0 + ? lastPublishTime - Math.max(...behinds) + : undefined, + cursorTimeFraction, + cursorEntryFraction: withEntries.length === usable.length && retainedSum > 0 ? Math.min(1, ordinalSum / retainedSum) : undefined, + cursorEntryOrdinal: withEntries.length === usable.length && withEntries.length > 0 ? ordinalSum : undefined, + retainedEntries: retainedSum > 0 ? retainedSum : undefined + }; +} diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.columns.test.tsx b/ui/components/ui/ConsumerSession/ConsumerSession.columns.test.tsx new file mode 100644 index 000000000..5294e3d32 --- /dev/null +++ b/ui/components/ui/ConsumerSession/ConsumerSession.columns.test.tsx @@ -0,0 +1,206 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The message table's draggable column order, driven through its real headers. The message table + * owns its own drag handlers rather than using the shared Table, so the gesture has to be pinned + * here as well - what the DOM asks for is half of what a drop means. + * + * The header only exists once the session is running and holding messages, so the session is + * started for real against a stubbed transport, exactly as the lifecycle suite does. + */ +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); + +// No gRPC endpoint in jsdom, and the generated clients would try to reach one on import. +const mockClients = { current: undefined as unknown }; +jest.mock('../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +// The message table virtualizes rows and measures its viewport; jsdom lays nothing out, so its +// header is rendered into a plain table here (the pattern TopicPositions.test.tsx uses). Only the +// header: these tests are about the columns, and the row cells are pinned elsewhere. +jest.mock('react-virtuoso', () => { + const ReactRuntime = require('react'); + return { + TableVirtuoso: ReactRuntime.forwardRef((props: any, _ref: unknown) => + ReactRuntime.createElement( + 'table', + null, + ReactRuntime.createElement('thead', null, props.fixedHeaderContent()) + )), + }; +}); + +import React from 'react'; +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import ConsumerSession from './ConsumerSession'; +import { getDefaultManagedItem } from '../LibraryBrowser/default-library-items'; +import { localStorageKeys } from '../../local-storage-keys'; +import { messageThMeta } from './message-columns'; +import { Status } from '../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../grpc-web/google/rpc/code_pb'; +import { StringValue } from 'google-protobuf/google/protobuf/wrappers_pb'; +import { + CreateConsumerResponse, + DeleteConsumerResponse, + Message, + PauseResponse, + ResumeResponse, +} from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; + +const topicContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, +}; + +const status = (code: number) => new Status().setCode(code).setMessage(''); + +/** The `on`/`removeListener`/`cancel` surface of a grpc-web ClientReadableStream, plus an emitter. */ +const fakeResumeStream = () => { + const listeners: Record void)[]> = {}; + return { + on(event: string, cb: (v: unknown) => void) { + (listeners[event] = listeners[event] || []).push(cb); + return this; + }, + removeListener(event: string, cb: (v: unknown) => void) { + listeners[event] = (listeners[event] || []).filter((it) => it !== cb); + return this; + }, + cancel() {}, + emit(event: string, v?: unknown) { + (listeners[event] || []).slice().forEach((cb) => cb(v)); + }, + }; +}; + +/** Renders the session, plays it, and delivers one message so the message table is on screen. */ +const renderRunningSession = async () => { + const stream = fakeResumeStream(); + const okStatus = { getStatus: () => ({ getCode: () => Code.OK, getMessage: () => '' }) }; + mockClients.current = { + consumerServiceClient: { + createConsumer: () => Promise.resolve(new CreateConsumerResponse().setStatus(status(Code.OK))), + resume: () => stream, + pause: () => Promise.resolve(new PauseResponse().setStatus(status(Code.OK))), + deleteConsumer: () => Promise.resolve(new DeleteConsumerResponse()), + resolveTopicSelector: () => Promise.reject(new Error('not used by these tests')), + }, + // The Tools panel's Produce tab creates a producer as soon as the session renders. + producerServiceClient: { + createProducer: () => Promise.resolve(okStatus), + deleteProducer: () => Promise.resolve(okStatus), + send: () => Promise.resolve(okStatus), + }, + libraryServiceClient: { + listLibraryItems: () => Promise.reject(new Error('no library in these tests')), + getLibraryItem: () => Promise.reject(new Error('no library in these tests')), + }, + }; + + const config = { type: 'value' as const, val: getDefaultManagedItem('consumer-session-config', topicContext) }; + await act(async () => { + render( + + + + ); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-play')); + }); + + const message = new Message(); + message.setValue(new StringValue().setValue('m-1')); + message.setNumMessageProcessed(1); + message.setNumMessageSent(1); + await act(async () => { + stream.emit('data', new ResumeResponse().setStatus(status(Code.OK)).setMessagesList([message])); + }); + + // The buffered message reaches the table on the flush ticker, not on arrival. + await act(async () => { + jest.advanceTimersByTime(1000); + }); + await act(async () => { + jest.advanceTimersByTime(500); + }); +}; + +// A column's test id is not always its key (`sessionTargetIndex` renders as `cs-th-target`), and +// the persisted order is written in KEYS - so the rendered header is read back through the same +// map, and both assertions below talk about the same thing. +const keyByTestId = new Map(Object.entries(messageThMeta).map(([key, meta]) => [meta.testId, key])); +const testIdByKey = new Map(Object.entries(messageThMeta).map(([key, meta]) => [key, meta.testId])); + +/** The reorderable columns in the order the header actually renders them. */ +const headerOrder = () => + Array.from(document.querySelectorAll('th[data-testid^="cs-th-"]')) + .map((el) => keyByTestId.get(el.getAttribute('data-testid')!)!) + // The sticky pair is pinned in front and is not reorderable. ('index' joined the meta record + // when it became resizable on 2026-08-11, so it now needs excluding by name too.) + .filter((key) => key !== undefined && key !== 'publishTime' && key !== 'index'); + +const th = (columnKey: string) => screen.getByTestId(testIdByKey.get(columnKey)!); + +/** One native drag gesture: pick a header up, hover another, drop it there. */ +const dragOnto = (dragged: string, target: string) => { + const dataTransfer = { setData: jest.fn(), getData: () => dragged, dropEffect: '', effectAllowed: '' }; + fireEvent.dragStart(th(dragged), { dataTransfer }); + fireEvent.dragOver(th(target), { dataTransfer }); + fireEvent.drop(th(target), { dataTransfer }); + fireEvent.dragEnd(th(dragged), { dataTransfer }); +}; + +describe('dragging a message-table column header', () => { + beforeEach(() => { + jest.useFakeTimers(); + // The flush scrolls the table to the bottom; jsdom has no scrollTo on elements. + (Element.prototype as { scrollTo?: () => void }).scrollTo = () => undefined; + window.localStorage.clear(); + // The Tools panel opens by default; nothing here needs it on screen. + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'false'); + }); + + afterEach(() => { + cleanup(); + jest.useRealTimers(); + window.localStorage.clear(); + }); + + it('can move a column to the LAST position', async () => { + await renderRunningSession(); + const before = headerOrder(); + expect(before[0]).toBe('key'); + const last = before[before.length - 1]; + + // The last position is the one no drop could reach: every drop inserted BEFORE the header it + // landed on, so the trailing slot needed a header to its right, and there is none. + dragOnto('key', last); + + const expected = [...before.slice(1), 'key']; + expect(headerOrder()).toEqual(expected); + // ...and the order on screen is the order that survives a reload. + expect(JSON.parse(window.localStorage.getItem('table:consumer-session-messages:column-order')!)) + .toEqual(expected); + }); + + it('can move a column to the FIRST position', async () => { + await renderRunningSession(); + const before = headerOrder(); + const last = before[before.length - 1]; + + dragOnto(last, 'key'); + + expect(headerOrder()).toEqual([last, ...before.slice(0, before.length - 1)]); + }); +}); diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.degradation.test.tsx b/ui/components/ui/ConsumerSession/ConsumerSession.degradation.test.tsx new file mode 100644 index 000000000..b2b1c9e9f --- /dev/null +++ b/ui/components/ui/ConsumerSession/ConsumerSession.degradation.test.tsx @@ -0,0 +1,257 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The start-position degradation banner's RENDER BOUND. + * + * The banner is sticky and disclosed on purpose - a start cut that could not be resolved against + * every topic really is degraded, and the session must keep saying so. What must NOT scale with the + * degradation is the DOM: a topic selector admits up to 2,000 streams, and the banner used to build + * one `
  • ` per abandoned stream, so the worst-case session grew a ~2,000-node subtree at exactly + * the moment it was already under stress. + * + * So the bound is asserted at the wide end, where the difference is real, and the same tests pin + * that nothing was hidden by it: the count is still stated, the full list is one click away, and + * the copy affordance yields every name. + * + * The banner only exists once the session left `new`, so the session is started for real against a + * stubbed transport and the degradation arrives the way the server sends it - on a resume frame's + * `consumer_stats` - exactly as the lifecycle and columns suites drive it. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); + +// No gRPC endpoint in jsdom, and the generated clients would try to reach one on import. +const mockClients = { current: undefined as unknown }; +jest.mock('../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +import React from 'react'; +import '@testing-library/jest-dom'; +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import ConsumerSession from './ConsumerSession'; +import { getDefaultManagedItem } from '../LibraryBrowser/default-library-items'; +import { localStorageKeys } from '../../local-storage-keys'; +import { Status } from '../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../grpc-web/google/rpc/code_pb'; +import { + ConsumerStats, + CreateConsumerResponse, + DeleteConsumerResponse, + PauseResponse, + ResumeResponse, + StartFromProgress, +} from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; + +const topicContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, +}; + +const status = (code: number) => new Status().setCode(code).setMessage(''); + +/** The worst case a topic selector admits. */ +const maxSelectorStreams = 2000; + +/** + * The ceiling this suite holds the banner to. Deliberately NOT the component's own preview size: + * a test that imported that constant would move its goalpost along with the code and could never + * go red for rendering more. + */ +const maxRenderedTopics = 10; + +/** What the server names: `@`. The banner shows the TOPIC half. */ +const abandonedStreams = (count: number) => + Array.from({ length: count }, (_, i) => `dekaf-session@persistent://public/default/silent-topic-${i}`); + +const topicOf = (streamId: string) => streamId.slice(streamId.indexOf('@') + 1); + +/** The `on`/`removeListener`/`cancel` surface of a grpc-web ClientReadableStream, plus an emitter. */ +const fakeResumeStream = () => { + const listeners: Record void)[]> = {}; + return { + on(event: string, cb: (v: unknown) => void) { + (listeners[event] = listeners[event] || []).push(cb); + return this; + }, + removeListener(event: string, cb: (v: unknown) => void) { + listeners[event] = (listeners[event] || []).filter((it) => it !== cb); + return this; + }, + cancel() {}, + emit(event: string, v?: unknown) { + (listeners[event] || []).slice().forEach((cb) => cb(v)); + }, + }; +}; + +/** + * A resume frame reporting a FINISHED but degraded start-from resolution: the skip panel is gone + * (`complete`), and what remains is the record of the streams the resolution gave up on. + */ +const degradedFrame = (streams: string[]) => { + const progress = new StartFromProgress(); + progress.setComplete(true); + progress.setDegraded(true); + progress.setAbandonedStreamsList(streams); + return new ResumeResponse() + .setStatus(status(Code.OK)) + .setConsumerStats(new ConsumerStats().setStartFromProgress(progress)); +}; + +/** Renders the session, plays it, and delivers one degraded start-from report. */ +const renderDegradedSession = async (streams: string[]) => { + const stream = fakeResumeStream(); + const okStatus = { getStatus: () => ({ getCode: () => Code.OK, getMessage: () => '' }) }; + mockClients.current = { + consumerServiceClient: { + createConsumer: () => Promise.resolve(new CreateConsumerResponse().setStatus(status(Code.OK))), + resume: () => stream, + pause: () => Promise.resolve(new PauseResponse().setStatus(status(Code.OK))), + deleteConsumer: () => Promise.resolve(new DeleteConsumerResponse()), + getTopicPositions: () => Promise.reject(new Error('not used by these tests')), + resolveTopicSelector: () => Promise.reject(new Error('not used by these tests')), + }, + // The Tools panel's Produce tab creates a producer as soon as the session renders. + producerServiceClient: { + createProducer: () => Promise.resolve(okStatus), + deleteProducer: () => Promise.resolve(okStatus), + send: () => Promise.resolve(okStatus), + }, + libraryServiceClient: { + listLibraryItems: () => Promise.reject(new Error('no library in these tests')), + getLibraryItem: () => Promise.reject(new Error('no library in these tests')), + }, + }; + + const config = { type: 'value' as const, val: getDefaultManagedItem('consumer-session-config', topicContext) }; + await act(async () => { + render( + + + + ); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-play')); + }); + + await act(async () => { + stream.emit('data', degradedFrame(streams)); + }); +}; + +const banner = () => screen.getByTestId('cs-start-from-degraded'); +/** The topic names the banner actually put in the document. */ +const renderedTopics = () => Array.from(banner().querySelectorAll('li')).map((li) => li.textContent); +const click = async (testId: string) => { + await act(async () => { + fireEvent.click(screen.getByTestId(testId)); + }); +}; + +describe('the start-position degradation banner', () => { + /** Whatever the copy affordance handed to the clipboard. */ + let clipboardWrites: string[] = []; + + beforeEach(() => { + clipboardWrites = []; + // `copyToClipboard` takes the modern path only in a secure context; jsdom provides neither the + // flag nor a clipboard, and its `execCommand` fallback does not exist either. + Object.defineProperty(window, 'isSecureContext', { configurable: true, value: true }); + Object.defineProperty(window.navigator, 'clipboard', { + configurable: true, + value: { + writeText: (text: string) => { + clipboardWrites.push(text); + return Promise.resolve(); + }, + }, + }); + window.localStorage.clear(); + // The Tools panel opens by default; nothing here needs it on screen. + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'false'); + }); + + afterEach(() => { + cleanup(); + window.localStorage.clear(); + }); + + it('does not build one row per abandoned stream at selector scale', async () => { + // The defect: a 2,000-stream selector produced a ~2,000-node banner subtree, disclosed by + // default, on a session that is already degraded and under load. + await renderDegradedSession(abandonedStreams(maxSelectorStreams)); + + expect(renderedTopics().length).toBeLessThanOrEqual(maxRenderedTopics); + // ...and nothing else in the banner scales with the abandoned list either. + expect(banner().querySelectorAll('*').length).toBeLessThanOrEqual(4 * maxRenderedTopics); + }); + + it('still states the true number of silent topics, not the number it rendered', async () => { + await renderDegradedSession(abandonedStreams(maxSelectorStreams)); + + expect(banner().textContent).toContain(String(maxSelectorStreams)); + }); + + it('stays disclosed and collapsible - the degradation is not hidden behind a click', async () => { + await renderDegradedSession(abandonedStreams(maxSelectorStreams)); + + // Expanded is the default: the compact toggle is what a COLLAPSED banner shows. + expect(screen.queryByTestId('cs-start-from-degraded-expand')).not.toBeInTheDocument(); + expect(screen.getByTestId('cs-start-from-degraded-topics')).toBeInTheDocument(); + expect(renderedTopics().length).toBeGreaterThan(0); + + await click('cs-start-from-degraded-collapse'); + expect(screen.getByTestId('cs-start-from-degraded-expand')).toBeInTheDocument(); + + await click('cs-start-from-degraded-expand'); + expect(screen.getByTestId('cs-start-from-degraded-topics')).toBeInTheDocument(); + }); + + it('offers the rest of the list, counted, and shows every name when asked', async () => { + const streams = abandonedStreams(maxSelectorStreams); + await renderDegradedSession(streams); + + const previewed = renderedTopics().length; + // Without this the assertion below is vacuous: a banner that rendered everything would offer + // "and 0 more" and satisfy it. + expect(previewed).toBeLessThan(maxSelectorStreams); + const showAll = screen.getByTestId('cs-start-from-degraded-show-all'); + // The affordance has to name what is still out of view, or the bound silently loses names. + expect(showAll.textContent).toContain(String(maxSelectorStreams - previewed)); + + await click('cs-start-from-degraded-show-all'); + + expect(renderedTopics()).toEqual(streams.map(topicOf)); + }); + + it('copies every abandoned topic, not only the ones on screen', async () => { + const streams = abandonedStreams(maxSelectorStreams); + await renderDegradedSession(streams); + + await click('cs-start-from-degraded-copy'); + + expect(clipboardWrites).toHaveLength(1); + expect(clipboardWrites[0]!.split('\n')).toEqual(streams.map(topicOf)); + }); + + it('shows a short list in full, with no "more" affordance to click', async () => { + const streams = abandonedStreams(3); + await renderDegradedSession(streams); + + expect(renderedTopics()).toEqual(streams.map(topicOf)); + expect(screen.queryByTestId('cs-start-from-degraded-show-all')).not.toBeInTheDocument(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.lifecycle.test.tsx b/ui/components/ui/ConsumerSession/ConsumerSession.lifecycle.test.tsx new file mode 100644 index 000000000..022dfa020 --- /dev/null +++ b/ui/components/ui/ConsumerSession/ConsumerSession.lifecycle.test.tsx @@ -0,0 +1,1153 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * Session LIFECYCLE, driven through the real component: pause, the resume stream's failure modes, + * and what Play does with a configuration that cannot be converted. + * + * Everything here is about the session claiming a state the SERVER is not in, which is only + * observable where the state machine, the RPCs and the stream meet - so the component is rendered + * for real and only the gRPC transport is replaced. `data-cs-state` on the session container is the + * state machine's own output, and it is what the Playwright specs assert too. + * + * The transport stub answers with genuine protobuf responses; the resume stream is a hand-rolled + * emitter with the same `on`/`removeListener`/`cancel` surface grpc-web's ClientReadableStream has, + * so listeners the component never installs simply never fire - exactly as in the browser. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); + +// There is no gRPC endpoint in jsdom, and the generated clients would try to reach one on import. +// The holder is mutable so each test installs its own answers; the name has to start with `mock` +// for jest's out-of-scope check to allow it inside the hoisted factory. +const mockClients = { current: undefined as unknown }; +jest.mock('../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +import React from 'react'; +import '@testing-library/jest-dom'; +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import ConsumerSession from './ConsumerSession'; +import { getDefaultManagedItem } from '../LibraryBrowser/default-library-items'; +import { Status } from '../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../grpc-web/google/rpc/code_pb'; +import { StringValue } from 'google-protobuf/google/protobuf/wrappers_pb'; +import { + CreateConsumerResponse, + DeleteConsumerResponse, + GetTopicPositionsResponse, + Message, + PauseResponse, + ResumeResponse, +} from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; +import { localStorageKeys } from '../../local-storage-keys'; + +const topicContext = (topicPersistency: 'persistent' | 'non-persistent') => ({ + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency, + topic: 'a-topic', + }, +}); + +const status = (code: number, message: string) => { + const s = new Status(); + s.setCode(code); + s.setMessage(message); + return s; +}; + +/** The `on`/`removeListener`/`cancel` surface of a grpc-web ClientReadableStream, plus an emitter. */ +const fakeResumeStream = () => { + const listeners: Record void)[]> = {}; + let isCancelled = false; + + return { + get isCancelled() { + return isCancelled; + }, + on(event: string, cb: (v: unknown) => void) { + (listeners[event] = listeners[event] || []).push(cb); + return this; + }, + removeListener(event: string, cb: (v: unknown) => void) { + listeners[event] = (listeners[event] || []).filter((it) => it !== cb); + return this; + }, + cancel() { + isCancelled = true; + }, + /** What the server said, delivered to whoever is listening - nobody, if nobody listens. */ + emit(event: string, v?: unknown) { + (listeners[event] || []).slice().forEach((cb) => cb(v)); + }, + }; +}; + +type HarnessOptions = { + pauseWith?: { code: number; message: string }; + pauseRejectsWith?: Error; + createConsumerWith?: { code: number; message: string }; + /** Per-attempt Create statuses; attempts past the end of the list succeed. */ + createConsumerStatuses?: { code: number; message: string }[]; + createRejectsWith?: Error; + /** + * Hold every Create open until `settleCreates()` - the server is still building the consumer. + * That window is where Stop, a hidden tab and an unload all have to behave. + */ + deferCreate?: boolean; + /** + * Hold every Pause open until `settlePauses()`. A Pause is not instant - it closes the intake and + * waits for the consumers - and the tab can come back inside that window, which is where the + * client's intent and the server's actual state can come apart. + */ + deferPause?: boolean; +}; + +const makeHarness = (options: HarnessOptions = {}) => { + const stream = fakeResumeStream(); + const createConsumerRequests: any[] = []; + const deleteConsumerRequests: any[] = []; + const pauseRequests: any[] = []; + const resumeRequests: any[] = []; + const resumeOptions: any[] = []; + const notifiedErrors: string[] = []; + const heldCreates: (() => void)[] = []; + const heldPauses: (() => void)[] = []; + + const consumerServiceClient = { + createConsumer: (req: unknown) => { + const attempt = createConsumerRequests.length; + createConsumerRequests.push(req); + if (options.createRejectsWith !== undefined) { + return Promise.reject(options.createRejectsWith); + } + const res = new CreateConsumerResponse(); + const s = options.createConsumerStatuses?.[attempt] + ?? options.createConsumerWith + ?? { code: Code.OK, message: '' }; + res.setStatus(status(s.code, s.message)); + if (!options.deferCreate) { + return Promise.resolve(res); + } + return new Promise((resolve) => heldCreates.push(() => resolve(res))); + }, + resume: (req: unknown, opts: unknown) => { + resumeRequests.push(req); + resumeOptions.push(opts); + return stream; + }, + pause: (req: unknown) => { + pauseRequests.push(req); + if (options.pauseRejectsWith !== undefined) { + return Promise.reject(options.pauseRejectsWith); + } + const res = new PauseResponse(); + const s = options.pauseWith ?? { code: Code.OK, message: '' }; + res.setStatus(status(s.code, s.message)); + if (!options.deferPause) { + return Promise.resolve(res); + } + return new Promise((resolve) => heldPauses.push(() => resolve(res))); + }, + deleteConsumer: (req: unknown) => { + deleteConsumerRequests.push(req); + return Promise.resolve(new DeleteConsumerResponse()); + }, + getTopicPositions: () => { + const res = new GetTopicPositionsResponse(); + res.setStatus(status(Code.OK, '')); + return Promise.resolve(res); + }, + // The target editor asks for the topics a selector resolves to; nothing here depends on it. + resolveTopicSelector: () => Promise.reject(new Error('not used by these tests')), + }; + + mockClients.current = { + consumerServiceClient, + // The Console's Produce tab creates a producer as soon as the session renders. + producerServiceClient: { + createProducer: () => Promise.resolve({ getStatus: () => status(Code.OK, '') }), + deleteProducer: () => Promise.resolve({ getStatus: () => status(Code.OK, '') }), + send: () => Promise.resolve({ getStatus: () => status(Code.OK, '') }), + }, + libraryServiceClient: { + listLibraryItems: () => Promise.reject(new Error('no library in these tests')), + getLibraryItem: () => Promise.reject(new Error('no library in these tests')), + }, + }; + + return { + stream, + createConsumerRequests, + deleteConsumerRequests, + pauseRequests, + resumeRequests, + resumeOptions, + notifiedErrors, + /** The Create the server was still working on finally answers. */ + settleCreates: async () => { + await act(async () => { + heldCreates.splice(0).forEach((resolve) => resolve()); + }); + }, + /** The Pause the server was still performing finally answers. */ + settlePauses: async () => { + await act(async () => { + heldPauses.splice(0).forEach((resolve) => resolve()); + }); + }, + }; +}; + +const defaultConfig = (context: ReturnType) => ({ + type: 'value' as const, + val: getDefaultManagedItem('consumer-session-config', context), +}); + +/** Renders the session; `rerenderWith` re-renders it in place, as a route change would. */ +const renderSession = async (config: unknown, context: ReturnType) => { + const tree = (ctx: ReturnType) => ( + + + + ); + + let rerender: any; + await act(async () => { + ({ rerender } = render(tree(context))); + }); + + return { + rerenderWith: async (ctx: ReturnType) => { + await act(async () => { + rerender(tree(ctx)); + }); + }, + }; +}; + +const sessionState = () => screen.getByTestId('cs-session').getAttribute('data-cs-state'); +const playButton = () => screen.getByTestId('cs-play') as HTMLButtonElement; +const stopButton = () => screen.getByTestId('cs-stop') as HTMLButtonElement; +const clickPlay = async () => { + await act(async () => { + fireEvent.click(playButton()); + }); +}; + +/** Stop and flush: the session is remounted from scratch under a new key. */ +const clickStop = async () => { + await act(async () => { + fireEvent.click(stopButton()); + }); +}; + +/** Play once from `new`, and wait for the create+resume round trip that lands it in `running`. */ +const startSession = async () => { + await clickPlay(); + expect(sessionState()).toBe('running'); +}; + +/** What `document.visibilityState` reports, plus the event the browser fires when it changes. */ +const setTabHidden = async (isHidden: boolean) => { + Object.defineProperty(document, 'visibilityState', { + configurable: true, + get: () => (isHidden ? 'hidden' : 'visible'), + }); + await act(async () => { + window.dispatchEvent(new Event('visibilitychange')); + }); +}; + +const consumerNames = (requests: any[]) => requests.map((req) => req.getConsumerName()); + +describe('a pause the server refuses', () => { + it('does not present the session as paused', async () => { + // FAILED_PRECONDITION is what ConsumerServiceImpl.pause answers for a session it no longer + // knows - and it comes back as a RESOLVED response, not a rejected call. The server stream is + // still live, so a "paused" label would be a claim about the server that is simply false, and + // messages can still arrive underneath it. + const harness = makeHarness({ pauseWith: { code: Code.FAILED_PRECONDITION, message: 'No such consumer' } }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await clickPlay(); + + expect(sessionState()).not.toBe('paused'); + // ...and it says so: the session is still consuming, which is what the server is doing. + expect(sessionState()).toBe('running'); + expect(harness.resumeOptions.length).toBeGreaterThanOrEqual(1); + }); + + it('does not present the session as paused when the pause call itself fails', async () => { + makeHarness({ pauseRejectsWith: new Error('transport down') }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await clickPlay(); + + expect(sessionState()).not.toBe('paused'); + }); + + it('still pauses when the server confirms it', async () => { + // The counterpart: the ordinary path must keep working, or "never claim paused" would be + // trivially satisfiable by never pausing at all. + makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await clickPlay(); + + expect(sessionState()).toBe('paused'); + }); +}); + +describe('the resume stream ending under the session', () => { + it('leaves a recoverable state when the stream errors, instead of a frozen "running"', async () => { + // A transport failure, a cancelled server call or an expired deadline all arrive as `error`. + // With no listener for it the session sits in `running` for ever, counters frozen, waiting for + // messages that can no longer come. + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('error', { code: 14, message: 'transport is closing' }); + }); + + expect(sessionState()).not.toBe('running'); + }); + + it('leaves a recoverable state when the stream ends normally', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('end'); + }); + + expect(sessionState()).not.toBe('running'); + }); + + it('does not cap the stream with a deadline a long skip would outlive', async () => { + // Resume is a long-lived server stream: a Skip-N over millions of messages can spend longer + // than any fixed budget resolving before it delivers anything, and the deadline would kill it + // mid-skip. + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + expect(harness.resumeOptions).toHaveLength(1); + expect(harness.resumeOptions[0]?.deadline).toBeUndefined(); + }); +}); + +describe('a configuration Play cannot execute', () => { + // Shape-check-passing but not convertible: `pauseTriggerChain` is missing, so + // consumerSessionConfigFromValOrRef throws and the session has no runtime config to send. + const unconvertibleConfig = () => { + const config = defaultConfig(topicContext('persistent')); + const spec = { ...(config.val as any).spec }; + delete spec.pauseTriggerChain; + return { type: 'value' as const, val: { ...(config.val as any), spec } }; + }; + + it('disables Play instead of letting the session hang on "initializing"', async () => { + makeHarness(); + await renderSession(unconvertibleConfig(), topicContext('persistent')); + + expect(playButton().disabled).toBe(true); + }); + + it('never asks the server to create a consumer it has no config for', async () => { + const harness = makeHarness(); + await renderSession(unconvertibleConfig(), topicContext('persistent')); + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(0); + expect(sessionState()).not.toBe('initializing'); + }); + + /** The same default config, with the start-from replaced by a message id of `hexString`. */ + const messageIdConfig = (hexString: string) => { + const config = defaultConfig(topicContext('persistent')); + const startFrom = (config.val as any).spec.startFrom; + startFrom.val.spec.startFrom = { + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString }, + }, + }, + }; + return config; + }; + + it.each([[''], [' ']])('never sends a message-id start-from of %p as zero bytes', async (hexString) => { + // The shared hex parser reads blank text as an EMPTY byte array - correct for a byte payload, + // meaningless as a start position. The server parses the field as a real message id and refuses + // it, so the round trip is spent to end up back where Play started, with nothing on screen + // saying which field was at fault. + const harness = makeHarness(); + await renderSession(messageIdConfig(hexString), topicContext('persistent')); + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(0); + expect(sessionState()).not.toBe('initializing'); + expect(playButton().disabled).toBe(false); + }); + + it('still sends a message id that is actually filled in', async () => { + const harness = makeHarness(); + await renderSession(messageIdConfig('08 c3 03 10 cd 04 20 00 30 01'), topicContext('persistent')); + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(1); + expect(sessionState()).toBe('running'); + }); + + it('recovers from a configuration that cannot be serialised into a request', async () => { + // A message id that is not hex. The runtime config converts fine - the id is still just text at + // that point - and the hex parser only runs while the create request is being built, after the + // click and outside any catch. The session was left sitting on "initializing" for ever. + const config = defaultConfig(topicContext('persistent')); + const startFrom = (config.val as any).spec.startFrom; + startFrom.val.spec.startFrom = { + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString: 'zz' }, + }, + }, + }; + + const harness = makeHarness(); + await renderSession(config, topicContext('persistent')); + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(0); + expect(sessionState()).not.toBe('initializing'); + }); + + it('keeps Play working for a configuration that does convert', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + expect(playButton().disabled).toBe(false); + await clickPlay(); + expect(harness.createConsumerRequests).toHaveLength(1); + }); +}); + +describe('the topic the session is mounted on', () => { + /** The topic FQNs the request's first target resolved "the current topic" to. */ + const requestedTopicFqns = (req: any) => + req + .getConsumerSessionConfig() + .getTargetsList()[0] + .getTopicSelector() + .getMultiTopicSelector() + .getTopicFqnsList(); + + it('sends the topic currently being viewed, not the one it was first rendered with', async () => { + // `persistent://t/n/x` and `non-persistent://t/n/x` are two different topics that differ only + // in the scheme. Navigating from one to the other changes nothing else about the page, so a + // session that captured the FQN once keeps consuming the topic the user left. + const harness = makeHarness(); + const { rerenderWith } = await renderSession( + defaultConfig(topicContext('persistent')), + topicContext('persistent') + ); + + await rerenderWith(topicContext('non-persistent')); + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(1); + expect(requestedTopicFqns(harness.createConsumerRequests[0])).toEqual([ + 'non-persistent://public/default/a-topic', + ]); + }); + + it('sends the mounted topic when nothing moved', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + + expect(requestedTopicFqns(harness.createConsumerRequests[0])).toEqual([ + 'persistent://public/default/a-topic', + ]); + }); +}); + +/** + * A Create that does not succeed leaves NOTHING running - no consumer on the server, no stream, no + * timer. The only thing left is the session's own claim about itself, and `initializing` is a claim + * that something is still happening. Play is disabled there, so the claim is also a dead end: the + * only way out was Stop, which throws away the loaded messages. + */ +describe('a Create the server does not complete', () => { + it('returns to a state Play can retry after a refused Create, instead of hanging on "initializing"', async () => { + // FAILED_PRECONDITION is what ConsumerServiceImpl answers for a config it cannot act on (an + // empty message id, a topic that vanished) - a RESOLVED response carrying a non-OK status. + const harness = makeHarness({ + createConsumerWith: { code: Code.FAILED_PRECONDITION, message: 'Message ID is empty' }, + }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(1); + expect(sessionState()).not.toBe('initializing'); + expect(playButton().disabled).toBe(false); + }); + + it('lets Play actually retry after a refused Create', async () => { + // Not merely "the button is enabled": the retry has to reach the server, which it cannot do + // from `initializing` (Play is a no-op there even when it is clickable). + const harness = makeHarness({ + createConsumerWith: { code: Code.FAILED_PRECONDITION, message: 'Message ID is empty' }, + }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(2); + }); + + it('returns to a state Play can retry when the Create call itself fails', async () => { + // A rejected call - transport down, deadline exceeded - never produces a response at all, and + // the `res === undefined` branch used to return without saying anything about the session. + const harness = makeHarness({ createRejectsWith: new Error('transport down') }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + + expect(sessionState()).not.toBe('initializing'); + expect(playButton().disabled).toBe(false); + + await clickPlay(); + expect(harness.createConsumerRequests).toHaveLength(2); + }); + + it('still runs the session when the Create succeeds', async () => { + // The counterpart: "never strand on initializing" is trivially satisfiable by never starting. + makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await startSession(); + }); +}); + +/** + * Stop is enabled while a Create is in flight, and it remounts the whole session. The consumer the + * server is still building belongs to a UI that no longer exists by the time it exists itself, and + * its name is generated in the component - so nothing else can ever name it again. + */ +describe('Stop while the session is still being created', () => { + it('deletes the consumer whose Create landed after the session was abandoned', async () => { + const harness = makeHarness({ deferCreate: true }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + expect(sessionState()).toBe('initializing'); + expect(harness.createConsumerRequests).toHaveLength(1); + + await clickStop(); + const createdName = harness.createConsumerRequests[0].getConsumerName(); + const deletesBeforeCreateLanded = harness.deleteConsumerRequests.length; + + await harness.settleCreates(); + + // The server now HAS this consumer, subscribed and consuming. Whatever the unmount already + // deleted, the late arrival has to be deleted too, or it stays live for ever. + expect(harness.deleteConsumerRequests.length).toBeGreaterThan(deletesBeforeCreateLanded); + expect(consumerNames(harness.deleteConsumerRequests.slice(deletesBeforeCreateLanded))).toContain(createdName); + }); + + it('deletes only the abandoned consumer, not the session that replaced it', async () => { + // Both Creates are in flight at once, and they are indistinguishable from the outside - the + // deletion has to follow which SESSION asked for each one, not "some create was abandoned". + const harness = makeHarness({ deferCreate: true }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + await clickStop(); + await clickPlay(); + expect(harness.createConsumerRequests).toHaveLength(2); + const deletesBeforeCreatesLanded = harness.deleteConsumerRequests.length; + + await harness.settleCreates(); + + expect(harness.deleteConsumerRequests.length - deletesBeforeCreatesLanded).toBe(1); + expect(sessionState()).toBe('running'); + }); + + it('does not delete the consumer a retry created', async () => { + // Returning to `new` after a refused Create is itself a cleanup, so a session that retries has + // been "abandoned" once by the time its second Create lands. Reading that as "this consumer is + // orphaned" deletes the one consumer the user is actually watching. + const harness = makeHarness({ + createConsumerStatuses: [{ code: Code.FAILED_PRECONDITION, message: 'Message ID is empty' }], + }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + const deletesBeforeRetry = harness.deleteConsumerRequests.length; + + await clickPlay(); + + expect(harness.createConsumerRequests).toHaveLength(2); + expect(sessionState()).toBe('running'); + expect(harness.deleteConsumerRequests.length).toBe(deletesBeforeRetry); + }); +}); + +/** + * The tab going away pauses a RUNNING stream so the browser does not silently stall it. Every other + * state owns no stream: `new` and `initializing` have no consumer on the server yet, and `paused` + * already stopped. Pausing those asks the server about a session it does not know, and its refusal + * used to be read as "then it must still be running". + */ +describe('the tab being hidden', () => { + afterEach(() => setTabHidden(false)); + + it('does not pause a session that was never started', async () => { + const harness = makeHarness({ + pauseWith: { code: Code.FAILED_PRECONDITION, message: 'No such consumer consumer session' }, + }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await setTabHidden(true); + + expect(harness.pauseRequests).toHaveLength(0); + // ...and above all it is not "running": a refused pause flipped the session into the one state + // that offers Resume, for a consumer that was never created. + expect(sessionState()).toBe('new'); + }); + + it('does not pause a session whose Create is still in flight', async () => { + const harness = makeHarness({ deferCreate: true }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await clickPlay(); + await setTabHidden(true); + + expect(harness.pauseRequests).toHaveLength(0); + expect(sessionState()).toBe('initializing'); + + // ...and the Create that lands while the tab is STILL HIDDEN must not resume into it: a + // stream nobody watches, with nothing armed for the return, used to stall the session + // forever. It parks as a hidden-tab pause instead, and becoming visible resumes it. + await harness.settleCreates(); + expect(sessionState()).toBe('paused'); + expect(harness.resumeRequests).toHaveLength(0); + + await setTabHidden(false); + expect(sessionState()).toBe('running'); + expect(harness.resumeRequests).toHaveLength(1); + }); + + it('does not pause a session that is already paused', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + await clickPlay(); + expect(sessionState()).toBe('paused'); + const pausesBefore = harness.pauseRequests.length; + + await setTabHidden(true); + + expect(harness.pauseRequests.length).toBe(pausesBefore); + expect(sessionState()).toBe('paused'); + }); + + it('still pauses a running session, and resumes it when the tab comes back', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await setTabHidden(true); + expect(harness.pauseRequests).toHaveLength(1); + expect(sessionState()).toBe('paused'); + + await setTabHidden(false); + expect(sessionState()).toBe('running'); + }); + + /** + * The window between "Pause sent" and "Pause answered". A Pause is not instant on the server, and + * a tab can be hidden and shown again inside that window - alt-tab, a notification, a screen + * lock. + * + * Abandoning the in-flight Pause's response and sending a Resume immediately does NOT make the + * server agree: the per-session lifecycle lock is not FIFO, so a Resume sent second can be + * granted first and the older Pause then lands last. The session ends up paused on the server + * while the client says Running - and nothing further arrives to correct it, because a paused + * consumer sends nothing. + * + * So the client serializes its own intent: the return is recorded, and the Resume goes out only + * once the Pause it would race has settled. + */ + it('does not race a Resume against a Pause the server has not answered yet', async () => { + const harness = makeHarness({ deferPause: true }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + expect(harness.resumeRequests).toHaveLength(1); + + await setTabHidden(true); + expect(harness.pauseRequests).toHaveLength(1); + // The server is still pausing: nothing has confirmed it yet. + expect(sessionState()).toBe('pausing'); + + await setTabHidden(false); + + // The intent is recorded, not sent: a second Resume here is the one that can overtake the + // Pause on the wire and leave the server paused underneath a Running session. + expect(harness.resumeRequests).toHaveLength(1); + + await harness.settlePauses(); + + // With the Pause settled the operations can no longer cross, so the session resumes - and what + // it shows now is what the server is really doing. + expect(harness.resumeRequests).toHaveLength(2); + expect(sessionState()).toBe('running'); + }); + + it('stays paused when the tab is hidden again before the Pause settles', async () => { + // The recorded return is withdrawn by the tab going away again: the last thing the user did is + // still "leave", and the Pause already in flight is exactly what that wants. + const harness = makeHarness({ deferPause: true }); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await setTabHidden(true); + await setTabHidden(false); + await setTabHidden(true); + await harness.settlePauses(); + + expect(sessionState()).toBe('paused'); + expect(harness.resumeRequests).toHaveLength(1); + + // ...and the ordinary return still resumes it, from a Pause that has settled. + await setTabHidden(false); + expect(sessionState()).toBe('running'); + expect(harness.resumeRequests).toHaveLength(2); + }); + + it('leaves a session the user paused paused when the tab comes back', async () => { + // Coming back must not restart something the USER stopped - only what the hidden tab stopped. + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + await clickPlay(); + expect(sessionState()).toBe('paused'); + + await setTabHidden(true); + await setTabHidden(false); + + expect(sessionState()).toBe('paused'); + expect(harness.resumeOptions).toHaveLength(1); + }); +}); + +/** + * The retention that keeps a long session from growing until the tab dies. It is the only bound on + * the message buffer, it runs on a timer rather than per message, and nothing else in the tree + * exercises it - the count of rendered rows is not the count retained, because the table is + * virtualized. + */ +describe('the number of messages kept on screen', () => { + beforeEach(() => { + jest.useFakeTimers(); + // The flush scrolls the table to the bottom; jsdom has no scrollTo on elements. + (Element.prototype as any).scrollTo = () => undefined; + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + /** The same default config with an explicit display limit. */ + const configWithLimit = (numDisplayItems: number) => { + const config = defaultConfig(topicContext('persistent')); + (config.val as any).spec.numDisplayItems = numDisplayItems; + return config; + }; + + /** A resume response carrying `count` real (valued) messages, numbered from `from`. */ + const messages = (from: number, count: number) => { + const res = new ResumeResponse(); + res.setStatus(status(Code.OK, '')); + res.setMessagesList( + Array.from({ length: count }, (_, i) => { + const m = new Message(); + m.setValue(new StringValue().setValue(`m-${from + i}`)); + m.setNumMessageProcessed(from + i); + m.setNumMessageSent(from + i); + return m; + }) + ); + return res; + }; + + const retained = () => Number(screen.getByTestId('cs-session').getAttribute('data-cs-retained')); + + /** Let the per-second rate tick, then the flush that moves the buffer into the table. */ + const flush = async () => { + await act(async () => { + jest.advanceTimersByTime(1000); + }); + await act(async () => { + jest.advanceTimersByTime(500); + }); + }; + + it('keeps everything that arrives while the limit is not reached', async () => { + const harness = makeHarness(); + await renderSession(configWithLimit(5), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('data', messages(1, 3)); + }); + await flush(); + + expect(retained()).toBe(3); + }); + + it('drops the oldest once more than the limit has arrived', async () => { + const harness = makeHarness(); + await renderSession(configWithLimit(5), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('data', messages(1, 12)); + }); + await flush(); + + expect(retained()).toBe(5); + }); + + it('keeps the limit across several deliveries, rather than only within one', async () => { + const harness = makeHarness(); + await renderSession(configWithLimit(5), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('data', messages(1, 4)); + }); + await flush(); + await act(async () => { + harness.stream.emit('data', messages(5, 4)); + }); + await flush(); + + expect(retained()).toBe(5); + }); + + // A saved session carrying a limit of 0 (what an emptied field used to commit) once turned + // `slice(-limit)` into `slice(0)` and removed the only bound the session has. The runtime proof + // for that case had to deliver PAST the default limit - the distinction is invisible below it - + // which stopped being feasible here when the default became 1,000,000 (owner instruction, + // 2026-08-11): a million pb messages through a jsdom stream is not a test, it is a hang. What + // remains pinned: `displayItemLimit` maps 0/-5/2.5/NaN/Infinity to the safe default (unit tests + // beside it), and the retention path consults it (the small-limit test above). + +}); + +/** + * The unload cleanup is a listener on `window`, and `window` outlives every session. Registering it + * from the initialize path meant nothing ever removed it: each Stop remount left one more stale + * closure installed, each holding a dead session's consumer name. + */ +describe('the unload cleanup', () => { + it('deletes only the session that is on screen, however often the session was restarted', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await startSession(); + await clickStop(); + await startSession(); + await clickStop(); + await startSession(); + + const deletesBeforeUnload = harness.deleteConsumerRequests.length; + await act(async () => { + window.dispatchEvent(new Event('beforeunload')); + }); + + expect(harness.deleteConsumerRequests.length - deletesBeforeUnload).toBe(1); + }); + + it('deletes nothing on unload for a session that was never started', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + await act(async () => { + window.dispatchEvent(new Event('beforeunload')); + }); + + expect(harness.deleteConsumerRequests).toHaveLength(0); + }); +}); + +describe('the browser-wide More tools visibility', () => { + afterEach(() => { + cleanup(); + window.localStorage.clear(); + }); + + it('is OPEN by default, and a STORED preference wins in both directions', async () => { + // Owner decision (2026-08-11, reversing the closed-by-default from earlier the same day): + // a browser with no stored preference starts with the tools pane open. + // + // Driven through STORAGE rather than the toolbar toggle on purpose: on a fresh mount the + // healing effect writes the default asynchronously, so a click racing that write gets undone - + // a jsdom ordering artifact, not a product behaviour. The toggle itself is covered by the + // malformed-value cell below, which clicks it once storage has settled. + window.localStorage.clear(); + makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + expect(await screen.findByTestId('cs-tools-resize-handle')).toBeInTheDocument(); + await waitFor(() => { + expect(window.localStorage.getItem(localStorageKeys.consumerSessionToolsOpen)).toBe('true'); + }); + + // Closed once, it STAYS closed across a remount - the default never reasserts itself. + cleanup(); + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'false'); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + expect(screen.queryByTestId('cs-tools-resize-handle')).toBeNull(); + + // ...and re-opened, it stays open. + cleanup(); + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'true'); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + expect(await screen.findByTestId('cs-tools-resize-handle')).toBeInTheDocument(); + }); + + it('falls back to the default and heals a malformed saved visibility value', async () => { + // The point of the fallback is that corrupt storage cannot leave the pane in an unreachable + // state - it lands on the default (open) and REWRITES storage, so the next read is well-formed. + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, JSON.stringify('not-a-boolean')); + makeHarness(); + + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + + expect(await screen.findByTestId('cs-tools-resize-handle')).toBeInTheDocument(); + await waitFor(() => { + expect(window.localStorage.getItem(localStorageKeys.consumerSessionToolsOpen)).toBe('true'); + }); + // ...and it can still be closed from the toolbar, which is what "stuck" would mean. + fireEvent.click(screen.getByTestId('cs-tools-close')); + await waitFor(() => expect(screen.queryByTestId('cs-tools-resize-handle')).toBeNull()); + }); +}); + +describe('the browser-wide delivery controls', () => { + // Both live in localStorage and must be scrubbed, or one test's setting becomes the next + // test's surprise. + afterEach(() => window.localStorage.clear()); + + /** A data frame whose trailing counters put the LOADED count at `sent`. */ + const dataFrame = (sent: number) => { + const res = new ResumeResponse(); + res.setStatus(status(Code.OK, '')); + const msg = new Message(); + const value = new StringValue(); + value.setValue('{"a":1}'); + msg.setValue(value); + msg.setNumMessageProcessed(sent); + msg.setNumMessageSent(sent); + res.setMessagesList([msg]); + return res; + }; + + it('the rate limit rides every resume request, straight from localStorage', async () => { + window.localStorage.setItem('consumerSessionRateLimit', '250'); + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + expect(harness.resumeRequests).toHaveLength(1); + expect(harness.resumeRequests[0].getMaxMessagesPerSecond()).toBe(250); + }); + + it('the pause-after threshold rides every resume request as the server-side delivery budget', async () => { + window.localStorage.setItem('consumerSessionPauseAfterLoaded', '10'); + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + // The server enforces "at most 10 loaded on this stream"; the client threshold below is only + // the state-machine driver that turns the quiet stream into a paused session. + expect(harness.resumeRequests[0].getMaxMessagesToDeliver()).toBe(10); + }); + + it('no setting means UNLIMITED on the wire - zero, not a stale number', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + expect(harness.resumeRequests[0].getMaxMessagesPerSecond()).toBe(0); + }); + + it('pauses itself when n more messages have loaded - the same pause the button sends', async () => { + window.localStorage.setItem('consumerSessionPauseAfterLoaded', '3'); + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + // Two loaded: below the threshold, nothing happens. + await act(async () => { + harness.stream.emit('data', dataFrame(2)); + }); + expect(harness.pauseRequests).toHaveLength(0); + expect(sessionState()).toBe('running'); + + // The third crosses it: the session pauses ITSELF. + await act(async () => { + harness.stream.emit('data', dataFrame(3)); + }); + expect(harness.pauseRequests).toHaveLength(1); + expect(sessionState()).toBe('paused'); + }); + + it('one crossing fires exactly one pause, however many chunks arrive during it', async () => { + window.localStorage.setItem('consumerSessionPauseAfterLoaded', '3'); + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + // Both frames are past the threshold; the second lands while the pause is in flight and must + // not send a second one. + await act(async () => { + harness.stream.emit('data', dataFrame(3)); + harness.stream.emit('data', dataFrame(4)); + }); + + expect(harness.pauseRequests).toHaveLength(1); + }); + + it('re-arms on resume: Play loads the NEXT n and pauses again', async () => { + window.localStorage.setItem('consumerSessionPauseAfterLoaded', '3'); + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('data', dataFrame(3)); + }); + expect(sessionState()).toBe('paused'); + + // Play again: the threshold is re-armed at loaded + n = 6, so 5 is quiet and 6 pauses. + await clickPlay(); + expect(sessionState()).toBe('running'); + await act(async () => { + harness.stream.emit('data', dataFrame(5)); + }); + expect(sessionState()).toBe('running'); + await act(async () => { + harness.stream.emit('data', dataFrame(6)); + }); + + expect(harness.pauseRequests).toHaveLength(2); + }); + + it('no threshold set means the session NEVER pauses itself', async () => { + const harness = makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await act(async () => { + harness.stream.emit('data', dataFrame(1000)); + }); + + expect(harness.pauseRequests).toHaveLength(0); + expect(sessionState()).toBe('running'); + }); +}); + +/** + * What a pause COSTS, said where the pause happened. + * + * A paused session hands its prefetched messages back to the broker and picks them up again on + * resume - on a PERSISTENT topic. A non-persistent topic keeps no log at all, so whatever is + * published while the session is paused exists nowhere afterwards: resuming shows only what comes + * next. The session used to present both cases identically, so a user watching a non-persistent + * topic could pause believing the pause was free. + * + * The disclosure is the paused state's own, not a permanent caveat: it appears when a pause is + * actually in effect over topics that cannot retain anything, and nowhere else. + */ +describe('pausing a session that reads non-persistent topics', () => { + const note = () => screen.queryByTestId('cs-paused-non-persistent-note'); + + it('says the paused gap is not recoverable', async () => { + makeHarness(); + await renderSession(defaultConfig(topicContext('non-persistent')), topicContext('non-persistent')); + await startSession(); + + await clickPlay(); + expect(sessionState()).toBe('paused'); + + // Both halves have to be on screen: WHICH property of the topics causes it, and WHAT it costs. + // Either alone reads as a description of a paused session rather than as a warning. + expect(note()).toBeInTheDocument(); + expect(note()!.textContent).toMatch(/non-persistent/i); + expect(note()!.textContent).toMatch(/paused/i); + }); + + it('says nothing while the session is running - a running session misses nothing', async () => { + makeHarness(); + await renderSession(defaultConfig(topicContext('non-persistent')), topicContext('non-persistent')); + await startSession(); + + expect(note()).not.toBeInTheDocument(); + }); + + it('does not claim it on a persistent topic, where the pause really does lose nothing', async () => { + // The counterpart that keeps the warning meaningful: a disclosure shown on every pause is + // noise, and noise is what gets ignored on the one topic where it matters. + makeHarness(); + await renderSession(defaultConfig(topicContext('persistent')), topicContext('persistent')); + await startSession(); + + await clickPlay(); + expect(sessionState()).toBe('paused'); + + expect(note()).not.toBeInTheDocument(); + }); + + it('retires the disclosure when the session resumes', async () => { + makeHarness(); + await renderSession(defaultConfig(topicContext('non-persistent')), topicContext('non-persistent')); + await startSession(); + + await clickPlay(); + expect(note()).toBeInTheDocument(); + + await clickPlay(); + expect(sessionState()).toBe('running'); + expect(note()).not.toBeInTheDocument(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.module.css b/ui/components/ui/ConsumerSession/ConsumerSession.module.css index 6b20a47b9..d4622f925 100644 --- a/ui/components/ui/ConsumerSession/ConsumerSession.module.css +++ b/ui/components/ui/ConsumerSession/ConsumerSession.module.css @@ -41,20 +41,29 @@ .Content { display: grid; - grid-template-columns: 1fr min-content; + grid-template-columns: minmax(0, 1fr); + /* The inspector tabs have very different intrinsic heights. Keep them inside the session's + fixed middle row so switching tabs cannot resize Virtuoso and disturb its scroll position. */ + grid-template-rows: minmax(0, 1fr); + min-height: 0; flex: 1; overflow: hidden; } +.ContentWithMessageDetails { + /* Keep a reachable share of the viewport for the table when a preference saved on a large + display is restored on a smaller one. The hook still retains the preferred pixel width. */ + grid-template-columns: minmax(0, 1fr) minmax(0, min(var(--message-details-width), 85%)); +} + .MessageDetails { position: relative; - overflow: hidden; + overflow: visible; display: flex; border-left: 1px solid var(--border-color); - width: 600rem; - - /* Prevent accidental pages history navigation using touch-pad gestures. */ - overscroll-behavior-x: contain; + width: 100%; + min-width: 0; + min-height: 0; } .CloseMessageDetails { @@ -83,3 +92,133 @@ /* Prevent accidental pages history navigation using touch-pad gestures. */ overscroll-behavior-x: contain; } + +/* The toolbar and anything disclosed under it, as ONE grid child: the session grid declares + exactly three rows, so extra direct children would shift the content into the console's row. */ +.Top { + display: flex; + flex-direction: column; + min-width: 0; +} + +/* The sticky best-effort banner: a degraded start-from stays disclosed for the session's life. */ +.StartFromDegraded { + padding: 6rem 12rem; + background: var(--warning-background-color, #fff7e0); + color: var(--warning-text-color, #7a5b00); + border-bottom: 1rem solid var(--border-color, #e0d5a8); + font-size: 12rem; +} + +.StartFromDegradedTopics { + margin: 4rem 0; + padding-left: 20rem; + max-height: 80rem; + overflow-y: auto; +} + +/* The banner's affordances: reveal the rest of the bounded list, copy it all, collapse. */ +.StartFromDegradedActions { + display: flex; + flex-wrap: wrap; + gap: 12rem; +} + +.StartFromDegradedToggle { + background: none; + border: none; + padding: 0; + font-size: 12rem; + color: inherit; + text-decoration: underline; + cursor: pointer; +} + +/* The guaranteed replay's boundary pause, DOCKED by the session itself - not a toast. It lived in + a persistent notification first, and react-toastify's lifecycle made every boundary hand-off a + race: removal is animation-mediated (a dismissed toast lingers in the DOM past the dismiss + call) and a toast created under a still-exiting id is silently dropped, so resume-then-recatch + either stacked two panels (an instant catch-up re-announces within milliseconds - e2e + CS-DM-R3B) or swallowed the new one (CS-DM-R2). Rendered conditionally by the component, at + most one panel can exist, closing is plain state, and leaving the page takes it along. Fixed at + bottom-right - the position the notification had - and out of layout, so the session grid never + learns it is there. */ +.ReplayCaughtUpDock { + position: fixed; + bottom: 16rem; + right: 16rem; + /* Above the table's sticky cells (999, which resolve in the same root stacking context) and + where the notification layer used to paint; below tooltips (9999) - the panel's own button + titles must render on top of it. */ + z-index: 1000; + animation: replay-caught-up-in 0.2s ease-out both; +} + +@keyframes replay-caught-up-in { + from { opacity: 0; transform: translateX(24rem); } + to { opacity: 1; transform: none; } +} + +/* The panel owns its surface now (the notification used to paint it): the app's white card, sized + to its content - the two action buttons side by side set the width - and capped so a long topic + list cannot grow it across the viewport. Column with a single gap so every optional line spaces + itself; the actions are the shared SmallButton, so no button styling lives here. The right + padding clears the close button riding the top-right corner. */ +.ReplayCaughtUp { + position: relative; + display: flex; + flex-direction: column; + gap: 4rem; + padding: 8rem 30rem 8rem 12rem; + font-size: 12rem; + background: var(--background-color, #fff); + color: var(--text-color); + border: 1rem solid var(--border-color, #d9dbe3); + border-radius: 8rem; + box-shadow: 0rem 2rem 4rem rgb(0 0 0 / 27%); + width: max-content; + max-width: min(640rem, calc(100vw - 48rem)); +} + +.ReplayCaughtUpClose { + position: absolute; + top: 4rem; + right: 4rem; +} + +/* What HAPPENED, at a glance and above everything else: the panel is read at a distance, and the + two lines under it are the detail for whoever wants it. Set in the notification's own size + rather than the panel's dense 12rem so it carries as a heading. */ +.ReplayCaughtUpTitle { + font-size: 15rem; + font-weight: 700; + margin-bottom: 2rem; +} + +/* WHEN it happened - the fact most likely to be acted on, so it keeps weight of its own. */ +.ReplayCaughtUpHeadline { + font-weight: 600; +} + +/* One ROW, always: the two ways on are peers, and stacking them read as a list of steps. The + panel is sized to fit them (.ReplayCaughtUp's max-content width). */ +.ReplayCaughtUpActions { + display: flex; + flex-wrap: nowrap; + gap: 8rem; + margin-top: 2rem; +} + +/* What a pause costs on topics that retain nothing - shown only while the pause is in effect. */ +.NonPersistentPauseNote { + padding: 6rem 12rem; + background: var(--warning-background-color, #fff7e0); + color: var(--warning-text-color, #7a5b00); + border-bottom: 1rem solid var(--border-color, #e0d5a8); + font-size: 12rem; +} + +/* Drop-position indicator while a header drag hovers this column. */ +.ThDragOver { + box-shadow: inset 3rem 0 0 0 var(--accent-color, #4a72ff); +} diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.replay.test.tsx b/ui/components/ui/ConsumerSession/ConsumerSession.replay.test.tsx new file mode 100644 index 000000000..cb676c5ff --- /dev/null +++ b/ui/components/ui/ConsumerSession/ConsumerSession.replay.test.tsx @@ -0,0 +1,368 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The ROW-LEVEL disclosures of the guaranteed replay: the per-row seam-violation marker and the + * switch-point marker left behind by "Switch to Best effort and follow live". + * + * Both live INSIDE the virtualized message table, which jsdom cannot lay out - so, exactly like + * the columns suite, react-virtuoso is replaced with a plain table that renders every row through + * the session's own `itemContent`. The rows are the real MessageComponent cells; only the + * virtualization is gone. That substitution is also why the markers here are DATA-DRIVEN by + * design: a marker attached to the row's message survives scrolling, retention trimming and + * re-sorting because it re-renders with the row, wherever the row is. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); + +// No gRPC endpoint in jsdom, and the generated clients would try to reach one on import. +const mockClients = { current: undefined as unknown }; +jest.mock('../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +// The message table virtualizes rows and measures its viewport; jsdom lays nothing out, so the +// header AND every row are rendered into a plain table here (the columns suite renders the header +// the same way). `itemContent` is the session's real row renderer. +jest.mock('react-virtuoso', () => { + const ReactRuntime = require('react'); + return { + TableVirtuoso: ReactRuntime.forwardRef((props: any, _ref: unknown) => + ReactRuntime.createElement( + 'table', + null, + ReactRuntime.createElement('thead', null, props.fixedHeaderContent()), + ReactRuntime.createElement( + 'tbody', + null, + (props.data ?? []).map((message: unknown, i: number) => + ReactRuntime.createElement('tr', { key: i, 'data-testid': 'cs-rendered-row' }, props.itemContent(i, message))) + ) + )), + }; +}); + +import React from 'react'; +import '@testing-library/jest-dom'; +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import * as Notifications from '../../app/contexts/Notifications'; +import ConsumerSession from './ConsumerSession'; +import { getDefaultManagedItem } from '../LibraryBrowser/default-library-items'; +import { localStorageKeys } from '../../local-storage-keys'; +import { Status } from '../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../grpc-web/google/rpc/code_pb'; +import { Int32Value, StringValue } from 'google-protobuf/google/protobuf/wrappers_pb'; +import { + ConsumerStats, + CreateConsumerResponse, + DeleteConsumerResponse, + Message, + PauseResponse, + ResumeResponse, + SetDeliveryOrderResponse, +} from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; + +const topicContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, +}; + +const status = (code: number) => new Status().setCode(code).setMessage(''); + +/** The `on`/`removeListener`/`cancel` surface of a grpc-web ClientReadableStream, plus an emitter. */ +const fakeResumeStream = () => { + const listeners: Record void)[]> = {}; + return { + on(event: string, cb: (v: unknown) => void) { + (listeners[event] = listeners[event] || []).push(cb); + return this; + }, + removeListener(event: string, cb: (v: unknown) => void) { + listeners[event] = (listeners[event] || []).filter((it) => it !== cb); + return this; + }, + cancel() { }, + emit(event: string, v?: unknown) { + (listeners[event] || []).slice().forEach((cb) => cb(v)); + }, + }; +}; + +/** A valued message numbered `n`, optionally flagged as a cross-seam ordering violation. */ +const message = (n: number, opts: { seamViolation?: boolean } = {}) => { + const m = new Message(); + m.setValue(new StringValue().setValue(`m-${n}`)); + m.setNumMessageProcessed(n); + m.setNumMessageSent(n); + // The server stamps every delivered message with its target; the row renderer dereferences it. + m.setSessionTargetIndex(new Int32Value().setValue(0)); + if (opts.seamViolation) { + m.setDeliveredOutOfOrder(true); + } + return m; +}; + +const dataFrame = (messages: ReturnType[]) => + new ResumeResponse().setStatus(status(Code.OK)).setMessagesList(messages); + +/** The message-less stats frame the server pushes when the replay reaches its boundary. */ +const caughtUpFrame = () => { + const stats = new ConsumerStats(); + stats.setDeliveryOrderActive(true); + stats.setReplayCaughtUp(true); + stats.setReplayBoundaryAtMs(1_754_700_000_000); + return new ResumeResponse().setStatus(status(Code.OK)).setConsumerStats(stats); +}; + +const makeHarness = () => { + const streams: Array> = []; + const okStatus = { getStatus: () => ({ getCode: () => Code.OK, getMessage: () => '' }) }; + mockClients.current = { + consumerServiceClient: { + createConsumer: () => Promise.resolve(new CreateConsumerResponse().setStatus(status(Code.OK))), + resume: () => { + const stream = fakeResumeStream(); + streams.push(stream); + return stream; + }, + pause: () => Promise.resolve(new PauseResponse().setStatus(status(Code.OK))), + deleteConsumer: () => Promise.resolve(new DeleteConsumerResponse()), + setDeliveryOrder: () => Promise.resolve(new SetDeliveryOrderResponse().setStatus(status(Code.OK))), + resolveTopicSelector: () => Promise.reject(new Error('not used by these tests')), + }, + // The Tools panel's Produce tab creates a producer as soon as the session renders. + producerServiceClient: { + createProducer: () => Promise.resolve(okStatus), + deleteProducer: () => Promise.resolve(okStatus), + send: () => Promise.resolve(okStatus), + }, + libraryServiceClient: { + listLibraryItems: () => Promise.reject(new Error('no library in these tests')), + getLibraryItem: () => Promise.reject(new Error('no library in these tests')), + }, + }; + return { streams }; +}; + +/** Renders the session with an explicit Guaranteed order and plays it. */ +const renderRunningSession = async () => { + const harness = makeHarness(); + const config = getDefaultManagedItem('consumer-session-config', topicContext) as { + spec: { messageDeliveryOrder?: string }; + }; + // Pinned explicitly, like the e2e matrix does: the replay scenarios are Guaranteed's. + config.spec.messageDeliveryOrder = 'guaranteed'; + + await act(async () => { + render( + + {/* The caught-up panel - which carries the switch this suite clicks - is rendered by the + toast container since 2026-08-11, so the provider has to be mounted here too. */} + + + + + ); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-play')); + }); + + return harness; +}; + +/** Let the per-second gauge tick, then the flush that moves the buffer into the table. */ +const flush = async () => { + await act(async () => { + jest.advanceTimersByTime(1000); + }); + await act(async () => { + jest.advanceTimersByTime(500); + }); +}; + +const sessionState = () => screen.getByTestId('cs-session').getAttribute('data-cs-state'); +const rowTexts = () => screen.getAllByTestId('cs-rendered-row').map((row) => row.textContent ?? ''); + +describe('the per-row out-of-order marker', () => { + beforeEach(() => { + jest.useFakeTimers(); + (Element.prototype as { scrollTo?: () => void }).scrollTo = () => undefined; + window.localStorage.clear(); + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'false'); + }); + + afterEach(() => { + cleanup(); + jest.useRealTimers(); + window.localStorage.clear(); + }); + + it('marks exactly the flagged row, and explains the flag in plain language', async () => { + const { streams } = await renderRunningSession(); + + await act(async () => { + streams[0].emit('data', dataFrame([message(1), message(2, { seamViolation: true }), message(3)])); + }); + await flush(); + + expect(screen.getAllByTestId('cs-rendered-row')).toHaveLength(3); + const markers = screen.getAllByTestId('cs-out-of-order-marker'); + expect(markers).toHaveLength(1); + // The marker sits on the row that carried the flag - m-2, not its neighbours. + expect(markers[0].closest('tr')?.textContent).toContain('m-2'); + // The explanation is on the marker itself, in the user's terms - through the app-wide + // tooltip since 2026-08-11 (a native title rendered nothing readable), and mode-aware: this + // session runs Guaranteed, so the causes are the stored inversion and the pause seam. + const tooltip = markers[0].getAttribute('data-tooltip-html') ?? ''; + expect(markers[0].getAttribute('title')).toBeNull(); + expect(tooltip).toMatch(/out of order/i); + expect(tooltip).toMatch(/several producers/i); + expect(tooltip).toMatch(/pause/i); + }); + + it('marks nothing when no message carries the flag - absent field means no marker', async () => { + const { streams } = await renderRunningSession(); + + await act(async () => { + streams[0].emit('data', dataFrame([message(1), message(2)])); + }); + await flush(); + + expect(screen.getAllByTestId('cs-rendered-row')).toHaveLength(2); + expect(screen.queryByTestId('cs-out-of-order-marker')).toBeNull(); + }); +}); + +/** + * The switch-point marker (the plan's OPEN DETAIL, decided during this work): after "Continue + * live with Best effort", rows above the switch are the exact replay and rows below are + * grace-ordered - the transition may not ship unmarked. The marker is a divider band attached to + * the FIRST live-delivered row, driven by the session's monotonic processed counter: it renders + * wherever that row renders, so it survives scrolling, retention trimming and re-sorting rather + * than being a DOM insertion at a fixed offset. + */ +describe('the delivery-order switch-point marker', () => { + beforeEach(() => { + jest.useFakeTimers(); + (Element.prototype as { scrollTo?: () => void }).scrollTo = () => undefined; + window.localStorage.clear(); + window.localStorage.setItem(localStorageKeys.consumerSessionToolsOpen, 'false'); + }); + + afterEach(() => { + cleanup(); + jest.useRealTimers(); + window.localStorage.clear(); + }); + + const switchToLive = async () => { + // The switch rides the caught-up panel, which the toast container mounts a tick after the + // boundary lands - so it is FOUND, not assumed present. + const button = await screen.findByTestId('cs-replay-continue-best-effort'); + await act(async () => { + fireEvent.click(button); + }); + }; + + it('appears exactly at the transition: the first row delivered after the switch', async () => { + const { streams } = await renderRunningSession(); + + // The replay chunk: two rows, then the boundary. + await act(async () => { + streams[0].emit('data', dataFrame([message(1), message(2)])); + }); + await flush(); + await act(async () => { + streams[0].emit('data', caughtUpFrame()); + }); + expect(sessionState()).toBe('paused'); + + await switchToLive(); + expect(sessionState()).toBe('running'); + + // Live rows arrive on the NEW stream - the switch resumed delivery. + await act(async () => { + streams[streams.length - 1].emit('data', dataFrame([message(3), message(4)])); + }); + await flush(); + + expect(rowTexts()).toHaveLength(4); + const markers = screen.getAllByTestId('cs-order-switch-point'); + expect(markers).toHaveLength(1); + // On the first LIVE row - m-3 - not on the last replay row and not on later live rows. + expect(markers[0].closest('tr')?.textContent).toContain('m-3'); + expect(markers[0].closest('tr')?.textContent).not.toContain('m-2'); + }); + + it('stays on that row as more live rows arrive - one transition, one marker', async () => { + const { streams } = await renderRunningSession(); + + await act(async () => { + streams[0].emit('data', dataFrame([message(1)])); + }); + await flush(); + await act(async () => { + streams[0].emit('data', caughtUpFrame()); + }); + await switchToLive(); + + await act(async () => { + streams[streams.length - 1].emit('data', dataFrame([message(2)])); + }); + await flush(); + await act(async () => { + streams[streams.length - 1].emit('data', dataFrame([message(3), message(4)])); + }); + await flush(); + + const markers = screen.getAllByTestId('cs-order-switch-point'); + expect(markers).toHaveLength(1); + expect(markers[0].closest('tr')?.textContent).toContain('m-2'); + }); + + it('marks nothing while the session never switched', async () => { + const { streams } = await renderRunningSession(); + + await act(async () => { + streams[0].emit('data', dataFrame([message(1), message(2)])); + }); + await flush(); + + expect(screen.queryByTestId('cs-order-switch-point')).toBeNull(); + }); + + it('does not survive into the next session - Stop clears the record with the table', async () => { + const { streams } = await renderRunningSession(); + + await act(async () => { + streams[0].emit('data', dataFrame([message(1)])); + }); + await flush(); + await act(async () => { + streams[0].emit('data', caughtUpFrame()); + }); + await switchToLive(); + await act(async () => { + streams[streams.length - 1].emit('data', dataFrame([message(2)])); + }); + await flush(); + expect(screen.getAllByTestId('cs-order-switch-point')).toHaveLength(1); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-stop')); + }); + + expect(screen.queryByTestId('cs-order-switch-point')).toBeNull(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.test.ts b/ui/components/ui/ConsumerSession/ConsumerSession.test.ts new file mode 100644 index 000000000..ebebe61c0 --- /dev/null +++ b/ui/components/ui/ConsumerSession/ConsumerSession.test.ts @@ -0,0 +1,1468 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * BUG-1 regression: a message-less ResumeResponse must surface the server error, not throw. + * BUG-2 regression: a non-OK PauseResponse must be reported, not silently treated as a pause. + * + * Both server paths are real: ConsumerServiceImpl.resume() emits a status-only, message-less + * ResumeResponse when the session is missing or the resume threw, and ConsumerServiceImpl.pause() + * answers FAILED_PRECONDITION (a resolved response, never a transport rejection) in the same + * situations. The responses below are the genuine protobuf messages, not stubs. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +import { Status } from '../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../grpc-web/google/rpc/code_pb'; +import { StringValue } from 'google-protobuf/google/protobuf/wrappers_pb'; +import { + ConsumerStats, + CreateConsumerResponse, + DeleteConsumerResponse, + Message, + MessageDeliveryOrder, + PauseResponse, + ResumeResponse, + SetDeliveryOrderResponse, + StartFromProgress, +} from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; + +// nanoid@4 ships ESM only and jest does not transform node_modules, so importing ConsumerSession +// (which uses it purely to name the consumer/subscription) would fail to parse. Nothing under test +// depends on the generated id. +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); +// Same story for mermaid (ESM-only, pulled in far away through the library item editor's markdown +// preview). Neither library is exercised by these tests. +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); +// The More-tools panel is a sibling feature with its own suite (Console.test.tsx); rendered here it +// would only drag every tab body (producer, topic positions, ...) into each live-session test. +jest.mock('./Console/Console', () => ({ __esModule: true, default: () => null })); +// The live-session tests below need the session to talk to a scriptable consumer service. Fakes +// are installed per-test on globalThis (see installFakeGrpcClients) rather than via a module +// binding, because jest hoists this factory above every declaration in the file; tests that +// install nothing keep the real (inert) default clients. +jest.mock('../../app/contexts/GrpcClient/GrpcClient', () => { + const actual = jest.requireActual('../../app/contexts/GrpcClient/GrpcClient'); + return { + ...actual, + useContext: () => ({ + ...actual.useContext(), + ...((globalThis as Record)['__csTestGrpcClients'] as object | undefined), + }), + }; +}); + +import React from 'react'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import ConsumerSession, { + configValOrRefWithDeliveryOrder, + handleDrainingResumeResponse, + handleResumeResponse, + pauseConsumer, +} from './ConsumerSession'; +import { getDefaultManagedItem } from '../LibraryBrowser/default-library-items'; +import { defaultValue as notifications } from '../../app/contexts/Notifications'; +import * as Notifications from '../../app/contexts/Notifications'; + +// react-virtuoso (the message table) measures itself with ResizeObserver, which jsdom does not +// provide - same stub as TopicPositions.test.tsx. And the session auto-scrolls its table; +// jsdom implements scrolling on window only, not on elements. +class ResizeObserverStub { + observe() { } + unobserve() { } + disconnect() { } +} +(globalThis as { ResizeObserver?: unknown }).ResizeObserver = + (globalThis as { ResizeObserver?: unknown }).ResizeObserver ?? ResizeObserverStub; +if (Element.prototype.scrollTo === undefined) { + Element.prototype.scrollTo = (() => undefined) as never; +} + +const makeStatus = (code: number, message: string) => { + const s = new Status(); + s.setCode(code); + s.setMessage(message); + return s; +}; + +const makeMessage = (opts: { value?: string; processed: number; sent: number }) => { + const m = new Message(); + if (opts.value !== undefined) { + const v = new StringValue(); + v.setValue(opts.value); + m.setValue(v); + } + m.setNumMessageProcessed(opts.processed); + m.setNumMessageSent(opts.sent); + return m; +}; + +const makeSinks = () => ({ + messagesBuffer: { current: [] as ReturnType[] }, + messagesProcessed: { current: 7 }, + messagesLoaded: { current: 9 }, + notifyError: jest.fn(), + setStartFromProgress: jest.fn(), + setStartFromDegradation: jest.fn(), + setOrderingLateDeliveries: jest.fn(), + setOrderingActive: jest.fn(), + setOrderKeyFallbacks: jest.fn(), + setOrderingWaitingStreams: jest.fn(), + setReplayCaughtUp: jest.fn(), + setReplaySeamViolations: jest.fn(), +}); + +/** A ResumeResponse carrying the consumer stats the server emits while a big skip is resolving. */ +const makeResumeWithProgress = (progress?: { skipped: number; toSkip: number; complete?: boolean }) => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + if (progress !== undefined) { + const p = new StartFromProgress(); + p.setMessagesSkipped(progress.skipped); + p.setMessagesToSkip(progress.toSkip); + p.setComplete(progress.complete ?? false); + stats.setStartFromProgress(p); + } + res.setConsumerStats(stats); + return res; +}; + +const lastProgress = (sinks: ReturnType) => { + const calls = sinks.setStartFromProgress.mock.calls; + return calls[calls.length - 1][0]; +}; + +describe('BUG-1: message-less ResumeResponse', () => { + it('surfaces the server error instead of throwing on a status-only response', () => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.FAILED_PRECONDITION, 'No such consumer consumer session: __dekaf_x')); + + const sinks = makeSinks(); + expect(() => handleResumeResponse(res, sinks)).not.toThrow(); + + expect(sinks.notifyError).toHaveBeenCalledTimes(1); + expect(String(sinks.notifyError.mock.calls[0][0])).toContain('No such consumer consumer session'); + // Nothing arrived, so the counters must keep their previous values. + expect(sinks.messagesProcessed.current).toBe(7); + expect(sinks.messagesLoaded.current).toBe(9); + expect(sinks.messagesBuffer.current).toHaveLength(0); + }); + + it('buffers valued messages and advances the counters on an OK response', () => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + // The runner also emits count-only placeholders (no value) - those advance counters but render + // no row, so they must not reach the buffer. + res.setMessagesList([ + makeMessage({ value: '{"a":1}', processed: 41, sent: 40 }), + makeMessage({ processed: 42, sent: 40 }), + ]); + + const sinks = makeSinks(); + handleResumeResponse(res, sinks); + + expect(sinks.notifyError).not.toHaveBeenCalled(); + expect(sinks.messagesBuffer.current).toHaveLength(1); + expect(sinks.messagesProcessed.current).toBe(42); + expect(sinks.messagesLoaded.current).toBe(40); + }); + + it('reports per-message errors while still processing the messages that came with them', () => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.UNKNOWN, 'filter failed: boom')); + res.setMessagesList([makeMessage({ value: '{"a":1}', processed: 5, sent: 5 })]); + + const sinks = makeSinks(); + handleResumeResponse(res, sinks); + + expect(String(sinks.notifyError.mock.calls[0][0])).toContain('filter failed: boom'); + expect(sinks.messagesBuffer.current).toHaveLength(1); + expect(sinks.messagesProcessed.current).toBe(5); + }); +}); + +describe('start-from skip progress carried on a ResumeResponse', () => { + it('surfaces an in-flight skip', () => { + const sinks = makeSinks(); + + handleResumeResponse(makeResumeWithProgress({ skipped: 2_000_000, toSkip: 10_000_000 }), sinks); + + expect(lastProgress(sinks)).toEqual({ messagesSkipped: 2_000_000, messagesToSkip: 10_000_000 }); + }); + + it('surfaces a SMALL unresolved skip too - a frame only arrives for one when it is stalled', () => { + // There used to be a 1,000,000-message display threshold, on the theory that small skips + // resolve too fast to need a UI. They do - which is exactly why a frame for one means the + // opposite: the server pushes it when the positioning is STALLED waiting on a silent stream, + // and suppressing it left "Awaiting for new messages..." over a skip that was stuck. + const sinks = makeSinks(); + + handleResumeResponse(makeResumeWithProgress({ skipped: 10, toSkip: 500 }), sinks); + + expect(lastProgress(sinks)).toEqual({ messagesSkipped: 10, messagesToSkip: 500 }); + }); + + it('clears the indicator once the skip completes', () => { + const sinks = makeSinks(); + + handleResumeResponse(makeResumeWithProgress({ skipped: 2_000_000, toSkip: 10_000_000 }), sinks); + handleResumeResponse(makeResumeWithProgress({ skipped: 10_000_000, toSkip: 10_000_000, complete: true }), sinks); + + expect(lastProgress(sinks)).toBeUndefined(); + }); + + it('clears the indicator when the stats stop carrying progress', () => { + const sinks = makeSinks(); + + handleResumeResponse(makeResumeWithProgress({ skipped: 2_000_000, toSkip: 10_000_000 }), sinks); + // `consumer_stats` is present but `start_from_progress` is not - the documented steady state. + handleResumeResponse(makeResumeWithProgress(), sinks); + + expect(lastProgress(sinks)).toBeUndefined(); + }); + + it('clears the indicator when consumer_stats is absent altogether', () => { + const sinks = makeSinks(); + + handleResumeResponse(makeResumeWithProgress({ skipped: 2_000_000, toSkip: 10_000_000 }), sinks); + + // Every ordinary data response: status + messages, no stats at all. It must not leave the + // "skipping..." panel on screen forever. + const plain = new ResumeResponse(); + plain.setStatus(makeStatus(Code.OK, '')); + plain.setMessagesList([makeMessage({ value: '{"a":1}', processed: 1, sent: 1 })]); + expect(() => handleResumeResponse(plain, sinks)).not.toThrow(); + + expect(lastProgress(sinks)).toBeUndefined(); + }); + + it('handles a stats-only, message-less response without throwing or disturbing the counters', () => { + // While the skip runs the server has nothing to deliver, so these responses carry NO messages - + // the same shape that used to kill the stream by dereferencing a message that was not there. + const sinks = makeSinks(); + + expect(() => + handleResumeResponse(makeResumeWithProgress({ skipped: 3_000_000, toSkip: 9_000_000 }), sinks) + ).not.toThrow(); + + expect(lastProgress(sinks)).toEqual({ messagesSkipped: 3_000_000, messagesToSkip: 9_000_000 }); + expect(sinks.notifyError).not.toHaveBeenCalled(); + expect(sinks.messagesProcessed.current).toBe(7); + expect(sinks.messagesLoaded.current).toBe(9); + expect(sinks.messagesBuffer.current).toHaveLength(0); + }); + + it('still reports progress on a response whose status is an error', () => { + // The status is handled first and then returns nothing; the progress must not be lost with it. + const res = makeResumeWithProgress({ skipped: 4_000_000, toSkip: 8_000_000 }); + res.setStatus(makeStatus(Code.UNKNOWN, 'filter failed: boom')); + + const sinks = makeSinks(); + handleResumeResponse(res, sinks); + + expect(sinks.notifyError).toHaveBeenCalledTimes(1); + expect(lastProgress(sinks)).toEqual({ messagesSkipped: 4_000_000, messagesToSkip: 8_000_000 }); + }); +}); + +describe('BUG-4: a crash inside the session stays local', () => { + const libraryContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, + }; + + const renderSession = (broken: unknown) => + render( + React.createElement( + SWRConfig, + { value: { shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false } }, + React.createElement(ConsumerSession, { + initialConfig: { type: 'value', val: broken } as never, + libraryContext, + }) + ) + ); + + it('a foreign persisted item in a target slot is refused before it can throw', () => { + // This used to reach the per-target editor and throw mid-render, leaving the ErrorBoundary to + // catch it. The recursive decoder now names the offending level instead, which is the same + // outcome for the app (no blank document) and a far better one for the reader. + const config = getDefaultManagedItem('consumer-session-config', libraryContext); + const foreignTarget = getDefaultManagedItem('message-filter', libraryContext); + const broken = { + ...config, + spec: { ...(config as { spec: Record }).spec, targets: [{ type: 'value', val: foreignTarget }] }, + }; + + renderSession(broken); + + expect(screen.getByTestId('cs-invalid-config')).toBeTruthy(); + expect(screen.getByTestId('cs-invalid-config-problem').textContent).toContain('spec.targets[0].val.metadata.type'); + expect(screen.queryByTestId('cs-crashed')).toBeNull(); + }); + + it('renders a visible error instead of unmounting the app when a render still throws', () => { + // The boundary is still the last resort, because the decoder stops where the document does: + // the filter operator tree belongs to the filter editor's own contract, so a filter whose + // operator is missing passes the shape check and throws a few frames deeper. The route must + // not end up as an empty document. + const config = getDefaultManagedItem('consumer-session-config', libraryContext) as any; + const filter = getDefaultManagedItem('message-filter', libraryContext) as any; + delete filter.spec.filter.op; + config.spec.messageFilterChain.val.spec.filters = [{ type: 'value', val: filter }]; + + renderSession(config); + + expect(screen.getByTestId('cs-crashed')).toBeTruthy(); + expect(document.body.textContent).toContain('could not be rendered'); + }); +}); + +describe('BUG-2: pause failures', () => { + // `status: undefined` builds a PauseResponse with no status at all. + const pauseRespondingWith = (status?: { code: number; message: string }) => { + const res = new PauseResponse(); + if (status !== undefined) { + res.setStatus(makeStatus(status.code, status.message)); + } + return { pause: (_request: unknown, _metadata: unknown) => Promise.resolve(res) }; + }; + + it('reports a non-OK PauseResponse', async () => { + const notifyError = jest.fn(); + const outcome = await pauseConsumer({ + client: pauseRespondingWith({ + code: Code.FAILED_PRECONDITION, + message: 'No such consumer consumer session: __dekaf_x', + }), + consumerName: '__dekaf_x', + notifyError, + }); + + expect(notifyError).toHaveBeenCalledTimes(1); + expect(String(notifyError.mock.calls[0][0])).toContain('No such consumer consumer session'); + // Reporting is not enough: the caller decides whether the session may CALL ITSELF paused, and + // it can only do that if the refusal comes back to it. + expect(outcome).toBe('failed'); + }); + + it('reports a PauseResponse that carries no status at all', async () => { + const notifyError = jest.fn(); + const outcome = await pauseConsumer({ client: pauseRespondingWith(), consumerName: '__dekaf_x', notifyError }); + + expect(notifyError).toHaveBeenCalledTimes(1); + expect(outcome).toBe('failed'); + }); + + it('stays silent on an OK response', async () => { + const notifyError = jest.fn(); + const outcome = await pauseConsumer({ + client: pauseRespondingWith({ code: Code.OK, message: '' }), + consumerName: '__dekaf_x', + notifyError, + }); + + expect(notifyError).not.toHaveBeenCalled(); + expect(outcome).toBe('paused'); + }); + + it('still reports a rejected pause call', async () => { + const notifyError = jest.fn(); + const outcome = await pauseConsumer({ + client: { pause: () => Promise.reject(new Error('transport down')) }, + consumerName: '__dekaf_x', + notifyError, + }); + + expect(notifyError).toHaveBeenCalledTimes(1); + expect(String(notifyError.mock.calls[0][0])).toContain('transport down'); + expect(outcome).toBe('failed'); + }); +}); + +describe('the start-from DEGRADATION record', () => { + const progressFrame = (over: { degraded?: boolean; abandoned?: string[] } = {}) => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + const progress = new StartFromProgress(); + progress.setMessagesSkipped(10); + progress.setMessagesToSkip(100); + progress.setDegraded(over.degraded ?? false); + progress.setAbandonedStreamsList(over.abandoned ?? []); + stats.setStartFromProgress(progress); + res.setConsumerStats(stats); + return res; + }; + + it('a degraded frame hands the abandoned streams to the sink - size threshold does NOT apply', () => { + // The skip here (100) is far below the progress panel's display threshold; the degradation + // must surface anyway - a best-effort answer on a small skip is still best-effort. + const sinks = makeSinks(); + handleResumeResponse(progressFrame({ degraded: true, abandoned: ['cs-1@persistent://t/ns/a-partition-1'] }), sinks); + + expect(sinks.setStartFromDegradation).toHaveBeenCalledTimes(1); + expect(sinks.setStartFromDegradation.mock.calls[0][0]).toEqual(['cs-1@persistent://t/ns/a-partition-1']); + }); + + it('an ordinary frame never touches the degradation sink - the record is sticky, not cleared per frame', () => { + const sinks = makeSinks(); + handleResumeResponse(progressFrame({ degraded: false }), sinks); + + expect(sinks.setStartFromDegradation).not.toHaveBeenCalled(); + }); +}); + +describe('the best-effort order late counter carried on a ResumeResponse', () => { + it('reaches the sink when the server confesses late deliveries', () => { + const sinks = makeSinks(); + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + stats.setOrderingLateDeliveries(3); + res.setConsumerStats(stats); + + handleResumeResponse(res, sinks); + + expect(sinks.setOrderingLateDeliveries).toHaveBeenCalledWith(3); + }); + + it('stays quiet at zero, so an ordinary session never touches the state', () => { + const sinks = makeSinks(); + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + + handleResumeResponse(res, sinks); + + expect(sinks.setOrderingLateDeliveries).not.toHaveBeenCalled(); + expect(sinks.setOrderingActive).not.toHaveBeenCalled(); + }); + + it('the ACTIVE flag reaches its sink - the chip may only claim a layer that is really running', () => { + const sinks = makeSinks(); + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + stats.setDeliveryOrderActive(true); + res.setConsumerStats(stats); + + handleResumeResponse(res, sinks); + + expect(sinks.setOrderingActive).toHaveBeenCalledWith(true); + expect(sinks.setOrderingWaitingStreams).toHaveBeenCalledWith(0); + }); + + /** A frame whose stats assert an active layer waiting on `waiting` streams. */ + const waitingFrame = (waiting: number) => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + stats.setDeliveryOrderActive(true); + stats.setDeliveryOrderWaitingStreams(waiting); + res.setConsumerStats(stats); + return res; + }; + + it('surfaces and clears a delivery-order wait (stats keep coming, the field stops)', () => { + const sinks = makeSinks(); + + handleResumeResponse(waitingFrame(2), sinks); + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(2); + + const resumed = new ResumeResponse(); + resumed.setStatus(makeStatus(Code.OK, '')); + const resumedStats = new ConsumerStats(); + resumedStats.setDeliveryOrderActive(true); + resumed.setConsumerStats(resumedStats); + handleResumeResponse(resumed, sinks); + + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(0); + }); + + it('clears the wait on a frame with no consumer_stats at all - the frame that actually arrives', () => { + // The clear used to live inside the delivery_order_active guard, so the most ordinary frame + // there is - status plus messages, no stats - left the last non-zero count on a chip that is + // sticky-visible. "Waiting" is a claim about NOW: present means SET, anything else means + // CLEAR. The server is also growing the habit of carrying this field on stats frames outside + // any skip; present-SET/absent-CLEAR is exactly what composes with that. + const sinks = makeSinks(); + + handleResumeResponse(waitingFrame(3), sinks); + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(3); + + const plain = new ResumeResponse(); + plain.setStatus(makeStatus(Code.OK, '')); + plain.setMessagesList([makeMessage({ value: '{"a":1}', processed: 1, sent: 1 })]); + handleResumeResponse(plain, sinks); + + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(0); + }); + + it('clears the wait on stats that stopped asserting the layer, without un-sticking the active flag', () => { + const sinks = makeSinks(); + + handleResumeResponse(waitingFrame(2), sinks); + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(2); + + // Stats present, delivery_order_active false (the proto3 default), no waiting field. + const inactive = new ResumeResponse(); + inactive.setStatus(makeStatus(Code.OK, '')); + inactive.setConsumerStats(new ConsumerStats()); + handleResumeResponse(inactive, sinks); + + expect(sinks.setOrderingWaitingStreams).toHaveBeenLastCalledWith(0); + // The ACTIVE flag stays sticky: it is only ever raised, never written false by a frame. + expect(sinks.setOrderingActive).not.toHaveBeenCalledWith(false); + }); +}); + +/** + * The guaranteed replay's caught-up state carried on a ResumeResponse. The server auto-pauses the + * runner at the replay boundary and the CLIENT learns only through this stats frame - and while + * the session stays boundary-paused, every later response re-asserts the state, so the record + * mirrors the frame exactly like the waiting status does: present means SET, anything else means + * CLEAR. Leaving a stale record would pin a "caught up" banner over a session that resumed. + */ +describe('the guaranteed replay caught-up state carried on a ResumeResponse', () => { + const caughtUpFrame = (over: { + boundaryAtMs?: number; + newerEntriesApprox?: number; + seamViolations?: number; + excludedTopics?: string[]; + excludedTopicCount?: number; + } = {}) => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + stats.setDeliveryOrderActive(true); + stats.setReplayCaughtUp(true); + stats.setReplayBoundaryAtMs(over.boundaryAtMs ?? 1_754_700_000_000); + stats.setReplayNewerEntriesApprox(over.newerEntriesApprox ?? 0); + stats.setReplaySeamViolations(over.seamViolations ?? 0); + stats.setReplayExcludedTopicsList(over.excludedTopics ?? []); + stats.setReplayExcludedTopicCount(over.excludedTopicCount ?? 0); + res.setConsumerStats(stats); + return res; + }; + + const lastCaughtUp = (sinks: ReturnType) => { + const calls = sinks.setReplayCaughtUp.mock.calls; + return calls[calls.length - 1][0]; + }; + + it('hands the caught-up record to its sink - boundary, indicators and exclusions included', () => { + const sinks = makeSinks(); + + handleResumeResponse(caughtUpFrame({ + boundaryAtMs: 1_754_700_000_000, + newerEntriesApprox: 42, + excludedTopics: ['persistent://t/ns/a', 'persistent://t/ns/b'], + excludedTopicCount: 7, + }), sinks); + + // The frame still CARRIES `replay_newer_entries_approx`; the client stopped reading it on + // 2026-08-11, when the "~N entries" line was removed - it was approximate in a unit (broker + // entries, not messages) that no reader thinks in. + expect(lastCaughtUp(sinks)).toEqual({ + boundaryAtMs: 1_754_700_000_000, + excludedTopics: ['persistent://t/ns/a', 'persistent://t/ns/b'], + excludedTopicCount: 7, + }); + }); + + it('clears the record on stats that stop asserting caught-up - the first frame after a Resume', () => { + const sinks = makeSinks(); + + handleResumeResponse(caughtUpFrame(), sinks); + expect(lastCaughtUp(sinks)).toBeDefined(); + + // Stats present (the ordering layer is running again), replay_caught_up absent - the proto3 + // default the server sends once the boundary was extended. + const resumed = new ResumeResponse(); + resumed.setStatus(makeStatus(Code.OK, '')); + const stats = new ConsumerStats(); + stats.setDeliveryOrderActive(true); + resumed.setConsumerStats(stats); + handleResumeResponse(resumed, sinks); + + expect(lastCaughtUp(sinks)).toBeUndefined(); + }); + + it('clears the record on a frame with no consumer_stats at all - the ordinary data frame', () => { + const sinks = makeSinks(); + + handleResumeResponse(caughtUpFrame(), sinks); + expect(lastCaughtUp(sinks)).toBeDefined(); + + const plain = new ResumeResponse(); + plain.setStatus(makeStatus(Code.OK, '')); + plain.setMessagesList([makeMessage({ value: '{"a":1}', processed: 1, sent: 1 })]); + handleResumeResponse(plain, sinks); + + expect(lastCaughtUp(sinks)).toBeUndefined(); + }); + + it('the seam-violation counter reaches its sink when the server confesses one', () => { + // Monotonic from the server, exactly like the late counter: the count only ever grows, and a + // frame that stops carrying stats leaves the last value standing. + const sinks = makeSinks(); + + handleResumeResponse(caughtUpFrame({ seamViolations: 2 }), sinks); + + expect(sinks.setReplaySeamViolations).toHaveBeenCalledWith(2); + }); + + it('stays quiet at zero seam violations, so an ordinary replay never touches the state', () => { + const sinks = makeSinks(); + + handleResumeResponse(caughtUpFrame(), sinks); + + expect(sinks.setReplaySeamViolations).not.toHaveBeenCalled(); + }); +}); + +describe('a frame from a superseded (draining) stream', () => { + // The policy handleDrainingResumeResponse enforces: a superseded stream's tail frames deliver + // their messages - sent and acknowledged server-side, existing nowhere else - and NOTHING else. + // The counters and stats on such a frame describe an older moment than what the live stream + // already reported; the wiring that swaps the full handler for this one is pinned by the + // live-session test below. + it('appends only the valued messages', () => { + const res = new ResumeResponse(); + res.setStatus(makeStatus(Code.OK, '')); + res.setMessagesList([ + makeMessage({ value: '{"a":1}', processed: 3, sent: 3 }), + // A count-only placeholder: advances counters on the live handler, renders no row - and on + // a draining stream must reach neither the buffer nor the counters. + makeMessage({ processed: 4, sent: 4 }), + ]); + + const messagesBuffer = { current: [] as ReturnType[] }; + handleDrainingResumeResponse(res, messagesBuffer); + + expect(messagesBuffer.current).toHaveLength(1); + expect(messagesBuffer.current[0].getValue()?.getValue()).toBe('{"a":1}'); + }); + + it('tolerates the message-less shapes a closing stream sends', () => { + const statusOnly = new ResumeResponse(); + statusOnly.setStatus(makeStatus(Code.FAILED_PRECONDITION, 'No such consumer')); + + const messagesBuffer = { current: [] as ReturnType[] }; + expect(() => handleDrainingResumeResponse(statusOnly, messagesBuffer)).not.toThrow(); + expect(messagesBuffer.current).toHaveLength(0); + }); +}); + +/** + * Live-session tests: the full component against a scriptable consumer service and hand-driven + * resume streams. Everything the session believes - counters, panels, chips - is read back + * through the DOM, exactly as a user would see it. + */ +describe('a live consumer session', () => { + /** A minimal grpc-web ClientReadableStream the session can subscribe to and the test can drive. */ + const makeFakeStream = () => { + const listeners = new Map void>>(); + const stream = { + on: (ev: string, cb: (arg?: unknown) => void) => { + if (!listeners.has(ev)) { + listeners.set(ev, new Set()); + } + listeners.get(ev)!.add(cb); + return stream; + }, + removeListener: (ev: string, cb: (arg?: unknown) => void) => { + listeners.get(ev)?.delete(cb); + }, + cancel: jest.fn(), + emit: (ev: string, arg?: unknown) => { + Array.from(listeners.get(ev) ?? []).forEach((cb) => cb(arg)); + }, + listenerCount: (ev: string) => listeners.get(ev)?.size ?? 0, + }; + return stream; + }; + + const okStatus = () => makeStatus(Code.OK, ''); + + const installFakeGrpcClients = (opts: { setDeliveryOrderWith?: { code: number; message: string } } = {}) => { + const streams: Array> = []; + const consumerServiceClient = { + // The live delivery-order switch. A refusal (a direction the server does not perform on a + // running session, an unknown session) is a RESOLVED response carrying a reason, exactly as + // ConsumerServiceImpl answers - never a rejected call. + setDeliveryOrder: jest.fn(() => { + const res = new SetDeliveryOrderResponse(); + const s = opts.setDeliveryOrderWith ?? { code: Code.OK, message: '' }; + res.setStatus(makeStatus(s.code, s.message)); + return Promise.resolve(res); + }), + createConsumer: jest.fn(() => { + const res = new CreateConsumerResponse(); + res.setStatus(okStatus()); + return Promise.resolve(res); + }), + resume: jest.fn(() => { + const stream = makeFakeStream(); + streams.push(stream); + return stream; + }), + pause: jest.fn(() => { + const res = new PauseResponse(); + res.setStatus(okStatus()); + return Promise.resolve(res); + }), + deleteConsumer: jest.fn(() => { + const res = new DeleteConsumerResponse(); + res.setStatus(okStatus()); + return Promise.resolve(res); + }), + // The configuration view's topic-selector info resolves the selection through the same + // client. An empty OK answer renders "No topics found", which is all these tests need. + resolveTopicSelector: jest.fn(() => Promise.resolve({ + getStatus: () => okStatus(), + getTopicFqnsList: () => [] as string[], + })), + }; + (globalThis as Record)['__csTestGrpcClients'] = { consumerServiceClient }; + return { consumerServiceClient, streams }; + }; + + /** A ResumeResponse assembled from plain parts: valued/placeholder messages, progress, ordering. */ + const frame = (opts: { + messages?: Array<{ value?: string; processed: number; sent: number }>; + progress?: { skipped: number; toSkip: number; complete?: boolean }; + waiting?: number; + active?: boolean; + /** The guaranteed replay reached its boundary: the server auto-paused and says so. */ + caughtUp?: { + boundaryAtMs?: number; + newerEntriesApprox?: number; + excludedTopics?: string[]; + excludedTopicCount?: number; + }; + seamViolations?: number; + }) => { + const res = new ResumeResponse(); + res.setStatus(okStatus()); + res.setMessagesList((opts.messages ?? []).map(makeMessage)); + if (opts.progress !== undefined || opts.waiting !== undefined || opts.active !== undefined + || opts.caughtUp !== undefined || opts.seamViolations !== undefined) { + const stats = new ConsumerStats(); + if (opts.progress !== undefined) { + const p = new StartFromProgress(); + p.setMessagesSkipped(opts.progress.skipped); + p.setMessagesToSkip(opts.progress.toSkip); + p.setComplete(opts.progress.complete ?? false); + stats.setStartFromProgress(p); + } + if (opts.active !== undefined) { + stats.setDeliveryOrderActive(opts.active); + } + if (opts.waiting !== undefined) { + stats.setDeliveryOrderWaitingStreams(opts.waiting); + } + if (opts.caughtUp !== undefined) { + stats.setReplayCaughtUp(true); + stats.setReplayBoundaryAtMs(opts.caughtUp.boundaryAtMs ?? 1_754_700_000_000); + stats.setReplayNewerEntriesApprox(opts.caughtUp.newerEntriesApprox ?? 0); + stats.setReplayExcludedTopicsList(opts.caughtUp.excludedTopics ?? []); + stats.setReplayExcludedTopicCount(opts.caughtUp.excludedTopicCount ?? 0); + } + if (opts.seamViolations !== undefined) { + stats.setReplaySeamViolations(opts.seamViolations); + } + res.setConsumerStats(stats); + } + return res; + }; + + const topicLibraryContext = (topicPersistency: 'persistent' | 'non-persistent') => ({ + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency, + topic: 'a-topic', + }, + }); + + const renderSession = (opts: { + topicPersistency?: 'persistent' | 'non-persistent'; + startFrom?: unknown; + /** + * Pin the session's delivery order EXPLICITLY, like the e2e matrix does. Tests whose scenario + * needs a particular mode (a Guaranteed stall, say) must not inherit it from the default-item + * template - the default is a product decision that can move (and did: Best effort since + * 2026-08-09), and a scenario riding on it silently stops testing what it says. + */ + deliveryOrder?: 'guaranteed' | 'best-effort' | 'as-received'; + /** + * Render the session against a REFERENCE to a library item rather than an own value. The + * resolved item still rides along in `val` (the browser-local draft the config editor caches), + * which is what lets the session run at all - see consumerSessionConfigFromValOrRef. + */ + asReferenceTo?: string; + } = {}) => { + const libraryContext = topicLibraryContext(opts.topicPersistency ?? 'persistent'); + const config = getDefaultManagedItem('consumer-session-config', libraryContext) as { + spec: { startFrom: { val: { spec: { startFrom: unknown } } }; messageDeliveryOrder?: string }; + }; + if (opts.startFrom !== undefined) { + config.spec.startFrom.val.spec.startFrom = opts.startFrom; + } + if (opts.deliveryOrder !== undefined) { + config.spec.messageDeliveryOrder = opts.deliveryOrder; + } + + const initialConfig = opts.asReferenceTo === undefined + ? { type: 'value', val: config } + : { type: 'reference', ref: opts.asReferenceTo, val: config }; + + render( + React.createElement( + SWRConfig, + { value: { shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false } }, + // The caught-up panel is rendered by the toast container (2026-08-11: it moved out of the + // inline strip into a persistent notification), so the provider has to be mounted for the + // panel to exist at all - without it these tests would assert against a DOM that never + // gets it, and pass or fail for the wrong reason. + React.createElement( + Notifications.DefaultProvider, + null, + React.createElement(ConsumerSession, { + initialConfig: initialConfig as never, + libraryContext, + }) + ) + ) + ); + }; + + const session = () => screen.getByTestId('cs-session') as HTMLElement; + const play = () => fireEvent.click(screen.getByTestId('cs-play')); + const stop = () => fireEvent.click(screen.getByTestId('cs-stop')); + const awaitState = (state: string) => waitFor(() => expect(session().dataset.csState).toBe(state)); + + beforeEach(() => { + window.localStorage.clear(); + }); + + afterEach(() => { + delete (globalThis as Record)['__csTestGrpcClients']; + window.localStorage.clear(); + }); + + it('does not resurface a previous run\'s skip progress after pause and resume on an idle topic', async () => { + const { streams } = installFakeGrpcClients(); + renderSession(); + + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + + // A big skip is resolving: the server pushes progress and nothing else. + act(() => { + streams[0].emit('data', frame({ progress: { skipped: 10, toSkip: 2_000_000 } })); + }); + const panel = await screen.findByTestId('cs-start-from-progress'); + expect(panel.dataset.csToSkip).toBe('2000000'); + + // Pause mid-skip: the panel hides (it is gated on `running`), the record used to stay behind. + play(); + await awaitState('paused'); + expect(screen.queryByTestId('cs-start-from-progress')).toBeNull(); + + // Play again. The NEW stream never sends a frame - an idle topic whose skip already resolved - + // so nothing would ever clear a stale record: the panel must not re-appear on its own. + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(2)); + await waitFor(() => expect(screen.queryByTestId('cs-start-from-progress')).toBeNull()); + expect(screen.getByText('Awaiting for new messages...')).toBeTruthy(); + }); + + it('a late frame from the superseded stream delivers its messages and moves nothing else', async () => { + const { streams } = installFakeGrpcClients(); + renderSession(); + + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + + act(() => { + streams[0].emit('data', frame({ messages: [{ value: '{"n":1}', processed: 5, sent: 5 }] })); + }); + await waitFor(() => expect(session().dataset.csRetained).toBe('1')); + await waitFor(() => expect(screen.getByTestId('cs-loaded').textContent).toBe('5')); + + play(); + await awaitState('paused'); + play(); + await awaitState('running'); + // The new stream's listener being attached proves the old one was superseded (the swap happens + // in the same effect turnover). + await waitFor(() => expect(streams).toHaveLength(2)); + await waitFor(() => expect(streams[1].listenerCount('data')).toBe(1)); + + // The predecessor's last in-flight frame arrives late: smaller (older) counters, a stale skip + // progress, a stale ordering wait - and one real message that exists nowhere else. + act(() => { + streams[0].emit('data', frame({ + messages: [{ value: '{"n":2}', processed: 3, sent: 3 }], + progress: { skipped: 1, toSkip: 100 }, + active: true, + waiting: 4, + })); + }); + + // The message is kept - cancelling the stream instead would have dropped it silently. + await waitFor(() => expect(session().dataset.csRetained).toBe('2')); + // The counters are NOT regressed to the stale frame's 3 (that is what rendered a negative + // per-second gauge), and the stale stats re-open nothing: no ordering chip, no skip panel. + expect(screen.getByTestId('cs-loaded').textContent).toBe('5'); + expect(screen.queryByTestId('cs-order-chip')).toBeNull(); + expect(screen.queryByTestId('cs-start-from-progress')).toBeNull(); + }); + + it('a history start-from on a non-persistent topic: Play sends the live tail, the stored config keeps the user\'s mode', async () => { + const { consumerServiceClient } = installFakeGrpcClients(); + renderSession({ topicPersistency: 'non-persistent', startFrom: { type: 'nthMessageAfterEarliest', n: 5 } }); + + // The configuration view says the substitution out loud, and the stored mode is NOT rewritten + // from under the user - the selector still shows it. + expect(await screen.findByTestId('cs-start-from-inapplicable-note')).toBeTruthy(); + expect((screen.getByTestId('cs-start-from') as HTMLSelectElement).value).toBe('nthMessageAfterEarliest'); + + play(); + await waitFor(() => expect(consumerServiceClient.createConsumer).toHaveBeenCalledTimes(1)); + + // What actually went to the server is the one start position a non-persistent topic has. + const req = (consumerServiceClient.createConsumer as jest.Mock).mock.calls[0][0] as { + getConsumerSessionConfig: () => { getStartFrom: () => { hasStartFromLatestMessage: () => boolean; hasStartFromNthMessageAfterEarliest: () => boolean } }; + }; + const startFromPb = req.getConsumerSessionConfig().getStartFrom(); + expect(startFromPb.hasStartFromLatestMessage()).toBe(true); + expect(startFromPb.hasStartFromNthMessageAfterEarliest()).toBe(false); + }); + + /** + * P2.16: a Guaranteed stall must be ESCAPABLE, not merely disclosed. + * + * Guaranteed - the product default since 2026-08-11 - waits, by design, + * indefinitely for every topic or partition - on a silent one that is forever, and it looks + * exactly like an empty topic. The chip discloses the wait and carries the escape; what is + * asserted here is everything the escape has to do to be real: + * + * - it reaches the LIVE session (a config-only edit would need a Play, which re-reads from the + * start position and throws away everything already held - the opposite of the point); + * - it is PERSISTED, so the next Play asks for what the session is now actually doing; + * - a reference-typed configuration becomes a VALUE first, because `val` on a reference is a + * browser-local draft that the wire conversion drops - writing the order there would leave + * the session disagreeing with the library item it names, unpersisted; + * - the disclosed wait is retired, because "waiting" only ever arrives on a response frame and + * a switch that releases nothing owes no frame; + * - and a refusal is SHOWN, changing nothing. + */ + describe('ending a Guaranteed stall from the chip', () => { + /** Play, then let the server disclose a stall: the ordering layer is running and held. */ + const runIntoAStall = async (streams: Array>) => { + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + + act(() => { + streams[0].emit('data', frame({ active: true, waiting: 3 })); + }); + await screen.findByTestId('cs-order-waiting'); + }; + + const switchToBestEffort = async () => { + await act(async () => { + fireEvent.click(screen.getByTestId('cs-order-switch-best-effort')); + }); + }; + + const deliveryOrderSentByPlay = (client: { createConsumer: unknown }, call: number) => { + const req = (client.createConsumer as jest.Mock).mock.calls[call][0] as { + getConsumerSessionConfig: () => { getMessageDeliveryOrder: () => number }; + }; + return req.getConsumerSessionConfig().getMessageDeliveryOrder(); + }; + + let reportedErrors: string[] = []; + const deliveryOrderErrors = () => reportedErrors.filter((it) => it.includes('delivery order')); + + beforeEach(() => { + reportedErrors = []; + // The session reports through the notification context, whose default value is this object; + // nothing renders a toast container in jsdom, so the report is captured at the source. + jest.spyOn(notifications, 'notifyError').mockImplementation((content) => { + reportedErrors.push(String(content)); + }); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('switches the LIVE session, naming the consumer the session actually built', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runIntoAStall(streams); + + await switchToBestEffort(); + + // Not a config edit that waits for the next Play: the running consumer is switched in place, + // which is what releases the held set without re-reading or losing it. + expect(consumerServiceClient.setDeliveryOrder).toHaveBeenCalledTimes(1); + const req = (consumerServiceClient.setDeliveryOrder as jest.Mock).mock.calls[0][0] as { + getConsumerName: () => string; + getMessageDeliveryOrder: () => number; + }; + const built = (consumerServiceClient.createConsumer as jest.Mock).mock.calls[0][0] as { + getConsumerName: () => string; + }; + expect(req.getConsumerName()).toBe(built.getConsumerName()); + expect(req.getMessageDeliveryOrder()).toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + // The session was NOT recreated to apply it - that would have re-read from the start position. + expect(consumerServiceClient.createConsumer).toHaveBeenCalledTimes(1); + // ...and it went through silently. Filtered rather than compared whole: the library panel + // reports its own unrelated failures against the inert default client in this harness. + expect(deliveryOrderErrors()).toEqual([]); + }); + + it('persists the new order, so the next Play asks for what the session is already doing', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runIntoAStall(streams); + expect(deliveryOrderSentByPlay(consumerServiceClient, 0)).toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + + await switchToBestEffort(); + + // The proof the configuration moved is the WIRE: the next session built from it asks for + // Best effort. (The chip label that used to read it back was removed 2026-08-11.) + // Without the write, Play would silently put + // the user back into the stall they just escaped. + stop(); + await awaitState('new'); + play(); + await waitFor(() => expect(consumerServiceClient.createConsumer).toHaveBeenCalledTimes(2)); + expect(deliveryOrderSentByPlay(consumerServiceClient, 1)) + .toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + }); + + it('converts a referenced configuration to a value instead of writing into the reference', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ asReferenceTo: 'lib-cfg-1', deliveryOrder: 'guaranteed' }); + // The session starts out pointed at a library item - the configuration view says so. + expect(await screen.findByTestId('lib-reference-icon')).toBeTruthy(); + + await runIntoAStall(streams); + await switchToBestEffort(); + + expect(consumerServiceClient.setDeliveryOrder).toHaveBeenCalledTimes(1); + + // The entry is a VALUE now, not a reference still carrying a rewritten draft: the session + // owns its copy, and the library item it used to name is unchanged. + stop(); + await awaitState('new'); + await waitFor(() => expect(screen.queryByTestId('lib-reference-icon')).toBeNull()); + }); + + it('retires the disclosed wait the server just ended, without waiting for a frame', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runIntoAStall(streams); + + await switchToBestEffort(); + + // A switch that released nothing owes no response frame, and `waiting` only ever rides one - + // so nothing else would ever retire this status. + await waitFor(() => expect(screen.queryByTestId('cs-order-waiting')).toBeNull()); + expect(screen.queryByTestId('cs-order-switch-best-effort')).toBeNull(); + // With the disclosure retired the whole status area goes - since 2026-08-11 the bar has no + // always-on mode label, so a healthy merge shows nothing at all. + expect(screen.queryByTestId('cs-order-chip')).toBeNull(); + }); + + it('shows a refused switch and leaves the session, its configuration and the wait alone', async () => { + const refusal = 'Cannot promote a running session to Guaranteed'; + const { consumerServiceClient, streams } = installFakeGrpcClients({ + setDeliveryOrderWith: { code: Code.FAILED_PRECONDITION, message: refusal }, + }); + renderSession({ deliveryOrder: 'guaranteed' }); + await runIntoAStall(streams); + + await switchToBestEffort(); + + // The reason is the whole value of a refusal - swallowing it leaves a button that looks like + // it worked. + await waitFor(() => expect(deliveryOrderErrors().join('\n')).toContain(refusal)); + // Nothing moved: the session is still held, so the disclosure and its escape both stay + // exactly where they were. (The mode label that used to say "Replaying history" was removed + // from the bar 2026-08-11 - the stall disclosure itself is the observable now.) + expect(screen.getByTestId('cs-order-waiting').textContent).toContain('waiting for 3 topics/partitions'); + expect(screen.getByTestId('cs-order-switch-best-effort')).toBeTruthy(); + expect(deliveryOrderSentByPlay(consumerServiceClient, 0)).toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + }); + + it('offers no escape while Guaranteed is merging normally', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + act(() => { + streams[0].emit('data', frame({ active: true, waiting: 0 })); + }); + + // No stall means NOTHING renders - since 2026-08-11 the bar carries no always-on mode + // label, so a healthy merge shows no status area and offers no escape. + await act(async () => {}); + expect(screen.queryByTestId('cs-order-chip')).toBeNull(); + expect(screen.queryByTestId('cs-order-switch-best-effort')).toBeNull(); + expect(consumerServiceClient.setDeliveryOrder).not.toHaveBeenCalled(); + }); + }); + + /** + * The guaranteed replay reaching its boundary. The server AUTO-PAUSES the runner and announces + * it on a stats frame - the server cannot flip browser state, so this frame is the only way the + * session learns it is paused. The UI lands in the ordinary paused state (no parallel state) and + * says WHY with a reason-differentiated banner; Resume extends the boundary through the ordinary + * resume flow, and the banner's switch reuses the same applyDeliveryOrder the stall chip uses. + */ + describe('the guaranteed replay catching up', () => { + const banner = () => screen.queryByTestId('cs-replay-caught-up'); + + /** Play, then let the server announce the boundary auto-pause. */ + const runUntilCaughtUp = async ( + streams: Array>, + caughtUp: { boundaryAtMs?: number; newerEntriesApprox?: number; excludedTopics?: string[]; excludedTopicCount?: number } = {} + ) => { + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + + act(() => { + streams[0].emit('data', frame({ active: true, caughtUp })); + }); + await awaitState('paused'); + await screen.findByTestId('cs-replay-caught-up'); + }; + + let reportedErrors: string[] = []; + + beforeEach(() => { + reportedErrors = []; + jest.spyOn(notifications, 'notifyError').mockImplementation((content) => { + reportedErrors.push(String(content)); + }); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('auto-pauses the session UI on the caught-up frame, without sending a redundant Pause', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + await runUntilCaughtUp(streams); + + // The frame IS the server's confirmation - the runner is already holding everything - so + // the session lands on `paused` directly instead of asking the server to pause again. + expect(session().dataset.csState).toBe('paused'); + expect(consumerServiceClient.pause).not.toHaveBeenCalled(); + }); + + it('leads with WHAT happened, then says what it caught up TO in the reader\'s clock', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + // 2026-08-09 00:00:00 UTC; the banner renders it through the app's date formatting. + await runUntilCaughtUp(streams, { boundaryAtMs: Date.UTC(2026, 7, 9, 0, 0, 0) }); + + expect(banner()).toBeTruthy(); + // The heading is what the panel is read for at a distance; the boundary is the detail. + expect(banner()!.textContent).toContain('Guaranteed order consumer session finished'); + expect(banner()!.textContent).toContain('Caught up to'); + expect(banner()!.textContent).toContain('2026'); + }); + + it('names the topics excluded from this replay, with the TRUE count past the listed five', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + // The server caps the listed names at 5; the count field carries the truth. + await runUntilCaughtUp(streams, { + excludedTopics: ['persistent://t/ns/a', 'persistent://t/ns/b', 'persistent://t/ns/c', 'persistent://t/ns/d', 'persistent://t/ns/e'], + excludedTopicCount: 7, + }); + + const excluded = screen.getByTestId('cs-replay-caught-up-excluded'); + expect(excluded.textContent).toContain('7'); + expect(excluded.textContent).toMatch(/restart/i); + expect(excluded.textContent).toContain('persistent://t/ns/a'); + expect(excluded.textContent).toContain('+2 more'); + }); + + it('claims no exclusions when the server reported none', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + await runUntilCaughtUp(streams); + + expect(screen.queryByTestId('cs-replay-caught-up-excluded')).toBeNull(); + }); + + it('a closed panel must not spring back on the next frame - the close survives the render storm', async () => { + // The bug this pins (in its original, toast-era form): the announcing effect ran on EVERY + // render, and a session parked at the boundary re-renders constantly (the server re-asserts + // the state on every response, and the counters tick) - so closing the panel lasted about a + // second. The panel is component state now, but the invariant is the same: a close holds + // for as long as the session sits at ONE boundary, whatever re-renders around it. + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runUntilCaughtUp(streams); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-replay-caught-up-close')); + }); + expect(banner()).toBeNull(); + + // Re-render the session for reasons that have nothing to do with the boundary - which is + // what actually happens while it sits there. Identical caught-up frames alone would not do + // it: the sink collapses equal records, so they change no state and re-render nothing. + act(() => { + streams[0].emit('data', frame({ active: true, caughtUp: {}, seamViolations: 1 })); + }); + act(() => { + streams[0].emit('data', frame({ active: true, caughtUp: {}, seamViolations: 2 })); + }); + act(() => { + streams[0].emit('data', frame({ active: true, caughtUp: {}, seamViolations: 3 })); + }); + + // Still closed: the boundary did not change, so nothing re-opened what the reader closed. + expect(banner()).toBeNull(); + }); + + it('the panel leaves with the boundary and returns for the next episode - exactly one, always', async () => { + // History, because this sequence broke twice: the panel lived in a persistent toast, and + // resume-then-recatch-up (dismiss -> re-announce, milliseconds apart on an instant catch-up) + // lost both ways - react-toastify silently drops a toast created under a still-exiting id + // (one fixed id: the second boundary's panel never appeared, e2e CS-DM-R2), and fresh ids + // per episode let the retiring panel and its successor COEXIST while the exit animation + // played (two panels, a strict-mode ambiguity - e2e CS-DM-R3B). The panel is a conditional + // render of the session component now: at most one element can exist, in every timing. + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runUntilCaughtUp(streams); + expect(screen.getAllByTestId('cs-replay-caught-up')).toHaveLength(1); + + // Resume: the frames stop asserting the boundary, and the panel goes WITH the record - + // directly assertable here, with no toast layer's animationend between the state and the + // DOM. + await act(async () => { play(); }); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(2)); + act(() => { + streams[1].emit('data', frame({ active: true })); + }); + await waitFor(() => expect(banner()).toBeNull()); + + // The next boundary: the panel returns, and there is still exactly one of it. + act(() => { + streams[1].emit('data', frame({ active: true, caughtUp: {} })); + }); + await awaitState('paused'); + await screen.findByTestId('cs-replay-caught-up'); + expect(screen.getAllByTestId('cs-replay-caught-up')).toHaveLength(1); + }); + + it('a manual pause shows no caught-up banner - the reason differentiates the paused state', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + act(() => { + streams[0].emit('data', frame({ active: true, messages: [{ value: '{"n":1}', processed: 1, sent: 1 }] })); + }); + + play(); + await awaitState('paused'); + + await waitFor(() => expect(banner()).toBeNull()); + }); + + it('"Load new messages up to now" goes back to running through the ordinary resume flow and retires the banner', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runUntilCaughtUp(streams); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-replay-resume')); + }); + + await awaitState('running'); + // The ordinary resume flow: a new resume stream against the same consumer - the server + // extends the boundary on it. + await waitFor(() => expect(streams).toHaveLength(2)); + // And the panel goes with the boundary: it is a conditional render of the session now, so + // its retirement is plain DOM here (no toast layer, no animationend jsdom never fires). + await waitFor(() => expect(banner()).toBeNull()); + }); + + it('Switch to Best effort and follow live switches the LIVE session and resumes into live delivery', async () => { + const { consumerServiceClient, streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + await runUntilCaughtUp(streams); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-replay-continue-best-effort')); + }); + + // The EXISTING applyDeliveryOrder wiring: the live switch RPC, once. + expect(consumerServiceClient.setDeliveryOrder).toHaveBeenCalledTimes(1); + const req = (consumerServiceClient.setDeliveryOrder as jest.Mock).mock.calls[0][0] as { + getMessageDeliveryOrder: () => number; + }; + expect(req.getMessageDeliveryOrder()).toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + + // The server resumed live delivery on the switch; the session follows it back to running. + // (Panel retirement is e2e's to assert - see the note above on jsdom and `animationend`.) + await awaitState('running'); + + // ...and the configuration was persisted through the same wiring, so the next Play asks for + // what the session is now doing. + stop(); + await awaitState('new'); + play(); + await waitFor(() => expect(consumerServiceClient.createConsumer).toHaveBeenCalledTimes(2)); + const nextPlay = (consumerServiceClient.createConsumer as jest.Mock).mock.calls[1][0] as { + getConsumerSessionConfig: () => { getMessageDeliveryOrder: () => number }; + }; + expect(nextPlay.getConsumerSessionConfig().getMessageDeliveryOrder()) + .toBe(MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + }); + + it('a refused switch is shown and leaves the session paused at the boundary, banner intact', async () => { + const refusal = 'Cannot switch this session'; + const { consumerServiceClient, streams } = installFakeGrpcClients({ + setDeliveryOrderWith: { code: Code.FAILED_PRECONDITION, message: refusal }, + }); + renderSession({ deliveryOrder: 'guaranteed' }); + await runUntilCaughtUp(streams); + + await act(async () => { + fireEvent.click(screen.getByTestId('cs-replay-continue-best-effort')); + }); + + expect(reportedErrors.join('\n')).toContain(refusal); + // Nothing moved: the server did not switch, so the session must not claim to be running. + expect(session().dataset.csState).toBe('paused'); + expect(banner()).toBeTruthy(); + expect(consumerServiceClient.createConsumer).toHaveBeenCalledTimes(1); + }); + + it('the session-level seam count rides the warning mark whenever the server confesses one', async () => { + const { streams } = installFakeGrpcClients(); + renderSession({ deliveryOrder: 'guaranteed' }); + + play(); + await awaitState('running'); + await waitFor(() => expect(streams).toHaveLength(1)); + await waitFor(() => expect(streams[0].listenerCount('data')).toBe(1)); + act(() => { + streams[0].emit('data', frame({ active: true, seamViolations: 2 })); + }); + + const warning = await screen.findByTestId('cs-order-warning'); + expect(warning.getAttribute('data-tooltip-html')).toContain('2'); + }); + }); +}); + +/** + * The configuration half of the switch, on its own: what gets WRITTEN when the order changes. + * + * A reference names a library item and its `val` is only a browser-local draft that + * `managed...ValOrRefToPb` drops on the way to the server. Writing the new order into that draft + * while the entry still says `reference` produces a session disagreeing with the item it names, and + * does not even persist - which is why the entry is converted to a value first. + */ +describe('the delivery order written into a session configuration', () => { + const anItem = () => getDefaultManagedItem('consumer-session-config', { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, + }) as { spec: { messageDeliveryOrder?: string } }; + + const orderOf = (v: unknown) => (v as { val: { spec: { messageDeliveryOrder?: string } } }).val.spec.messageDeliveryOrder; + + it('converts a reference to a value rather than leaving a rewritten draft under it', () => { + const item = anItem(); + // Pin the source item to explicit Guaranteed - distinct from both the written order and the + // template default (Best effort since the 2026-08-09 owner decision), so an in-place edit of + // the library item cannot hide behind either. + item.spec.messageDeliveryOrder = 'guaranteed'; + + const written = configValOrRefWithDeliveryOrder( + { type: 'reference', ref: 'lib-cfg-1', val: item } as never, + 'best-effort' + ); + + expect(written).toBeDefined(); + expect(written!.type).toBe('value'); + // Not `val`-plus-`reference`: that shape claims the library item's order and carries a + // different one, and only the `ref` would ever reach the server. + expect(Object.prototype.hasOwnProperty.call(written, 'ref')).toBe(false); + expect(orderOf(written)).toBe('best-effort'); + // The item the reference named is not edited in place - other users of it are untouched. + expect(item.spec.messageDeliveryOrder).toBe('guaranteed'); + }); + + it('rewrites an own value in place, staying a value', () => { + // Start from explicit Guaranteed so the written Best effort is provably the write's doing, + // not the template default it now coincides with. + const item = anItem(); + item.spec.messageDeliveryOrder = 'guaranteed'; + const written = configValOrRefWithDeliveryOrder( + { type: 'value', val: item } as never, + 'best-effort' + ); + + expect(written!.type).toBe('value'); + expect(orderOf(written)).toBe('best-effort'); + }); + + it('writes nothing when there is no resolved configuration to carry the change', () => { + // An unresolved reference: the library item has not arrived, so there is no spec to rewrite and + // a half-written one would be worse than none. + expect(configValOrRefWithDeliveryOrder({ type: 'reference', ref: 'lib-cfg-1' } as never, 'best-effort')) + .toBeUndefined(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/ConsumerSession.tsx b/ui/components/ui/ConsumerSession/ConsumerSession.tsx index 747063b96..63835e537 100644 --- a/ui/components/ui/ConsumerSession/ConsumerSession.tsx +++ b/ui/components/ui/ConsumerSession/ConsumerSession.tsx @@ -9,22 +9,24 @@ import { ResumeResponse, DeleteConsumerRequest, PauseRequest, + PauseResponse, + SetDeliveryOrderRequest, } from '../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; import cts from "../../ui/ChildrenTable/ChildrenTable.module.css"; import MessageComponent from './Message/Message'; import { nanoid } from 'nanoid'; import * as Notifications from '../../app/contexts/Notifications'; import { ItemContent, TableVirtuoso, VirtuosoHandle } from 'react-virtuoso'; -import { ClientReadableStream } from 'grpc-web'; +import { ClientReadableStream, Metadata } from 'grpc-web'; import { createDeadline } from '../../../proto-utils/proto-utils'; import { Code } from '../../../grpc-web/google/rpc/code_pb'; import { useInterval } from '../../app/hooks/use-interval'; import { usePrevious } from '../../app/hooks/use-previous'; import Toolbar from './Toolbar/Toolbar'; -import { SessionState, MessageDescriptor, ConsumerSessionConfig } from './types'; +import { SessionState, MessageDescriptor, ConsumerSessionConfig, MessageDeliveryOrder, ReplayCaughtUpStats } from './types'; import SessionConfiguration from './SessionConfiguration/SessionConfiguration'; import Console from './Console/Console'; -import { consumerSessionConfigToPb, messageDescriptorFromPb } from './conversions/conversions'; +import { consumerSessionConfigToPb, messageDeliveryOrderToPb, messageDescriptorFromPb } from './conversions/conversions'; import { Sort, sortMessages } from './sort'; import { remToPx } from '../rem-to-px'; import { help } from './Message/fields'; @@ -35,14 +37,237 @@ import { getColoring } from './coloring'; import { getValueProjectionThs } from './value-projections/value-projections-utils'; import { Th } from './Th'; import { useColumnWidths } from '../resizable/useColumnWidths'; -import { MessageColumnKey, messageColumnDefaultWidths } from './message-columns'; +import useLocalStorage from 'use-local-storage-state'; +import { localStorageKeys } from '../../local-storage-keys'; +import { MessageColumnKey, ReorderableMessageColumnKey, messageColumnDefaultWidths, messageThMeta, reorderableMessageColumns } from './message-columns'; +import { useColumnOrder } from '../resizable/useColumnOrder'; import MessageDetails from './Message/MessageDetails/MessageDetails'; import ActionButton from '../ActionButton/ActionButton'; import { handleKeyDown } from './keyboard'; import { useDebounce } from 'use-debounce'; +import { ErrorBoundary } from 'react-error-boundary'; +import NothingToShow from '../NothingToShow/NothingToShow'; +import StartFromProgress, { StartFromSkipProgress } from './StartFromProgress/StartFromProgress'; +import StartFromDegradedBanner from './StartFromDegradedBanner'; +import ReplayCaughtUpBanner from './ReplayCaughtUpBanner'; +import { displayItemLimit } from './SessionConfiguration/display-items'; +import { effectiveStartFrom, targetTopicsPersistency } from './SessionConfiguration/StartFromInput/target-topics-persistency'; +import { useResizablePane } from '../resizable/useResizablePane'; +import PaneResizeHandle from '../resizable/PaneResizeHandle'; const consoleCss = "color: #276ff4; font-weight: var(--font-weight-bold);" as const; +export type ResumeResponseSinks = { + messagesBuffer: { current: Message[] }; + messagesProcessed: { current: number }; + messagesLoaded: { current: number }; + notifyError: (message: string) => void; + setStartFromProgress: (progress: StartFromSkipProgress | undefined) => void; + setStartFromDegradation: (abandonedStreams: string[]) => void; + setOrderingLateDeliveries: (count: number) => void; + setOrderingActive: (active: boolean) => void; + setOrderKeyFallbacks: (count: number) => void; + setOrderingWaitingStreams: (count: number) => void; + setReplayCaughtUp: (state: ReplayCaughtUpStats | undefined) => void; + setReplaySeamViolations: (count: number) => void; +}; + +// `consumer_stats` is absent on an ordinary data response, and `start_from_progress` inside it is +// absent unless the start-from is still being resolved - so EVERY field here is optional and the +// answer for "nothing to report" is `undefined`, which clears whatever is on screen. Returning a +// stale value on an absent field would pin a "skipping..." panel up forever. +function readStartFromProgress(res: ResumeResponse): StartFromSkipProgress | undefined { + const progress = res.getConsumerStats()?.getStartFromProgress(); + + if (progress === undefined || progress.getComplete()) { + return undefined; + } + + // No size threshold, deliberately. A small skip normally resolves before a frame is even sent - + // but a STALLED one does not, and the server pushes a frame the moment it starts waiting on a + // silent stream. Suppressing "small" skips here turned exactly those frames into "Awaiting for + // new messages..." over a positioning that was stuck. + return { messagesSkipped: progress.getMessagesSkipped(), messagesToSkip: progress.getMessagesToSkip() }; +} + +// The server reports a missing session or a failed resume with a status-only, MESSAGE-LESS +// ResumeResponse (see ConsumerServiceImpl.resume). The status is therefore handled first, and the +// trailing message's counters are read only when the response actually carries a message - +// otherwise the handler threw a TypeError, which killed the stream and swallowed the server error. +export function handleResumeResponse(res: ResumeResponse, sinks: ResumeResponseSinks): void { + if (res.getStatus()?.getCode() !== Code.OK) { + sinks.notifyError(`${res.getStatus()?.getMessage()}`); + } + + // Before any early return below: while a skip is being resolved these responses carry progress and + // NOTHING else, and once it finishes they stop carrying progress at all. + sinks.setStartFromProgress(readStartFromProgress(res)); + + // The DEGRADATION record is separate from the progress panel on purpose: it has no size + // threshold (a degraded small skip is still degraded) and it is STICKY - the sink keeps it for + // the session's life, because a best-effort answer does not become exact when the frame that + // reported it scrolls away. + const degraded = res.getConsumerStats()?.getStartFromProgress(); + if (degraded?.getDegraded()) { + sinks.setStartFromDegradation(degraded.getAbandonedStreamsList()); + } + + // The best-effort order's late counter rides the same stats. Monotonic on the server, so the + // sink can take the value as-is; zero frames (stats absent) simply leave the last value alone. + const lateDeliveries = res.getConsumerStats()?.getOrderingLateDeliveries() ?? 0; + if (lateDeliveries > 0) { + sinks.setOrderingLateDeliveries(lateDeliveries); + } + + // Whether an ordering layer is ACTUALLY RUNNING - the chip's gate. A single-stream session + // builds none (one log is already in order, no latency is paid), and the chip must not claim + // a window that does not exist. Sticky-true once seen: later frames may omit the stats. + const consumerStats = res.getConsumerStats(); + if (consumerStats?.getDeliveryOrderActive()) { + sinks.setOrderingActive(true); + } + + // Unlike the active flag and counters, "waiting" is a claim about NOW, so it mirrors the frame: + // present means the merge is held, anything else - a data response with no stats at all, stats + // that stopped carrying the field, an inactive layer - means it is moving again and CLEARS the + // status. This deliberately sits OUTSIDE the active-guard above: a frame that stops asserting + // the wait retires it, whatever else the frame says, so the chip cannot stick with a stale + // count (the chip itself stays gated on the sticky active flag). + sinks.setOrderingWaitingStreams(consumerStats?.getDeliveryOrderWaitingStreams() ?? 0); + + // The guaranteed replay's caught-up state mirrors the frame exactly as the waiting status does: + // the server auto-paused the runner at the replay boundary and RE-ASSERTS the state on every + // response while it holds (including the message-less refinement pushes that update the + // newer-entries indicator), and the first frame after a Resume extended the boundary stops + // asserting it - which is what retires the banner. Absent stats mean the same clear: a stale + // record would pin a "caught up" banner over a session that moved on. + sinks.setReplayCaughtUp(consumerStats?.getReplayCaughtUp() + ? { + boundaryAtMs: consumerStats.getReplayBoundaryAtMs(), + excludedTopics: consumerStats.getReplayExcludedTopicsList(), + excludedTopicCount: consumerStats.getReplayExcludedTopicCount(), + } + : undefined); + + // Replay ordering violations (a producer clock wrote an earlier timestamp into a pause window, + // or the source log itself stores an inversion; the replay delivered the message flagged rather + // than silently). Monotonic from the server, exactly like the late counter: zero frames leave + // the last value alone. + const seamViolations = res.getConsumerStats()?.getReplaySeamViolations() ?? 0; + if (seamViolations > 0) { + sinks.setReplaySeamViolations(seamViolations); + } + + // Messages missing the selected timestamp (no broker stamp / no event time) - ordered + // by publish time instead. Monotonic from the server; the first count triggers the one-shot + // remediation notice. + const keyFallbacks = res.getConsumerStats()?.getOrderKeyFallbacks() ?? 0; + if (keyFallbacks > 0) { + sinks.setOrderKeyFallbacks(keyFallbacks); + } + + const newMessages = res.getMessagesList(); + + for (let i = 0; i < newMessages.length; i++) { + if (newMessages[i]?.hasValue()) { + sinks.messagesBuffer.current.push(newMessages[i]); + } + } + + const lastMessage = newMessages[newMessages.length - 1]; + if (lastMessage === undefined) { + return; + } + + sinks.messagesProcessed.current = lastMessage.getNumMessageProcessed() + sinks.messagesLoaded.current = lastMessage.getNumMessageSent() +} + +// What a frame from a SUPERSEDED stream is still allowed to do. Its messages are real - sent and +// acknowledged server-side, in transit when the resume superseded the stream, existing nowhere but +// on it - so they are appended. NOTHING else on the frame may be applied: its counters and stats +// describe an OLDER moment of the same consumer than what the live stream has already reported. +// Applying them would regress the monotonic loaded/processed counters (the 1 s gauge then renders +// a negative rate) and could re-open panels the live stream has already cleared - the skip +// progress, the ordering wait. Ignoring the whole frame instead would silently drop the messages +// (the exact loss the draining design exists to prevent), and max'ing the counters would keep +// stale values out of the numbers but not out of the panels - and would also mask a genuine +// server-side counter reset. Messages only. +export function handleDrainingResumeResponse(res: ResumeResponse, messagesBuffer: { current: Message[] }): void { + const newMessages = res.getMessagesList(); + + for (let i = 0; i < newMessages.length; i++) { + if (newMessages[i]?.hasValue()) { + messagesBuffer.current.push(newMessages[i]); + } + } +} + +export type PauseCapableClient = { + pause: (request: PauseRequest, metadata: Metadata | null) => Promise; +}; + +/** Whether the SERVER confirmed the pause - not whether the UI stopped seeing messages. */ +export type PauseOutcome = 'paused' | 'failed'; + +// A pause that the server refuses (e.g. FAILED_PRECONDITION for a session it doesn't know) comes +// back as a RESOLVED response carrying a non-OK status, not as a rejected call - so the response +// status has to be inspected, or the failure passes unnoticed while the session claims to pause. +// +// The outcome is RETURNED rather than only reported: the session's `paused` state is a claim about +// the server, and a refused pause leaves the server stream running, so the caller has to be able to +// tell the two apart. +export async function pauseConsumer(args: { + client: PauseCapableClient; + consumerName: string; + notifyError: (message: string) => void; +}): Promise { + const pauseReq = new PauseRequest(); + pauseReq.setConsumerName(args.consumerName); + const res = await args.client.pause(pauseReq, { deadline: createDeadline(10) }) + .catch((err) => { + args.notifyError(`Unable to pause consumer ${args.consumerName}. ${err}`); + return undefined; + }); + + if (res === undefined) { + return 'failed'; + } + + if (res.getStatus()?.getCode() !== Code.OK) { + args.notifyError(`Unable to pause consumer ${args.consumerName}. ${res.getStatus()?.getMessage()}`); + return 'failed'; + } + + return 'paused'; +} + +/** + * The stored configuration with a different delivery order - always as a VALUE. + * + * A reference-typed entry names a library item; its `val` is only a browser-local draft, and + * `managed...ValOrRefToPb` sends nothing but the `ref`. Writing the new order into that draft while + * the entry still says `reference` would leave the session describing an order the named item does + * not have AND drop the write on the way to the server - so the entry is converted to a value + * first, exactly as the configuration screen's reference icon does (SessionConfiguration's + * `onConvertToValue`). The session then owns its own copy and the library item is left untouched. + * + * `undefined` when there is no value to rewrite: an unresolved reference has no configuration to + * carry the change, and a half-written one would be worse than none. + */ +export function configValOrRefWithDeliveryOrder( + configValOrRef: ManagedConsumerSessionConfigValOrRef, + order: MessageDeliveryOrder +): ManagedConsumerSessionConfigValOrRef | undefined { + const val = configValOrRef.val; + + if (val === undefined) { + return undefined; + } + + return { type: 'value', val: { ...val, spec: { ...val.spec, messageDeliveryOrder: order } } }; +} + export type SessionProps = { sessionKey: number; configValOrRef: ManagedConsumerSessionConfigValOrRef; @@ -81,25 +306,273 @@ const Session: React.FC = (props) => { const [sort, setSort] = useState({ key: 'publishTime', direction: 'asc' }); const { getWidth: getColumnWidth, startResize: startColumnResize, suppressSortClickRef } = useColumnWidths('consumer-session-messages', messageColumnDefaultWidths); + const toolsPane = useResizablePane('consumer-session-tools', { + // 340px by owner instruction (2026-08-11; was 400rem). A plain pixel value on purpose - the + // persisted drag values are pixels too. + defaultSize: 340, + minSize: remToPx(100), + maxSize: remToPx(1600), + resizeAxis: 'vertical', + side: 'bottom', + }); + const messageInspectorPane = useResizablePane('consumer-session-message-inspector', { + // Preserve the old 600rem default rather than making the panel grow on first render. + defaultSize: remToPx(600), + minSize: remToPx(180), + maxSize: remToPx(2000), + side: 'right', + }); + // Draggable column order for the message table, persisted like the widths are. Index and + // publish time stay pinned in front (they are the sticky pair whose offsets depend on each + // other); everything else reorders freely, and the ROWS follow the header via `columnOrder`. + const { order: messageColumnOrder, moveColumn: moveMessageColumn } = useColumnOrder( + 'consumer-session-messages', + reorderableMessageColumns + ); + const [dragOverMessageColumn, setDragOverMessageColumn] = useState(undefined); + const draggingMessageColumnRef = useRef(undefined); + const columnDragProps = (key: ReorderableMessageColumnKey): React.ThHTMLAttributes => ({ + draggable: true, + onDragStart: (e) => { + if (suppressSortClickRef.current) { + e.preventDefault(); + return; + } + draggingMessageColumnRef.current = key; + e.dataTransfer.setData('text/plain', key); + e.dataTransfer.effectAllowed = 'move'; + }, + onDragEnd: () => { + draggingMessageColumnRef.current = undefined; + setDragOverMessageColumn(undefined); + }, + onDragOver: (e) => { + if (draggingMessageColumnRef.current === undefined) { + return; + } + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + setDragOverMessageColumn(current => (current === key ? current : key)); + }, + onDragLeave: () => setDragOverMessageColumn(current => (current === key ? undefined : current)), + onDrop: (e) => { + e.preventDefault(); + const dragged = draggingMessageColumnRef.current; + draggingMessageColumnRef.current = undefined; + setDragOverMessageColumn(undefined); + if (dragged !== undefined && dragged !== key) { + moveMessageColumn(dragged, key); + } + }, + }); + const resizeProps = (key: MessageColumnKey) => ({ width: getColumnWidth(key), onResizeStart: (x: number) => startColumnResize(key, x), suppressSortClickRef, }); + // The index column's TOTAL cell width (content + the Td's 12rem-a-side padding): the sticky + // offset the publish-time pair sits at. Was the constant 60 while the index column was fixed; + // resizable since 2026-08-11, so the header offset and the body offset (through the + // --cs-index-cell-total variable read by Message.module.css) both track the live width. + const indexCellTotal = getColumnWidth('index') + 24; const [_searchInResults, setSearchInResults] = useState(''); const [searchInResults] = useDebounce(_searchInResults, 1000); + const [startFromProgress, _setStartFromProgress] = useState(undefined); + const startFromProgressRef = useRef(undefined); + // The streams the start-from resolution gave up waiting for. STICKY for the session: set once + // degraded, cleared only when a new session is created (Play after Stop). The setter tolerates + // the same record arriving on every progress frame without re-rendering. + const [startFromDegradation, _setStartFromDegradation] = useState(undefined); + const setStartFromDegradation = useCallback((abandonedStreams: string[] | undefined) => { + _setStartFromDegradation(prev => + prev !== undefined && abandonedStreams !== undefined && prev.join('\u0000') === abandonedStreams.join('\u0000') + ? prev + : abandonedStreams + ); + }, []); + // Selected-time reversals reported by the delivery-order layer - monotonic from the server, + // reset only with the session. + const [orderingLateDeliveries, setOrderingLateDeliveries] = useState(0); + // Whether the server actually runs an ordering layer for this session (single-stream sessions + // build none). Sticky per session, reset with it. + const [orderingActive, setOrderingActive] = useState(false); + // Set only after the server's stall-warning interval, so healthy head-to-head merging does not + // flicker a waiting status on every message. + const [orderingWaitingStreams, setOrderingWaitingStreams] = useState(0); + // Selected-time fallbacks (message carried no broker stamp / event time). The FIRST one raises a + // one-shot notification with the remediation (in the effect below, where the config is in + // scope); the count itself rides on the chip. + const [orderKeyFallbacks, setOrderKeyFallbacks] = useState(0); + const orderKeyNoticeShown = useRef(false); + // The guaranteed replay's caught-up record. Mirrors the frames (see handleResumeResponse), so + // most frames hand over the value it already has - the ref keeps those from re-rendering the + // session, like the skip-progress record above. + const [replayCaughtUp, _setReplayCaughtUp] = useState(undefined); + const replayCaughtUpRef = useRef(undefined); + // Cross-seam ordering violations - monotonic from the server, reset only with the session. + const [replaySeamViolations, setReplaySeamViolations] = useState(0); + // The delivery-order switch point: the processed count at the moment a live Guaranteed session + // was switched away (the caught-up banner's "Switch to Best effort and follow live", or the stall + // escape on the chip). Rows at or below it are the exact replay; rows past it follow live + // traffic in the bounded reorder window - the first such row carries the divider marker. + // Session-scoped: survives later pauses and resumes, cleared only with the session. + const [orderSwitchProcessed, setOrderSwitchProcessed] = useState(undefined); + // Streams this session cancelled itself. grpc-web reports a client-side cancel as a stream + // `error`, and that must not be mistaken for the server dropping the session. + const selfCancelledStreams = useRef>(new WeakSet()); + // Superseded play streams still draining their last in-flight frames. A resume no longer + // cancels its predecessor: whatever the server sent on it was acknowledged AT THE SEND, so the + // frames in transit exist nowhere but on that stream - cancelling it was silent message loss + // (the old multi-second pause hid the window; an instant pause makes it real). The server + // completes the predecessor after its last write; these keep listening until that end, and a + // full session cleanup cancels whatever is still here (a dead session's tail must not append + // ghost rows into a cleared table). + const drainingStreams = useRef>>(new Set()); + // Bumped by every cleanup. A create that was in flight across one of those built a consumer for a + // session that no longer exists, and the consumer's name is generated HERE - nothing else can + // ever name it again, so nothing else could ever delete it. + const sessionGeneration = useRef(0); + // Whether THIS generation's Create ever succeeded. A failed create installs nothing server-side + // (the build releases everything and stores nothing), so cleanup must not send a Delete for it: + // fire-and-forget, that Delete could arrive after a retry's successful Create - same name - and + // remove the healthy replacement. + const createSucceeded = useRef(false); + // Whether the pause currently in effect is one the HIDDEN TAB asked for - the only kind a + // returning tab may undo. + const isPausedByHiddenTab = useRef(false); + // A return that arrived while the hidden-tab Pause was STILL IN FLIGHT. The two calls take the + // same per-session lifecycle lock on the server, but not in the order they were sent: a Resume + // issued now can be granted before the older Pause, which then lands last and leaves the server + // paused under a session that says Running - and a paused consumer sends nothing that would ever + // correct it. So the intent is recorded here and applied when the Pause settles. + const resumeWhenPauseSettles = useRef(false); + + // The browser-wide delivery controls (see local-storage-keys.ts for why they are NOT session + // config). Mirrored into refs because both are read inside stable callbacks - the rate when a + // resume request is built, the pause threshold on every stream chunk. + const [rateLimitSetting] = useLocalStorage(localStorageKeys.consumerSessionRateLimit, { defaultValue: 0 }); + const [pauseAfterSetting] = useLocalStorage(localStorageKeys.consumerSessionPauseAfterLoaded, { defaultValue: 0 }); + const rateLimitRef = useRef(0); + const pauseAfterRef = useRef(0); + const sessionStateRef = useRef('new'); + // The loaded count at which the session pauses itself, or Infinity when disarmed. Armed on every + // entry into `running` at "loaded + n", so Play behaves as "load the next n"; disarmed the + // moment it fires so one crossing triggers exactly one pause. + const nextPauseAtLoaded = useRef(Infinity); + + useEffect(() => { sessionStateRef.current = sessionState; }, [sessionState]); + useEffect(() => { + rateLimitRef.current = Number.isFinite(rateLimitSetting) && rateLimitSetting > 0 ? Math.floor(rateLimitSetting) : 0; + }, [rateLimitSetting]); + useEffect(() => { + pauseAfterRef.current = Number.isFinite(pauseAfterSetting) && pauseAfterSetting > 0 ? Math.floor(pauseAfterSetting) : 0; + // DELIBERATELY NOT re-armed mid-run. The server half of this setting - the delivery budget - + // is fixed when the Resume request is built, so a mid-run edit can only desync the two: the + // server still stops at the OLD n while the client waits for the new one (or, after a clear, + // for infinity), and the session sits in `running` with a silent stream forever. Both halves + // change together on the next Play, which is what the tooltip promises. + }, [pauseAfterSetting]); + + const cancelStream = useCallback((s: ClientReadableStream | undefined) => { + if (s === undefined) { + return; + } + + selfCancelledStreams.current.add(s); + s.cancel(); + }, []); + + // The stream calls this on every response, most of which report nothing - re-rendering the whole + // session for an unchanged value (usually `undefined`) would be pure waste. + const setStartFromProgress = useCallback((progress: StartFromSkipProgress | undefined) => { + const prev = startFromProgressRef.current; + const isSame = prev === undefined + ? progress === undefined + : progress !== undefined && prev.messagesSkipped === progress.messagesSkipped && prev.messagesToSkip === progress.messagesToSkip; + + if (isSame) { + return; + } + + startFromProgressRef.current = progress; + _setStartFromProgress(progress); + }, []); + + // The caught-up sink is where the AUTO-PAUSE lands in the browser: the server pauses the runner + // at the replay boundary and can only say so through a stats frame, so the frame that asserts + // the state moves the session into the ordinary `paused` state - the same state the pause + // button reaches, not a parallel one. Directly to `paused`, not through `pausing`: `pausing` + // exists to await the server's confirmation, and this frame IS the server's confirmation (the + // runner already holds everything; a Pause RPC would be redundant). Frames that stop asserting + // the state retire the record - see handleResumeResponse for the mirror contract. + const setReplayCaughtUp = useCallback((state: ReplayCaughtUpStats | undefined) => { + if (state !== undefined && sessionStateRef.current === 'running') { + setSessionState('paused'); + } + + const prev = replayCaughtUpRef.current; + const isSame = prev === undefined + ? state === undefined + : state !== undefined + && prev.boundaryAtMs === state.boundaryAtMs + && prev.excludedTopicCount === state.excludedTopicCount + && prev.excludedTopics.join('\u0000') === state.excludedTopics.join('\u0000'); + + if (isSame) { + return; + } + + replayCaughtUpRef.current = state; + _setReplayCaughtUp(state); + }, []); const currentTopic = useMemo(() => props.libraryContext.pulsarResource.type === 'topic' ? props.libraryContext.pulsarResource : undefined, [props.libraryContext]); const currentTopicFqn: string | undefined = useMemo(() => currentTopic === undefined ? undefined : `${currentTopic.topicPersistency}://${currentTopic.tenant}/${currentTopic.namespace}/${currentTopic.topic}`, [currentTopic]); const config = useMemo(() => { try { - return consumerSessionConfigFromValOrRef(props.configValOrRef, currentTopicFqn); + const converted = consumerSessionConfigFromValOrRef(props.configValOrRef, currentTopicFqn); + // A start-from the selected topics cannot honour (every history mode, when nothing selected + // retains history) runs as the live tail instead - HERE, as a session-local substitution, + // so the stored configuration keeps exactly what the user wrote (see effectiveStartFrom). + // The conversion above threw on a config whose own `val` is missing, so it is present here. + const startFrom = effectiveStartFrom(converted.startFrom, props.configValOrRef.val!.spec.targets, props.libraryContext); + return startFrom === converted.startFrom ? converted : { ...converted, startFrom }; } catch (err) { console.warn(err); return undefined; } - }, [props.configValOrRef]); + // `currentTopicFqn` is part of the result - a target that follows "the current topic" resolves + // to it. Leaving it out kept a session pointed at the topic it was FIRST rendered with, which + // matters most between `persistent://t/n/x` and `non-persistent://t/n/x`: two different topics + // whose pages differ in nothing else. + }, [props.configValOrRef, currentTopicFqn, props.libraryContext]); + + // Whether ANY topic this session reads keeps no log - the same classification the start-from + // selector uses to disable the history-based positions, so the two cannot disagree about which + // topics retain something. + const hasNonPersistentTargets = useMemo( + () => targetTopicsPersistency(props.configValOrRef.val?.spec.targets ?? [], props.libraryContext).hasNonPersistent, + [props.configValOrRef, props.libraryContext] + ); + + // The selected-time remediation notice, once per session: the first fallback means the configured + // timestamp is absent on real traffic. For broker time that is a BROKER capability gap, and the + // notice names the exact settings an administrator would need - Dekaf itself keeps working. + useEffect(() => { + if (orderKeyFallbacks > 0 && !orderKeyNoticeShown.current) { + orderKeyNoticeShown.current = true; + if (config?.deliveryOrderKey === 'broker-publish-time') { + notifyError( + 'Some messages have no broker publish time, so publish time was used instead. ' + + 'Enable and expose broker timestamp entry metadata to avoid fallbacks.' + ); + } else if (config?.deliveryOrderKey === 'event-time') { + notifyError('Some messages carry no event time, so they were ordered by publish time instead.'); + } + } + }, [orderKeyFallbacks, config, notifyError]); const scrollToBottom = () => { const scrollParent = tableRef.current?.children[0]; @@ -123,7 +596,10 @@ const Session: React.FC = (props) => { setMessages((messages) => { const newMessages = messages .concat(messagesBuffer.current.map(msg => messageDescriptorFromPb(msg))) - .slice(-(config?.numDisplayItems || 0)); + // `slice(-limit)` keeps the whole array for ANY non-positive limit - `slice(-0)` is + // `slice(0)` - so the one place that decides what a limit means decides it here too. + // `|| 0` used to be that decision, and it turned "limit the display" into "do not". + .slice(-displayItemLimit(config?.numDisplayItems)); newMessages.forEach((message, i) => { message.displayIndex = (i + 1); @@ -133,54 +609,231 @@ const Session: React.FC = (props) => { scrollToBottom(); return newMessages; }); - }, messagesLoadedPerSecond.now > 0 ? 250 : false); + // A FIXED cadence, deliberately not gated on the per-second gauge: `paused` now lands on the + // server's confirmation while the last in-flight frames are still arriving, and a gauge-gated + // ticker (the gauge lags a full second) could strand that tail in the buffer, unrendered - a + // sent-and-acknowledged message that exists nowhere but here. An empty buffer returns above. + }, 250); const streamDataHandler = useCallback((res: ResumeResponse) => { - const newMessages = res.getMessagesList(); + handleResumeResponse(res, { + messagesBuffer, + messagesProcessed, + messagesLoaded, + notifyError, + setStartFromProgress, + setStartFromDegradation, + setOrderingLateDeliveries, + setOrderingActive, + setOrderKeyFallbacks, + setOrderingWaitingStreams, + setReplayCaughtUp, + setReplaySeamViolations + }); - for (let i = 0; i < newMessages.length; i++) { - if (newMessages[i]?.hasValue()) { - messagesBuffer.current.push(newMessages[i]); - } + // The auto-pause: the same 'pausing' the toolbar button sends, triggered by the loaded + // counter crossing its armed threshold. Disarm FIRST - chunks keep arriving while the pause + // RPC is in flight, and this handler runs for every one of them. + if (messagesLoaded.current >= nextPauseAtLoaded.current && sessionStateRef.current === 'running') { + nextPauseAtLoaded.current = Infinity; + setSessionState('pausing'); } + }, []); - messagesProcessed.current = newMessages[newMessages.length - 1].getNumMessageProcessed() - messagesLoaded.current = newMessages[newMessages.length - 1].getNumMessageSent() - - if (res.getStatus()?.getCode() !== Code.OK) { - notifyError(`${res.getStatus()?.getMessage()}`); - } + // Installed in place of `streamDataHandler` the moment a stream is superseded (see the effect + // below): the tail frames still deliver their messages, and nothing else. + const drainingDataHandler = useCallback((res: ResumeResponse) => { + handleDrainingResumeResponse(res, messagesBuffer); }, []); + // A resume stream can stop delivering for reasons that are NOT "no messages yet": the transport + // drops, the server completes or aborts the call, the session disappears. Without listeners for + // those the session sits in `running` for ever with frozen counters, waiting for messages that + // can no longer arrive. `paused` is the recoverable landing state - Play resumes from there. useEffect(() => { streamRef.current = stream; - (async () => { - if (stream === undefined) { + if (stream === undefined) { + return; + } + + let isDisposed = false; + const isOurOwnCancel = () => selfCancelledStreams.current.has(stream); + + const stopStreaming = () => { + setSessionState((state) => (state === 'running' || state === 'pausing') ? 'paused' : state); + }; + + const errorHandler = (err: unknown) => { + if (isDisposed || isOurOwnCancel()) { return; } - stream.removeListener('data', streamDataHandler); - stream.on('data', streamDataHandler); - })() + notifyError(`Consumer session stream failed. ${(err as { message?: string })?.message ?? err}`); + stopStreaming(); + }; + const endHandler = () => { + if (isDisposed || isOurOwnCancel()) { + return; + } + + stopStreaming(); + }; + + stream.on('data', streamDataHandler); + stream.on('error', errorHandler); + stream.on('end', endHandler); + + return () => { + isDisposed = true; + stream.removeListener('error', errorHandler); + stream.removeListener('end', endHandler); + // The stream is now a superseded predecessor whose last sent-and-acknowledged frames may + // still be in transit, and they exist nowhere else - so a DATA listener stays attached + // until the server's completion ends it. But not the full one: from here on this stream's + // frames may only deliver their messages (see handleDrainingResumeResponse) - their + // counters and stats describe an older moment than what the live stream reports, and the + // full handler would regress the counters and re-open cleared panels with them. + stream.removeListener('data', streamDataHandler); + stream.on('data', drainingDataHandler); + drainingStreams.current.add(stream); + const forget = () => { + drainingStreams.current.delete(stream); + stream.removeListener('data', drainingDataHandler); + }; + stream.on('end', forget); + stream.on('error', forget); + }; }, [stream]); + const deleteConsumer = useCallback(async (name: string) => { + const deleteConsumerReq = new DeleteConsumerRequest(); + deleteConsumerReq.setConsumerName(name); + await consumerServiceClient.deleteConsumer(deleteConsumerReq, { deadline: createDeadline(10) }) + .catch((err) => notifyError(`Unable to delete consumer ${name}. ${err}`)); + }, []); + + // Switches the LIVE session's delivery order without recreating it - recreating would re-read + // from the start position and throw away everything already held. Reached from the two places + // a Guaranteed session offers the way out: the caught-up banner's "Continue live with Best + // effort", and the stall chip's escape for the time-seek corner (a stream a seek positioned + // past its end). + // + // Three things, in this order and only in this order: + // 1. the LIVE session is switched, which is what releases the held set (in merge order, once, + // with the counted start-from budget, ack identity and flow control preserved); + // 2. the stored configuration is updated, so the next Play asks for what the session is now + // actually doing - see configValOrRefWithDeliveryOrder for the reference case; + // 3. the disclosed wait is retired locally. + // A server that refused the change earns none of the other two. + // + // The outcome is RETURNED, like pauseConsumer's: the caught-up banner resumes the session on a + // granted switch, and it may only do that when the server really switched. + const applyDeliveryOrder = async (order: MessageDeliveryOrder): Promise<'applied' | 'failed'> => { + const req = new SetDeliveryOrderRequest(); + req.setConsumerName(consumerName.current); + req.setMessageDeliveryOrder(messageDeliveryOrderToPb(order)); + + const res = await consumerServiceClient.setDeliveryOrder(req, { deadline: createDeadline(10) }) + .catch((err) => { + notifyError(`Unable to change the delivery order. ${(err as { message?: string })?.message ?? err}`); + return undefined; + }); + + if (res === undefined) { + return 'failed'; + } + + // A refusal - a direction the server does not perform on a running session, a session it no + // longer knows - comes back as a RESOLVED response carrying the reason, exactly like Pause. It + // is the reason the user needs, so it is shown rather than swallowed; the live session did not + // change, so nothing below may claim it did. + if (res.getStatus()?.getCode() !== Code.OK) { + notifyError(`Unable to change the delivery order. ${res.getStatus()?.getMessage()}`); + return 'failed'; + } + + // The SWITCH POINT, recorded before anything resumes: everything delivered so far was the + // exact replay (or its held merge), everything after follows live traffic in the bounded + // reorder window - the # column changes meaning here, and the first row past this count + // carries the divider marker that says so. + if (config?.messageDeliveryOrder === 'guaranteed' && order !== 'guaranteed') { + setOrderSwitchProcessed(messagesProcessed.current); + } + + const nextConfigValOrRef = configValOrRefWithDeliveryOrder(props.configValOrRef, order); + if (nextConfigValOrRef !== undefined) { + props.onConfigValOrRefChange(nextConfigValOrRef); + } + + // The wait is over - the server released the held set - but "waiting" only ever arrives ON a + // response frame (see handleResumeResponse). If the switch released nothing and the session + // then stays silent, no frame arrives to retire the status and the chip goes on claiming a + // wait that ended. Clearing it here cannot go stale in the other direction either: the next + // real frame states the truth again, whatever it is. + setOrderingWaitingStreams(0); + return 'applied'; + }; + + // The caught-up banner's "Switch to Best effort and follow live": the granted switch releases the + // boundary holds server-side and live delivery resumes - so the session follows it back into + // `running`, which also starts a fresh resume stream through the ordinary path (the superseded + // one drains, as every resume's predecessor does). A refused switch was already reported and + // changed nothing, so the session stays paused at its boundary, banner intact. + const continueLiveWithBestEffort = async () => { + const outcome = await applyDeliveryOrder('best-effort'); + if (outcome === 'applied') { + setSessionState('running'); + } + }; + + const cleanup = useCallback(async () => { console.info(`%cCleaning up session: ${props.sessionKey}`, consoleCss); - streamRef.current?.cancel(); + // Whatever was still being created belongs to a session that no longer exists. The bump is what + // tells that in-flight create so - see createConsumer below. + sessionGeneration.current += 1; + + cancelStream(streamRef.current); streamRef.current?.removeListener('data', streamDataHandler); + // Predecessors still draining their tails belong to this same dead session: their frames must + // not append into the cleared table below. They carry the drain-only listener - the full one + // was removed when each was superseded. + drainingStreams.current.forEach((s) => { + cancelStream(s); + s.removeListener('data', drainingDataHandler); + }); + drainingStreams.current.clear(); setMessages([]); - - async function deleteConsumer() { - const deleteConsumerReq = new DeleteConsumerRequest(); - deleteConsumerReq.setConsumerName(consumerName.current); - await consumerServiceClient.deleteConsumer(deleteConsumerReq, { deadline: createDeadline(10) }) - .catch((err) => notifyError(`Unable to delete consumer ${consumerName.current}. ${err}`)); + // A NEW session gets a clean slate: the degradation record belongs to the session that + // produced it, not to the page. Clearing it unmounts the banner, which is also what makes the + // next disclosure start expanded and re-bounded again. + setStartFromDegradation(undefined); + // The skip-progress record is as session-scoped as its siblings above: left in place it + // would resurface the DEAD session's "Skipping n messages..." numbers the moment a new run + // renders with an empty table. + setStartFromProgress(undefined); + setOrderingLateDeliveries(0); + setOrderingActive(false); + setOrderingWaitingStreams(0); + setOrderKeyFallbacks(0); + orderKeyNoticeShown.current = false; + // The replay records are as session-scoped as the ordering ones above: a caught-up banner, a + // seam count or a switch-point divider left behind would describe a session that no longer + // exists, over a table that starts empty. + setReplayCaughtUp(undefined); + setReplaySeamViolations(0); + setOrderSwitchProcessed(undefined); + + // Only a consumer that was actually INSTALLED gets a cleanup Delete. A failed create stored + // nothing server-side, and its fire-and-forget Delete could arrive AFTER a retry's successful + // Create under the same name - deleting the healthy replacement. + if (createSucceeded.current) { + deleteConsumer(consumerName.current); // Don't await this } - - deleteConsumer(); // Don't await this + createSucceeded.current = false; }, [prevSessionState, sessionState]); useEffect(() => { @@ -189,55 +842,146 @@ const Session: React.FC = (props) => { } }, []); + // A closing tab has to delete the consumer too, and `window` outlives every session - so the + // registration must be OWNED by an effect. Registering it from the initialize path left one stale + // closure installed per Stop remount, each still holding a dead session's consumer name, and an + // eventual unload then fired a Delete for every session the tab had ever run. + // + // Nothing exists to delete before the first Play, so `new` registers nothing at all. + useEffect(() => { + if (sessionState === 'new') { + return; + } + + window.addEventListener('beforeunload', cleanup); + return () => window.removeEventListener('beforeunload', cleanup); + }, [cleanup, sessionState]); + const initializeSession = () => { async function createConsumer() { if (config === undefined) { + // Play is disabled in this state, so this is a guard rather than a path; still, staying on + // `initializing` would read as a hang. + notifyError('This session configuration could not be read. Check the configuration, or start a new session.'); + setSessionState('new'); return; } const req = new CreateConsumerRequest(); req.setConsumerName(consumerName.current); - const consumerSessionConfigPb = consumerSessionConfigToPb(config); + + // Building the request is where the parts of the configuration that are still TEXT get + // parsed - a message id, for one - so it can fail on a configuration that converted fine. + // Unhandled, the rejection killed this function silently and left the session on + // "initializing" for ever, looking like a hang instead of a configuration error. + let consumerSessionConfigPb; + try { + consumerSessionConfigPb = consumerSessionConfigToPb(config); + } catch (err) { + notifyError(`Unable to create consumer ${consumerName.current}. ${(err as Error)?.message ?? err}`); + setSessionState('new'); + return; + } + req.setConsumerSessionConfig(consumerSessionConfigPb); + // Which session this create belongs to. Everything after the await has to be checked against + // it: the user can Stop - which remounts the session - while the server is still working. + const generation = sessionGeneration.current; + const res = await consumerServiceClient.createConsumer(req, {}).catch(err => notifyError(`Unable to create consumer ${consumerName.current}. ${err}`)); + + // A create that did not succeed leaves NOTHING running: no consumer, no stream, nothing that + // can ever move the session on. Staying on `initializing` therefore claims a round trip is + // still in progress AND disables Play, so the only way out was Stop - which throws away every + // message loaded so far. `new` is the honest state, and Play can retry from it. if (res === undefined) { + setSessionState('new'); return; } const status = res.getStatus(); const code = status?.getCode(); - if (code === Code.OK) { - setSessionState('running'); - } - if (code !== Code.OK) { const errorMessage = status?.getMessage(); notifyError(`Unable to create consumer. ${errorMessage}`); + setSessionState('new'); + return; + } + + // The consumer now exists on the server, subscribed and consuming - but the session that + // asked for it is gone, so nothing on screen owns it and nothing knows its name. Delete it + // rather than leaving it running for the lifetime of the process. + if (generation !== sessionGeneration.current) { + deleteConsumer(consumerName.current); + return; + } + + createSucceeded.current = true; + // The tab can have gone HIDDEN while the server was still building the consumer. Resuming + // into a hidden tab starts a stream nobody is watching and - worse - arms nothing that the + // hidden-tab machinery would undo, so returning to the tab found it stalled with no + // recovery. The consumers exist but were never resumed (their gates start closed), so + // `paused` is the honest state; the visibility handler's ordinary return path then resumes + // it the moment the tab is visible again. + if (document.visibilityState === 'hidden') { + isPausedByHiddenTab.current = true; + setSessionStateBeforeWindowBlur('running'); + setSessionState('paused'); return; } + setSessionState('running'); } createConsumer(); - - window.addEventListener('beforeunload', cleanup); - return () => { - window.removeEventListener('beforeunload', cleanup); - }; }; // Stream's connection pauses on window blur and we don't receive new messages. // Here we are trying to handle this situation. + // + // Only `running` owns a live server stream, and only a live stream can stall. `new` and + // `initializing` have no consumer on the server at all, and `paused` already stopped one: asking + // the server to pause any of those asks about a session it does not know, and its refusal is then + // read as "still running" - which offers Resume for a consumer that never existed. Sessions the + // USER paused must also stay paused when the tab comes back, so only a pause this handler caused + // is undone here. const handleVisibilityChange = () => { if (document.visibilityState === 'hidden') { + // Hidden again before the Pause it already asked for came back: that Pause IS what this + // wants, so only the recorded return is withdrawn. + if (sessionState === 'pausing' && resumeWhenPauseSettles.current) { + resumeWhenPauseSettles.current = false; + isPausedByHiddenTab.current = true; + return; + } + + if (sessionState !== 'running') { + return; + } + + isPausedByHiddenTab.current = true; setSessionStateBeforeWindowBlur(sessionState); setSessionState('pausing'); return; } if (document.visibilityState === 'visible') { + if (!isPausedByHiddenTab.current) { + return; + } + + isPausedByHiddenTab.current = false; + + // The Pause is still on the wire. Resuming now would race it (see resumeWhenPauseSettles): + // record the intent and stay in `pausing`, and the pause handler below applies it as soon as + // the server has answered. + if (sessionState === 'pausing') { + resumeWhenPauseSettles.current = true; + return; + } + setSessionState(sessionStateBeforeWindowBlur); return; } @@ -261,12 +1005,44 @@ const Session: React.FC = (props) => { if (sessionState === 'pausing') { console.info(`%cPausing session: ${props.sessionKey}`, consoleCss); - const pauseReq = new PauseRequest(); - pauseReq.setConsumerName(consumerName.current); - consumerServiceClient.pause(pauseReq, { deadline: createDeadline(10) }) - .catch((err) => notifyError(`Unable to pause consumer ${consumerName.current}. ${err}`)); + let isAbandoned = false; - return; + pauseConsumer({ client: consumerServiceClient, consumerName: consumerName.current, notifyError }) + .then((outcome) => { + if (isAbandoned) { + return; + } + + // The tab came back while this Pause was in flight. The server has now answered, so the + // recorded return can be carried out without the two operations racing for the session + // lock - and whichever way the Pause went, the state the user asked for is `running` + // (a refused pause left the stream live; a granted one is resumed from here). + if (resumeWhenPauseSettles.current) { + resumeWhenPauseSettles.current = false; + setSessionState(sessionStateBeforeWindowBlur); + return; + } + + if (outcome === 'paused') { + // The server's confirmation IS the pause: its intake is closed and everything not yet + // sent is held for the next resume. The in-flight tail keeps rendering into the paused + // view (the flush ticker below runs regardless of state), so flipping here loses + // nothing - it only stops making the user wait for a cosmetic quiet second. + setSessionState('paused'); + return; + } + + // The server did not pause: its stream is still live and messages can still arrive, so + // the honest state is the one it is actually in. + setSessionState('running'); + }); + + return () => { + isAbandoned = true; + // Whatever leaves `pausing` - the answer above, a Play, a Stop - decides the state itself, + // so a recorded return must not outlive this pause and fire on the next one. + resumeWhenPauseSettles.current = false; + }; } if (sessionState === 'paused') { @@ -282,11 +1058,45 @@ const Session: React.FC = (props) => { if (sessionState === 'running') { console.info(`%cRunning session: ${props.sessionKey}`, consoleCss); + // Whatever skip progress is on record belongs to the PREVIOUS stream (a pause mid-skip + // leaves the last frame's numbers behind). The stream created below reports its own + // start-from work from its first frame; until then there is nothing to report - and if the + // skip already resolved and the topic is idle, no frame ever arrives to clear a stale + // "Skipping n messages..." panel over a session that is simply waiting. + setStartFromProgress(undefined); + + // The caught-up record belongs to the boundary pause that just ended: the Resume being + // issued below extends the boundary. The banner is gated on `paused` anyway, but the record + // itself must not linger - an idle topic sends no frame to retire it, and a later MANUAL + // pause would then resurface a "caught up" banner the server never asserted for it. + setReplayCaughtUp(undefined); + + // Arm the auto-pause for this run: "the next n loaded from here". Every entry into + // `running` re-arms, so resuming a session paused at n loads n more. + nextPauseAtLoaded.current = pauseAfterRef.current > 0 ? messagesLoaded.current + pauseAfterRef.current : Infinity; + const resumeReq = new ResumeRequest(); resumeReq.setConsumerName(consumerName.current); - stream?.cancel(); - stream?.removeListener('data', streamDataHandler); - const newStream = consumerServiceClient.resume(resumeReq, { deadline: createDeadline(60 * 10) }); + // The start-from progress rides along on ConsumerStats; without this the server has no reason + // to compute or send it. + resumeReq.setIncludeConsumerStats(true); + // The browser-wide delivery rate limit rides every resume - see local-storage-keys.ts for + // why it is per-request rather than session config. 0 = unlimited. + resumeReq.setMaxMessagesPerSecond(rateLimitRef.current); + // The server-side half of "pause after n": deliver at most n more on this stream, counted + // at the send. The client threshold below still drives the state machine, but the SERVER + // guarantees the count - a chunk that never leaves the server cannot overshoot a screenful. + resumeReq.setMaxMessagesToDeliver(pauseAfterRef.current); + // The predecessor stream is NOT cancelled: with pause landing on the server's confirmation, + // a prompt resume arrives while the pause's last frames - sent and acknowledged server-side - + // are still in transit on it. A cancel here dropped them, silently and permanently. Instead + // the effect cleanup keeps it draining; the server completes it after its final write (one + // resume generation at a time), and that end is what retires it. + // NO DEADLINE: resume is a long-lived server stream that lives until the user stops it or the + // transport drops. Any fixed budget is a wrong guess - a Skip-N over millions of messages can + // spend longer than that resolving before it delivers its first message, and the deadline + // would kill the stream mid-skip. The error/end listeners are what notice a stream that ends. + const newStream = consumerServiceClient.resume(resumeReq, {}); setStream(() => newStream); return; } @@ -304,12 +1114,6 @@ const Session: React.FC = (props) => { } }, [sessionState]); - useEffect(() => { - if (sessionState === 'pausing' && messagesLoadedPerSecond.now === 0) { - setSessionState('paused'); - } - }, [sessionState, messagesLoadedPerSecond]); - const isShowTooltips = sessionState !== 'running' && sessionState !== 'pausing'; const valueProjectionThs = useMemo(() => { @@ -320,6 +1124,26 @@ const Session: React.FC = (props) => { }) : []; }, [config, sort, setSort]); + // The row that carries the switch-point divider: the FIRST message delivered after the + // delivery-order switch - the smallest processed count past the watermark among the retained + // rows. Attached to the MESSAGE rather than to a table offset, so it renders wherever that row + // renders: it survives scrolling (the table is virtualized), retention trimming and re-sorting, + // and there is exactly one of it because processed counts are unique. + const orderSwitchMarkerProcessed = useMemo(() => { + if (orderSwitchProcessed === undefined) { + return undefined; + } + + let first: number | undefined = undefined; + messages.forEach((message) => { + if (message.numMessageProcessed !== null && message.numMessageProcessed > orderSwitchProcessed + && (first === undefined || message.numMessageProcessed < first)) { + first = message.numMessageProcessed; + } + }); + return first; + }, [messages, orderSwitchProcessed]); + const itemContent = useCallback>((i, message) => { if (config === undefined) { return; @@ -334,9 +1158,13 @@ const Session: React.FC = (props) => { isShowTooltips={isShowTooltips} sessionState={sessionState} selectedMessages={selectedMessages} + columnOrder={messageColumnOrder} coloring={coloring} valueProjectionThs={valueProjectionThs} getColumnWidth={getColumnWidth} + isOrderSwitchBoundary={ + message.numMessageProcessed !== null && message.numMessageProcessed === orderSwitchMarkerProcessed + } onClick={() => { if (sessionState !== 'paused') { setSessionState('pausing'); @@ -350,7 +1178,7 @@ const Session: React.FC = (props) => { }} /> ); - }, [sessionState, config, selectedMessages, getColumnWidth]); + }, [sessionState, config, selectedMessages, getColumnWidth, orderSwitchMarkerProcessed]); const onWheel = useCallback>((e) => { if (e.deltaY < 0 && sessionState === 'running') { @@ -359,6 +1187,27 @@ const Session: React.FC = (props) => { }, [sessionState]); const currentView: View = sessionState === 'new' ? 'configuration' : 'messages'; + + // THE CAUGHT-UP PANEL IS RENDERED BY THE SESSION ITSELF, docked bottom-right - not an inline + // strip above the table (the state is worth acting on, not worth a permanent band of chrome), + // and NOT a notification. It lived in a persistent toast for a day (2026-08-11) and every + // boundary hand-off was a race against react-toastify's lifecycle: a toast created under a + // still-exiting id is silently DROPPED ("Load new messages" -> dismiss -> re-announce produced + // exactly that, and the panel never returned - e2e CS-DM-R2), and per-episode fresh ids traded + // that for the retiring panel and its successor COEXISTING while the exit animation played - + // an instant catch-up re-announces within milliseconds of the dismissal, faster than any exit + // (two panels on screen; e2e CS-DM-R3B). Removal mediated by an animation cannot be made + // synchronous, so the panel left the toast layer instead: rendered conditionally below, at + // most one can exist, a close is plain state, and unmounting takes it along. + // + // The close is scoped to the EPISODE: a new caught-up record re-opens the panel (the sink + // collapses equal records, so the identity only changes when the boundary or its excluded + // set does - new information, worth re-showing), and every resume clears the record, so the + // next boundary starts open again. + const [caughtUpPanelClosed, setCaughtUpPanelClosed] = useState(false); + useEffect(() => { + setCaughtUpPanelClosed(false); + }, [replayCaughtUp]); const messagesToShow = useMemo(() => { let msgs = searchInResults === '' ? messages : messages.filter(msg => { return msg.key?.includes(searchInResults) || msg.value?.includes(searchInResults); @@ -377,38 +1226,84 @@ const Session: React.FC = (props) => { className={s.ConsumerSession} data-testid="cs-session" data-cs-state={sessionState} - style={{ gridTemplateRows: props.isShowConsole ? 'min-content 1fr 400rem' : 'min-content 1fr 0' }} + // How many messages are actually being held, after retention. The count of rendered rows is + // not this number - the table is virtualized - and nothing else on screen reports it. + data-cs-retained={messages.length} + style={{ + // Keep the message/configuration row reachable when a height saved on a large display is + // restored in a shorter window. The stored pixel value remains the user's preference. + gridTemplateRows: props.isShowConsole + ? `min-content minmax(0, 1fr) min(${toolsPane.size}px, 85%)` + : 'min-content minmax(0, 1fr) 0' + }} > - props.onSetIsShowConsole(!props.isShowConsole)} - searchInResults={_searchInResults} - onSearchInResultsChange={setSearchInResults} - numFoundInResults={messagesToShow.length} - /> - + {/* ONE grid child for the toolbar and whatever is disclosed under it: the grid declares + exactly three rows (top, content, console), and a banner as its own direct child used to + shift the content into the console's zero-height row while the console spilled into an + implicit fourth. */} +
    + props.onSetIsShowConsole(!props.isShowConsole)} + searchInResults={_searchInResults} + onSearchInResultsChange={setSearchInResults} + numFoundInResults={messagesToShow.length} + orderingLateDeliveries={orderingLateDeliveries} + orderingActive={orderingActive} + orderKeyFallbacks={orderKeyFallbacks} + orderingWaitingStreams={orderingWaitingStreams} + replaySeamViolations={replaySeamViolations} + onDeliveryOrderChange={applyDeliveryOrder} + /> + + {currentView === 'messages' && startFromDegradation !== undefined && ( + + )} + + {/* What this pause COSTS, stated while it is in effect. A paused session hands its + prefetched messages back for redelivery and picks them up on resume - which recovers + them only where the broker still has them. A non-persistent topic keeps no log, so the + pause is a gap in what the user sees and no resume can close it. Scoped to the paused + state on purpose: a session that is running misses nothing, and a caveat shown always + is a caveat nobody reads. */} + {(sessionState === 'paused' || sessionState === 'pausing') && hasNonPersistentTargets && ( +
    + The non-persistent topics in this session keep no history. Messages published to them while the session + is paused are missed, and resuming does not recover them. +
    + )} +
    {currentView === 'messages' && messages.length === 0 && (
    {sessionState === 'initializing' && 'Initializing session...'} - {sessionState === 'running' && 'Awaiting for new messages...'} + {/* A big skip delivers nothing until it lands, so this is exactly where the session would + otherwise sit on "Awaiting for new messages..." looking hung. */} + {sessionState === 'running' && (startFromProgress === undefined + ? 'Awaiting for new messages...' + : )} {sessionState === 'paused' && 'No messages where loaded.'}
    )} {currentView === 'messages' && messages.length > 0 && ( -
    +
    = (props) => { setSort={setSort} sortKey="index" style={{ position: 'sticky', left: 0, zIndex: 10 }} + {...resizeProps('index')} help={( <>

    - When consuming from multiple topics or a single partitioned topic, the order of messages cannot be assured. + Messages are numbered in the order this session delivers them.

    - The order of numbers in in this column represents the order in which messages were received by the consumer. + Pulsar preserves order within a topic or partition, but not across them. Guaranteed + and Best effort merge by the selected time; Fastest shows each stream independently.

    )} @@ -468,163 +1365,33 @@ const Session: React.FC = (props) => { sort={sort} setSort={setSort} sortKey="publishTime" - style={{ position: 'sticky', left: remToPx(60), zIndex: 10 }} + style={{ position: 'sticky', left: remToPx(indexCellTotal), zIndex: 10 }} help={help.publishTime} {...resizeProps('publishTime')} /> - - - {valueProjectionThs.map(vp => vp.th)} - - - - - - - - - - - - - - - + {messageColumnOrder.flatMap((columnKey) => { + const meta = messageThMeta[columnKey]; + const cells = [( + + )]; + // Value-projection columns stay glued after KEY wherever it sits - the same + // neighbourhood they have always rendered in. + if (columnKey === 'key') { + cells.push(...valueProjectionThs.map(vp => vp.th)); + } + return cells; + })} )} /> @@ -632,6 +1399,14 @@ const Session: React.FC = (props) => { {messageDetails !== undefined && (
    +
    = (props) => { consumerName={consumerName.current} currentTopic={currentTopicFqn} libraryContext={props.libraryContext} + onResizeStart={toolsPane.startResize} + resizePane={toolsPane} /> + + {/* Docked, fixed-position, out of the grid's layout entirely. Rendering it HERE is the + correctness: one conditional element cannot become two panels, whatever the timing of + boundary hand-offs (see the caughtUpPanelClosed comment above for the toast history). */} + {currentView === 'messages' && sessionState === 'paused' && replayCaughtUp !== undefined && !caughtUpPanelClosed && ( +
    + setSessionState('running')} + onContinueWithBestEffort={continueLiveWithBestEffort} + onClose={() => setCaughtUpPanelClosed(true)} + /> +
    + )}
    ); } @@ -684,20 +1475,56 @@ type ConsumerSessionProps = { const ConsumerSession: React.FC = (props) => { const [sessionKey, setSessionKey] = useState(0); const [config, setConfig] = useState(props.initialConfig); - const [isShowConsole, setIsShowConsole] = useState(false); + // This is a browser preference, not part of a saved consumer-session configuration. Read the + // durable value as unknown so corrupted/legacy storage cannot make the panel disappear forever. + // VISIBLE by default (owner decision 2026-08-11, reversing the closed-by-default from earlier + // the same day). A stored preference still wins - this is only what a browser that has never + // expressed one gets. + const [storedIsShowConsole, setStoredIsShowConsole] = useLocalStorage(localStorageKeys.consumerSessionToolsOpen, { + defaultValue: true + }); + const isShowConsole = typeof storedIsShowConsole === 'boolean' ? storedIsShowConsole : true; + + useEffect(() => { + if (typeof storedIsShowConsole !== 'boolean') { + setStoredIsShowConsole(true); + } + }, [storedIsShowConsole, setStoredIsShowConsole]); return ( - setIsShowConsole(!isShowConsole)} - {...props} - onStopSession={() => setSessionKey(n => n + 1)} - configValOrRef={config} - onConfigValOrRefChange={setConfig} - libraryContext={props.libraryContext} - /> + // A saved session can reference any persisted library item, and a stored item can be of a + // foreign type or incomplete. Without a boundary, such a config throws while rendering and React + // unmounts the whole app - the route then shows an EMPTY document with no way back. Keep the + // failure local and visible instead. + ( + + This consumer session could not be rendered. +
    + {String(error?.message || error)} +
    + Check the session configuration, or start a new session. +
    + )} + /> + )} + > + setSessionKey(n => n + 1)} + configValOrRef={config} + onConfigValOrRefChange={setConfig} + libraryContext={props.libraryContext} + /> + ); } diff --git a/ui/components/ui/ConsumerSession/Message/Field/Field.module.css b/ui/components/ui/ConsumerSession/Message/Field/Field.module.css index 13d9d1164..6609a7837 100644 --- a/ui/components/ui/ConsumerSession/Message/Field/Field.module.css +++ b/ui/components/ui/ConsumerSession/Message/Field/Field.module.css @@ -27,6 +27,12 @@ opacity: 0.5; } +.ClickableFieldValue:focus-visible { + border-radius: 2rem; + outline: 2px solid var(--accent-color, #4a72ff); + outline-offset: 2px; +} + .FieldValueLink { color: inherit !important; } @@ -38,4 +44,3 @@ .NoData { color: #aaa; } - diff --git a/ui/components/ui/ConsumerSession/Message/Field/Field.test.tsx b/ui/components/ui/ConsumerSession/Message/Field/Field.test.tsx new file mode 100644 index 000000000..3b85cea75 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Message/Field/Field.test.tsx @@ -0,0 +1,75 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * What the copy toast CALLS the value it just copied. + * + * The default names the column (" value copied"), which reads correctly while the column + * title names a field. It stopped reading correctly once the same Field was reused for Topic + * Positions, whose topic column is headed "Topic / partition" - a heading, not a thing you can + * hold, so the toast said "Topic / partition value copied to clipboard." `copyLabel` is the + * override for exactly that case, and these two tests are what keep the default from quietly + * coming back for it. + */ +import React from 'react'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import Field from './Field'; +import { defaultValue as notifications } from '../../../../app/contexts/Notifications'; +import { copyToClipboard } from '../../../../app/clipboard'; + +// The module's exports are not configurable, so the module - not the binding - is what gets mocked. +jest.mock('../../../../app/clipboard', () => ({ + copyToClipboard: jest.fn(), + copyFailureMessage: () => 'copy failed', +})); + +const renderField = (props: { title?: string; copyLabel?: string }) => + render( + <Field + title={props.title} + copyLabel={props.copyLabel} + value="persistent://t/ns/orders" + rawValue="persistent://t/ns/orders" + tooltip={undefined} + isShowTooltips={false} + testId="field-under-test" + /> + ); + +describe('the copy-to-clipboard toast', () => { + let reported: string[] = []; + + beforeEach(() => { + reported = []; + (copyToClipboard as jest.Mock).mockResolvedValue(true); + jest.spyOn(notifications, 'notifySuccess').mockImplementation((content) => { + reported.push(String(content)); + }); + }); + + afterEach(() => { + jest.restoreAllMocks(); + cleanup(); + }); + + it('names the column by default - correct wherever the title names a field', async () => { + renderField({ title: 'Publish time' }); + + fireEvent.click(screen.getByTestId('field-under-test')); + await Promise.resolve(); + + expect(reported).toEqual(['Publish time value copied to clipboard.']); + }); + + it('uses copyLabel when the column HEADING would not read as a thing you can hold', async () => { + // Topic Positions' topic column: headed "Topic / partition", but one FQN lands on the + // clipboard - so the toast names that, and never says "Topic / partition value". + renderField({ title: 'Topic / partition', copyLabel: 'Topic FQN' }); + + fireEvent.click(screen.getByTestId('field-under-test')); + await Promise.resolve(); + + expect(reported).toEqual(['Topic FQN copied to clipboard.']); + expect(reported[0]).not.toContain('Topic / partition'); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Message/Field/Field.tsx b/ui/components/ui/ConsumerSession/Message/Field/Field.tsx index 3155e4610..a81b0737f 100644 --- a/ui/components/ui/ConsumerSession/Message/Field/Field.tsx +++ b/ui/components/ui/ConsumerSession/Message/Field/Field.tsx @@ -9,6 +9,10 @@ export type FieldProps = { isShowTooltips: boolean, rawValue?: string, title?: string, + /** What the copy toast calls this value. Defaults to "<title> value", which reads correctly for + * columns whose title names a field ("Publish time value copied") - set it where the column + * title is a heading rather than a thing you can hold ("Topic / partition" -> "Topic FQN"). */ + copyLabel?: string, valueHref?: string, testId?: string, } @@ -22,7 +26,7 @@ const Field: React.FC<FieldProps> = (props) => { } void copyToClipboard(props.rawValue).then((ok) => { - if (ok) notifySuccess(`${props.title} value copied to clipboard.`); + if (ok) notifySuccess(`${props.copyLabel ?? `${props.title} value`} copied to clipboard.`); else notifyWarn(copyFailureMessage()); }); } @@ -35,15 +39,32 @@ const Field: React.FC<FieldProps> = (props) => { "data-tooltip-html": (!props.isShowTooltips || props.rawValue === undefined) ? undefined : "Click to copy" } : {}; + const handleCopyKeyDown = (event: React.KeyboardEvent<HTMLDivElement>) => { + if (props.rawValue === undefined || (event.key !== 'Enter' && event.key !== ' ')) { + return; + } + + // A copyable value is an interaction even though it deliberately remains a div to preserve + // the compact table layout. Give keyboard users the same action as a click; preventing the + // Space default also keeps the surrounding message list from scrolling unexpectedly. + event.preventDefault(); + event.stopPropagation(); + copyRawValue(); + }; + let valueElement = ( <div className={`${s.FieldValue} ${props.rawValue === undefined ? '' : s.ClickableFieldValue}`} title={props.rawValue} data-testid={props.testId} + role={props.rawValue === undefined ? undefined : 'button'} + tabIndex={props.rawValue === undefined ? undefined : 0} + aria-label={props.rawValue === undefined ? undefined : `Copy ${props.title ?? 'value'}`} onClick={(event) => { event.stopPropagation(); copyRawValue(); }} + onKeyDown={handleCopyKeyDown} {...dataTooltipProps} > {valueContent} diff --git a/ui/components/ui/ConsumerSession/Message/Message.module.css b/ui/components/ui/ConsumerSession/Message/Message.module.css index e02b5cbbb..4439354cb 100644 --- a/ui/components/ui/ConsumerSession/Message/Message.module.css +++ b/ui/components/ui/ConsumerSession/Message/Message.module.css @@ -34,7 +34,9 @@ tr:has(> .Td:hover) { .PublishTimeField { position: sticky; - left: 60rem; + /* Tracks the resizable index column (set on the table by ConsumerSession); 60rem is the + default-width fallback. */ + left: var(--cs-index-cell-total, 60rem); z-index: 1; } @@ -43,3 +45,47 @@ tr:has(> .Td:hover) { left: 285rem; z-index: 1; } + +/* The row-level out-of-order disclosure (Guaranteed's seam violations and, since 2026-08-11, + Best effort's late emissions). Sits beside the index so it is visible at any horizontal + scroll, and centers itself in the row - the index cell's flex box would otherwise hang it + from the text baseline. */ +.OutOfOrderMarker { + flex: none; + align-self: center; + margin-left: 4rem; + width: 14rem; + height: 14rem; + line-height: 12rem; + text-align: center; + border: 1rem solid currentColor; + border-radius: 50%; + font-size: 10rem; + font-weight: var(--font-weight-bold, bold); + color: var(--warning-text-color, #7a5b00); + background: var(--warning-background-color, #fff7e0); + cursor: help; +} + +/* The delivery-order switch point: a divider band across the first row delivered after + "Switch to Best effort and follow live". Data-driven per row (not a DOM insertion at an offset), so it + survives scrolling, retention trimming and re-sorting. */ +.OrderSwitchBoundaryTd { + border-top: 3rem solid var(--accent-color, #4a72ff) !important; +} + +.OrderSwitchPointBadge { + position: absolute; + top: -2rem; + left: 2rem; + transform: translateY(-50%); + z-index: 5; + background: var(--accent-color, #4a72ff); + color: #fff; + font-size: 10rem; + line-height: 14rem; + padding: 0 6rem; + border-radius: 3rem; + white-space: nowrap; + cursor: help; +} diff --git a/ui/components/ui/ConsumerSession/Message/Message.tsx b/ui/components/ui/ConsumerSession/Message/Message.tsx index 394af3fd9..bde325b77 100644 --- a/ui/components/ui/ConsumerSession/Message/Message.tsx +++ b/ui/components/ui/ConsumerSession/Message/Message.tsx @@ -5,7 +5,8 @@ import { ConsumerSessionConfig, MessageDescriptor, SessionState } from '../types import { Coloring } from '../coloring'; import { getValueProjectionTds, ValueProjectionTh } from '../value-projections/value-projections-utils'; import { Td } from './Td'; -import { MessageColumnKey } from '../message-columns'; +import { tooltipId } from '../../Tooltip/Tooltip'; +import { MessageColumnKey, ReorderableMessageColumnKey } from '../message-columns'; export type MessageProps = { isShowTooltips: boolean; @@ -16,6 +17,12 @@ export type MessageProps = { sessionConfig: ConsumerSessionConfig; valueProjectionThs: ValueProjectionTh[], getColumnWidth: (key: MessageColumnKey) => number, + /** The reorderable columns in render order (index and publishTime stay fixed in front). */ + columnOrder: ReorderableMessageColumnKey[], + /** This row is the first one delivered after the delivery-order switch (the caught-up banner's + * "Switch to Best effort and follow live"): rows above it are the exact replay, rows below follow live + * traffic in the bounded reorder window. The row carries the divider band that says so. */ + isOrderSwitchBoundary?: boolean, onClick: React.MouseEventHandler<HTMLTableCellElement> }; @@ -29,189 +36,118 @@ const MessageComponent: React.FC<MessageProps> = (props) => { false : props.selectedMessages.includes(props.message.numMessageProcessed); + // One cell per reorderable column, keyed exactly like the header - the row follows whatever + // order the header is dragged into. + // `index` and `publishTime` render as the bespoke sticky pair below, never through this map - + // Omit keeps the map honest about it while the record stays total over the rest. + const fieldContent: Record<Exclude<MessageColumnKey, 'index'>, React.ReactNode> = { + publishTime: <PublishTimeField isShowTooltips={props.isShowTooltips} message={msg} />, + key: <KeyField isShowTooltips={props.isShowTooltips} message={props.message} />, + value: <ValueField isShowTooltips={props.isShowTooltips} message={props.message} />, + sessionTargetIndex: <SessionTargetIndexField isShowTooltips={props.isShowTooltips} message={props.message} />, + topic: <TopicField isShowTooltips={props.isShowTooltips} message={props.message} />, + producerName: <ProducerNameField isShowTooltips={props.isShowTooltips} message={props.message} />, + schemaVersion: <SchemaVersionField isShowTooltips={props.isShowTooltips} message={props.message} />, + size: <SizeField isShowTooltips={props.isShowTooltips} message={props.message} />, + properties: <PropertiesField isShowTooltips={props.isShowTooltips} message={props.message} />, + eventTime: <EventTimeField isShowTooltips={props.isShowTooltips} message={props.message} />, + brokerPublishTime: <BrokerPublishTimeField isShowTooltips={props.isShowTooltips} message={props.message} />, + messageId: <MessageIdField isShowTooltips={props.isShowTooltips} message={props.message} />, + sequenceId: <SequenceIdField isShowTooltips={props.isShowTooltips} message={props.message} />, + orderingKey: <OrderingKeyField isShowTooltips={props.isShowTooltips} message={props.message} />, + redeliveryCount: <RedeliveryCountField isShowTooltips={props.isShowTooltips} message={props.message} />, + sessionContextState: <SessionContextStateJsonField isShowTooltips={props.isShowTooltips} message={props.message} />, + }; + + // The divider band rides every cell of the boundary row, so the line spans the table wherever + // it is scrolled to; the labeled badge itself sits on the sticky publish-time cell. + const boundaryClass = props.isOrderSwitchBoundary ? s.OrderSwitchBoundaryTd : ''; + + const fieldTd = (columnKey: Exclude<MessageColumnKey, 'index'>) => ( + <Td + key={columnKey} + testId={columnKey === 'value' ? 'cs-message-value' : undefined} + width={`${props.getColumnWidth(columnKey)}px`} + className={boundaryClass} + onClick={onClick} + coloring={props.coloring} + isSelected={isSelected} + > + {fieldContent[columnKey]} + </Td> + ); + return ( <> <Td key="index" testId="cs-message" - width="36rem" - className={s.IndexField} + width={`${props.getColumnWidth('index')}px`} + className={`${s.IndexField} ${boundaryClass}`} onClick={onClick} coloring={props.coloring} isSelected={isSelected} > {props.message.displayIndex} + {/* The ordering layer delivered this row out of order - loudly flagged, never silently + (the warning mark beside the loaded count carries the running total). Mode-aware + wording, through the app-wide tooltip like every other affordance. */} + {props.message.deliveredOutOfOrder && ( + <span + className={s.OutOfOrderMarker} + data-testid="cs-out-of-order-marker" + data-tooltip-id={tooltipId} + data-tooltip-html={ + (props.sessionConfig.messageDeliveryOrder ?? 'guaranteed') === 'best-effort' + ? 'Out of order: this message arrived after its ~0.75 s reorder window - newer messages were already shown.' + : 'Out of order: stored out of order in the topic itself - typical when several producers ' + + 'share it, because each stamps its own clock, and send retries or batch flushes reorder ' + + 'them further - or a pause fell between replays.' + } + > + ! + </span> + )} </Td> <Td key="publishTime" width={`${props.getColumnWidth('publishTime')}px`} - className={s.PublishTimeField} + className={`${s.PublishTimeField} ${boundaryClass}`} onClick={onClick} coloring={props.coloring} isSelected={isSelected} > + {props.isOrderSwitchBoundary && ( + <span + className={s.OrderSwitchPointBadge} + data-testid="cs-order-switch-point" + title={ + 'The delivery order switched to Best effort at this point. Rows above are the exact replay of ' + + 'recorded history; rows below follow live traffic within the bounded reorder window.' + } + > + Best effort from here + </span> + )} <PublishTimeField isShowTooltips={props.isShowTooltips} message={msg} /> </Td> - <Td - key="key" - width={`${props.getColumnWidth('key')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <KeyField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - {getValueProjectionTds({ - sessionConfig: props.sessionConfig, - valueProjectionThs: props.valueProjectionThs, - coloring: props.coloring, - message: props.message, - isSelected + {props.columnOrder.flatMap((columnKey) => { + const cells = [fieldTd(columnKey)]; + // Value-projection columns are glued after the KEY column wherever it sits - the same + // neighbourhood they have always rendered in, whatever order the rest takes. + if (columnKey === 'key') { + cells.push(...getValueProjectionTds({ + sessionConfig: props.sessionConfig, + valueProjectionThs: props.valueProjectionThs, + coloring: props.coloring, + message: props.message, + isSelected + })); + } + return cells; })} - - <Td - key="value" - testId="cs-message-value" - width={`${props.getColumnWidth('value')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <ValueField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="sessionTargetIndex" - width={`${props.getColumnWidth('sessionTargetIndex')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <SessionTargetIndexField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="topic" - width={`${props.getColumnWidth('topic')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <TopicField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="producerName" - width={`${props.getColumnWidth('producerName')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <ProducerNameField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="schemaVersion" - width={`${props.getColumnWidth('schemaVersion')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <SchemaVersionField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="size" - width={`${props.getColumnWidth('size')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <SizeField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="properties" - width={`${props.getColumnWidth('properties')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <PropertiesField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="eventTime" - width={`${props.getColumnWidth('eventTime')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <EventTimeField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="brokerPublishTime" - width={`${props.getColumnWidth('brokerPublishTime')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <BrokerPublishTimeField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="message" - width={`${props.getColumnWidth('messageId')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <MessageIdField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="sequence" - width={`${props.getColumnWidth('sequenceId')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <SequenceIdField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="ordering" - width={`${props.getColumnWidth('orderingKey')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <OrderingKeyField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="redeliveryCount" - width={`${props.getColumnWidth('redeliveryCount')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <RedeliveryCountField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> - - <Td - key="sessionContextState" - width={`${props.getColumnWidth('sessionContextState')}px`} - onClick={onClick} - coloring={props.coloring} - isSelected={isSelected} - > - <SessionContextStateJsonField isShowTooltips={props.isShowTooltips} message={props.message} /> - </Td> </> ); } diff --git a/ui/components/ui/ConsumerSession/Message/MessageDetails/MessageDetails.module.css b/ui/components/ui/ConsumerSession/Message/MessageDetails/MessageDetails.module.css index 3c283ba1d..5ba812e8f 100644 --- a/ui/components/ui/ConsumerSession/Message/MessageDetails/MessageDetails.module.css +++ b/ui/components/ui/ConsumerSession/Message/MessageDetails/MessageDetails.module.css @@ -3,6 +3,10 @@ overflow: hidden; flex: 1; + /* Prevent horizontal gestures inside the inspector from navigating browser history. The outer + resize wrapper intentionally overflows so its edge handle remains fully hit-testable. */ + overscroll-behavior-x: contain; + margin-top: -1px; } diff --git a/ui/components/ui/ConsumerSession/Message/fields.test.tsx b/ui/components/ui/ConsumerSession/Message/fields.test.tsx new file mode 100644 index 000000000..6bf221491 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Message/fields.test.tsx @@ -0,0 +1,77 @@ +/** + * @jest-environment jsdom + * + * Regression: ValueField shortens a long message value to 100 chars for display, but it also + * passed that shortened string as `rawValue` - and `rawValue` is exactly what Field puts on the + * clipboard on click (and into the `title` tooltip). So copying a long value handed the user the + * ellipsised display text instead of the payload. + * + * CS-25 (e2e) clicks a value cell but its fixture value is short, so it cannot discriminate. + */ +import React from 'react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { ValueField } from './fields'; +import { genEmptyMessageDescriptor } from '../testing'; + +const clipboardWrites: string[] = []; + +beforeAll(() => { + // jsdom implements neither of these; Field's copy helper needs both to take the modern path. + Object.defineProperty(window, 'isSecureContext', { value: true, configurable: true }); + Object.defineProperty(window.navigator, 'clipboard', { + configurable: true, + value: { + writeText: (text: string) => { + clipboardWrites.push(text); + return Promise.resolve(); + }, + }, + }); +}); + +beforeEach(() => { + clipboardWrites.length = 0; +}); + +function renderValue(value: string) { + render(<ValueField isShowTooltips={false} message={genEmptyMessageDescriptor({ value })} />); + return screen.getByTestId('cs-cell-value'); +} + +describe('ValueField copies the whole value, not the truncated display', () => { + const longValue = `"${'x'.repeat(400)}"`; // 402 chars - well past the 100 char display limit + + it('shortens the displayed text', () => { + expect(renderValue(longValue).textContent).toBe(`${longValue.slice(0, 100)}...`); + }); + + it('copies the complete value on click', async () => { + fireEvent.click(renderValue(longValue)); + await waitFor(() => expect(clipboardWrites).toEqual([longValue])); + }); + + it('exposes the complete value as the cell title', () => { + expect(renderValue(longValue).getAttribute('title')).toBe(longValue); + }); + + it('leaves a short value untouched in both the display and the clipboard', async () => { + const shortValue = '"msg-1"'; + const cell = renderValue(shortValue); + expect(cell.textContent).toBe(shortValue); + fireEvent.click(cell); + await waitFor(() => expect(clipboardWrites).toEqual([shortValue])); + }); + + it('exposes the copy action to the keyboard as well as the mouse', async () => { + const value = 'keyboard-copy'; + const cell = renderValue(value); + + expect(cell.getAttribute('role')).toBe('button'); + expect(cell.getAttribute('tabindex')).toBe('0'); + expect(cell.getAttribute('aria-label')).toBe('Copy Value'); + + fireEvent.keyDown(cell, { key: 'Enter' }); + fireEvent.keyDown(cell, { key: ' ' }); + await waitFor(() => expect(clipboardWrites).toEqual([value, value])); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Message/fields.tsx b/ui/components/ui/ConsumerSession/Message/fields.tsx index 423fd92cf..966577fd2 100644 --- a/ui/components/ui/ConsumerSession/Message/fields.tsx +++ b/ui/components/ui/ConsumerSession/Message/fields.tsx @@ -81,8 +81,10 @@ export const KeyField: React.FC<FieldProps> = (props) => { } export const ValueField: React.FC<FieldProps> = (props) => { - const value = props.message.value === null ? undefined : limitString(props.message.value, 100); - return <Field isShowTooltips={props.isShowTooltips} testId="cs-cell-value" title="Value" value={value} rawValue={value} tooltip={help.value} /> + // The cell shows a shortened value, but rawValue is what gets copied to the clipboard - so it has + // to stay the complete payload, otherwise a copy hands the user ellipsised text. + const value = props.message.value === null ? undefined : props.message.value; + return <Field isShowTooltips={props.isShowTooltips} testId="cs-cell-value" title="Value" value={value === undefined ? undefined : limitString(value, 100)} rawValue={value} tooltip={help.value} /> } export const SessionTargetIndexField: React.FC<FieldProps> = (props) => { diff --git a/ui/components/ui/ConsumerSession/ReplayCaughtUpBanner.tsx b/ui/components/ui/ConsumerSession/ReplayCaughtUpBanner.tsx new file mode 100644 index 000000000..1edc2395b --- /dev/null +++ b/ui/components/ui/ConsumerSession/ReplayCaughtUpBanner.tsx @@ -0,0 +1,93 @@ +import React, { FC } from 'react'; +import s from './ConsumerSession.module.css'; +import * as I18n from '../../app/contexts/I18n/I18n'; +import { ReplayCaughtUpStats } from './types'; +import SmallButton from '../SmallButton/SmallButton'; +import ActionButton from '../ActionButton/ActionButton'; +import resumeIcon from './Toolbar/icons/resume.svg'; + +export type ReplayCaughtUpBannerProps = { + caughtUp: ReplayCaughtUpStats; + /** "Load new messages up to now": the ordinary resume flow - the server extends the replay + * boundary to the present and replays the delta, then pauses at the new boundary. */ + onResume: () => void; + /** The one-click switch: the live session continues under Best effort, following live traffic. */ + onContinueWithBestEffort: () => void; + /** The explicit close - the only dismiss gesture the panel has (its body holds controls). */ + onClose: () => void; +}; + +/** + * The guaranteed replay's boundary pause, said out loud: everything recorded up to the boundary + * was delivered in exact order, the server auto-paused, and these are the two ways on. + * + * It deliberately says nothing about how much is waiting past the boundary. That figure could only + * ever be the broker's backlog, which counts ENTRIES rather than messages, so it was approximate in + * a unit no reader thinks in - removed 2026-08-11 rather than kept as a number needing a caveat. + * + * The excluded-topics line exists for regex sessions whose pattern matched NEW topics after Play: + * they are excluded from the running replay (a replay of a set fixed at Play), and only a restart + * includes them. Each line renders ONLY when its record says something - a zero is silence, not a + * claim. The actions are the shared SmallButton, so this banner ages with the rest of the app + * rather than carrying its own button styling. + */ +const ReplayCaughtUpBanner: FC<ReplayCaughtUpBannerProps> = (props) => { + const i18n = I18n.useContext(); + + const shownTopics = props.caughtUp.excludedTopics; + const numUnlistedTopics = Math.max(0, props.caughtUp.excludedTopicCount - shownTopics.length); + const numExcluded = props.caughtUp.excludedTopicCount; + + return ( + <div className={s.ReplayCaughtUp} data-testid="cs-replay-caught-up" role="status"> + <div className={s.ReplayCaughtUpClose}> + <ActionButton + testId="cs-replay-caught-up-close" + onClick={props.onClose} + title="Close" + action={{ type: 'predefined', action: 'close' }} + buttonProps={{ appearance: 'borderless-semitransparent' }} + /> + </div> + <div className={s.ReplayCaughtUpTitle}>Guaranteed order consumer session finished</div> + <div className={s.ReplayCaughtUpHeadline}> + Caught up to {i18n.formatDateTime(new Date(props.caughtUp.boundaryAtMs))} + </div> + <div>Everything recorded up to that moment was replayed in exact order.</div> + + {numExcluded > 0 && ( + <div data-testid="cs-replay-caught-up-excluded"> + {i18n.formatLongNumber(numExcluded)} topic{numExcluded === 1 ? '' : 's'} started matching after this + session began, so {numExcluded === 1 ? 'it is' : 'they are'} not part of this + replay - restart the session to include {numExcluded === 1 ? 'it' : 'them'}:{' '} + {shownTopics.join(', ')}{numUnlistedTopics > 0 ? `, +${i18n.formatLongNumber(numUnlistedTopics)} more` : ''} + </div> + )} + + <div className={s.ReplayCaughtUpActions}> + <SmallButton + type="primary" + svgIcon={resumeIcon} + text="Load new messages up to now" + testId="cs-replay-resume" + title="Replay everything recorded since this point, in exact order, then pause again when caught up." + onClick={props.onResume} + /> + <SmallButton + type="primary" + svgIcon={resumeIcon} + text="Switch to Best effort and follow live" + testId="cs-replay-continue-best-effort" + title={ + 'Stop replaying and follow live traffic from here. Best effort orders within a short ' + + 'window, so a late message can appear out of order - none are dropped. Nothing already ' + + 'loaded is lost, and the switch point is marked in the table.' + } + onClick={props.onContinueWithBestEffort} + /> + </div> + </div> + ); +}; + +export default ReplayCaughtUpBanner; diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/FilterChainEditor/FilterEditor/BasicFilterEditor/BasicMessageFilterOpInput/AnyTestOpInput/TestOpStringMatchesRegexInput/TestOpStringMatchesRegexInput.test.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/FilterChainEditor/FilterEditor/BasicFilterEditor/BasicMessageFilterOpInput/AnyTestOpInput/TestOpStringMatchesRegexInput/TestOpStringMatchesRegexInput.test.tsx new file mode 100644 index 000000000..e72f817a2 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/FilterChainEditor/FilterEditor/BasicFilterEditor/BasicMessageFilterOpInput/AnyTestOpInput/TestOpStringMatchesRegexInput/TestOpStringMatchesRegexInput.test.tsx @@ -0,0 +1,50 @@ +/** + * @jest-environment jsdom + * + * The regex `m` and `i` flags are addons on the shared Input, and an addon is a plain div with an + * `onClick` - so `disabled` on the field does nothing for them. In a read-only (library-owned) + * filter, one click used to rewrite the stored pattern's flags: the same regex, matching a + * different set of messages. + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import TestOpStringMatchesRegexInput from './TestOpStringMatchesRegexInput'; + +const renderOp = (isReadOnly: boolean, flags = '') => { + const onChange = jest.fn(); + render( + <TestOpStringMatchesRegexInput + value={{ type: 'string-matches-regex', pattern: 'a.*b', flags } as never} + onChange={onChange} + isReadOnly={isReadOnly} + /> + ); + return onChange; +}; + +describe('a read-only regex filter', () => { + it.each([['m'], ['i']])('does not let the %s flag be toggled', (flag) => { + const onChange = renderOp(true); + + fireEvent.click(screen.getByText(flag)); + + expect(onChange).not.toHaveBeenCalled(); + }); + + it.each([['m'], ['i']])('still lets the %s flag be toggled when the filter is editable', (flag) => { + const onChange = renderOp(false); + + fireEvent.click(screen.getByText(flag)); + + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange.mock.calls[0][0].flags).toContain(flag); + }); + + it('still shows which flags are set', () => { + // Read-only means "cannot change it", not "cannot see it". + renderOp(true, 'mi'); + + expect(screen.getByText('m')).toBeTruthy(); + expect(screen.getByText('i')).toBeTruthy(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/NumDisplayItemsInput.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/NumDisplayItemsInput.tsx new file mode 100644 index 000000000..367586754 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/NumDisplayItemsInput.tsx @@ -0,0 +1,67 @@ +import React, { useEffect, useState } from 'react'; +import s from './SessionConfiguration.module.css'; +import Input from '../../Input/Input'; +import { numDisplayItemsFromText } from './display-items'; + +export type NumDisplayItemsInputProps = { + /** The committed limit - how many messages a session started right now would keep on screen. */ + value: number; + onChange: (value: number) => void; + isReadOnly?: boolean; +}; + +/** + * How many messages the session keeps on screen. + * + * The typed text is kept in local state rather than derived from the committed limit on every + * render, so an in-progress or refused entry stays on screen (and stays correctable) without ever + * becoming the limit. Clearing the field to retype it is the ordinary case, and it used to commit + * `Number('')` - zero - which turns the retention `slice(-limit)` into "keep everything" and leaves + * the buffer growing until the tab dies. + */ +const NumDisplayItemsInput: React.FC<NumDisplayItemsInputProps> = (props) => { + const [draft, setDraft] = useState<string>(() => String(props.value)); + + // Adopt a limit that changed elsewhere (the toggle, a library item that resolved), but leave a + // draft that already means the same number alone. + useEffect(() => { + if (numDisplayItemsFromText(draft) !== props.value) { + setDraft(String(props.value)); + } + }, [props.value]); + + const onDraftChange = (v: string) => { + setDraft(v); + + const value = numDisplayItemsFromText(v); + if (value !== undefined) { + props.onChange(value); + } + }; + + const isInvalid = numDisplayItemsFromText(draft) === undefined; + + return ( + <div> + <Input + testId="cs-num-display-items" + type="number" + value={draft} + size='small' + onChange={onDraftChange} + isError={isInvalid} + inputProps={{ min: 1, step: 1 }} + isReadOnly={props.isReadOnly} + /> + {isInvalid && ( + <div className={s.FieldError} data-testid="cs-num-display-items-error"> + {/* A refused entry does not undo the last valid one, so the session still keeps THAT + many - saying which turns a silent difference into a visible one. */} + Enter a whole number of messages, 1 or more. The session still keeps {props.value}. + </div> + )} + </div> + ); +}; + +export default NumDisplayItemsInput; diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.module.css b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.module.css index 311b49d5e..05c3db4a6 100644 --- a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.module.css +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.module.css @@ -59,3 +59,9 @@ z-index: 1; display: flex; } + +.FieldError { + color: var(--accent-color-red); + font-size: x-small; + margin-top: 4rem; +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.test.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.test.tsx new file mode 100644 index 000000000..29a18dfee --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.test.tsx @@ -0,0 +1,757 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * BUG-4 regression (e2e RES-2): `/consumer-session?id=<library item id>` accepts ANY persisted + * library item id. When the stored item is not a consumer-session config, the editor used to + * dereference `spec.targets` (and the other chains) during render and throw + * "Cannot read properties of undefined (reading 'map')" - with no error boundary on the route, + * React unmounted the whole SPA and the document rendered EMPTY. + * + * The fixtures are built with the app's own `getDefaultManagedItem`, so the malformed case is the + * same shape the e2e produces by saving a message-filter through the Library. + * + * Note: with a jest.mock() in the file, esbuild-jest runs babel's hoisting pass over untyped JS, so + * imported bindings must not appear in type annotations here (inference only). + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; + +// mermaid/nanoid are ESM-only and jest does not transform node_modules; both are pulled in far away +// from what is under test (markdown preview in the library item editor / session id generation). +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); + +import SessionConfiguration from './SessionConfiguration'; +import { getDefaultManagedItem } from '../../LibraryBrowser/default-library-items'; +import { consumerSessionConfigFromValOrRef } from '../../LibraryBrowser/model/resolved-items-conversions'; +import { defaultNumDisplayItems } from './display-items'; +import { decodeConsumerSessionConfig, describeProblem } from './decode-session-config'; + +const contextForTopic = (topicPersistency: 'persistent' | 'non-persistent') => ({ + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency, + topic: 'a-topic', + }, +}); + +const libraryContext = contextForTopic('persistent'); + +// The same SWR settings the app itself installs (components/app/app.tsx) - without them the data +// hooks deep in the editor keep a retry timer alive past the jsdom teardown. +const renderConfig = (val: unknown, context: unknown = libraryContext) => + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <SessionConfiguration + value={{ type: 'value', val } as never} + onChange={() => undefined} + libraryContext={context as never} + /> + </SWRConfig> + ); + +describe('BUG-4: a malformed persisted consumer session config', () => { + it('shows an error instead of crashing when the item is a foreign type', () => { + // Exactly what the e2e does: a message-filter saved through the Library, opened as `?id=`. + const foreignItem = getDefaultManagedItem('message-filter', libraryContext); + + expect(() => renderConfig(foreignItem)).not.toThrow(); + expect(screen.getByText(/not a valid Consumer Session configuration/i)).toBeTruthy(); + expect(document.body.textContent).not.toBe(''); + }); + + it('shows an error instead of crashing when the stored spec is incomplete', () => { + // Right item type, but the persisted spec is missing `targets` - the field whose `.map` threw. + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const truncated = { ...item, spec: { ...item.spec, targets: undefined } }; + + expect(() => renderConfig(truncated)).not.toThrow(); + expect(screen.getByText(/not a valid Consumer Session configuration/i)).toBeTruthy(); + }); + + // Every field the runtime conversion dereferences has to be checked here, not just the ones whose + // absence happened to throw during RENDER. A spec that renders happily but cannot be converted + // leaves the session with no runtime config at all, and the editor claiming it is fine. + it.each([ + ['targets'], + ['startFrom'], + ['messageFilterChain'], + ['coloringRuleChain'], + ['valueProjectionList'], + ['pauseTriggerChain'], + ])('shows an error when the stored spec is missing %s', (field) => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const spec = { ...(item as any).spec }; + delete spec[field]; + + expect(() => renderConfig({ ...item, spec })).not.toThrow(); + expect(screen.getByText(/not a valid Consumer Session configuration/i)).toBeTruthy(); + }); + + it.each([ + ['an empty object', {}], + ['a value wrapper with no val', { type: 'value' }], + ['a reference wrapper with no ref', { type: 'reference' }], + ])('shows an error when startFrom is %s - malformed must say broken, not spin', (_name, startFrom) => { + // `{ startFrom: {} }` used to pass the shallow object check and then sit in + // useManagedItemValue forever: neither a value to render nor a reference to resolve - + // an endless spinner where an error belongs. + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const spec = { ...(item as any).spec, startFrom }; + + expect(() => renderConfig({ ...item, spec })).not.toThrow(); + expect(screen.getByText(/not a valid Consumer Session configuration/i)).toBeTruthy(); + }); + + it('still renders the editor for a well-formed config', () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + + renderConfig(item); + + expect(screen.queryByText(/not a valid Consumer Session configuration/i)).toBeNull(); + // The default config has exactly one target column. + expect(screen.getAllByTestId('cs-target')).toHaveLength(1); + }); +}); + +/** + * P2.5 - the shape check has to be RECURSIVE. + * + * `/consumer-session?id=` accepts any persisted library item, and a persisted item is JSON on disk: + * written by an older build, hand-edited, truncated, or saved by a build with a different model. A + * check that asks only "is targets an array" and "are the four chains objects" lets a whole family + * of corrupt items through the front door, where they crash a few frames deeper (`topic.val.metadata.id`) + * or sit in `useManagedItemValue` forever - neither a value to render nor a reference to resolve. + * + * So this is a TABLE, walked at every level of the document: the item, its spec, each val-or-ref + * wrapper, each target, and the chains inside a target. Each row asserts the same two things - the + * decoder REFUSES and names the offending path, and the editor degrades through the one existing + * `cs-invalid-config` path instead of throwing. + */ +describe('malformed saved items, level by level', () => { + /** The default item is Date-free, so a JSON round trip is a faithful deep clone here. */ + const clone = <T,>(value: T): T => JSON.parse(JSON.stringify(value)) as T; + + const base = () => clone(getDefaultManagedItem('consumer-session-config', libraryContext)) as any; + + const corrupt = (mutate: (item: any) => void) => { + const item = base(); + mutate(item); + return item; + }; + + /** A second target, so a row can corrupt one target and leave a sound one beside it. */ + const withTwoTargets = (mutate: (item: any) => void) => { + const item = base(); + const second = clone(item.spec.targets[0]); + second.val.metadata.id = 'second-target'; + item.spec.targets.push(second); + mutate(item); + return item; + }; + + const targetSpec = (item: any, index: number) => item.spec.targets[index].val.spec; + + const cases: [string, any, string][] = [ + // The item itself. + [ + 'a foreign library item type', + clone(getDefaultManagedItem('markdown-document', libraryContext)), + 'metadata.type' + ], + ['no spec at all', corrupt((item) => delete item.spec), 'spec'], + ['a spec that is an array, not an object', corrupt((item) => (item.spec = [])), 'spec'], + + // The config spec's own fields. + ['targets missing', corrupt((item) => delete item.spec.targets), 'spec.targets'], + ['targets not an array', corrupt((item) => (item.spec.targets = {})), 'spec.targets'], + ['targets EMPTY - a session with nothing to consume', corrupt((item) => (item.spec.targets = [])), 'spec.targets'], + ['startFrom missing', corrupt((item) => delete item.spec.startFrom), 'spec.startFrom'], + ['numDisplayItems stored as text', corrupt((item) => (item.spec.numDisplayItems = '500')), 'spec.numDisplayItems'], + [ + 'a delivery order this build does not have', + corrupt((item) => (item.spec.messageDeliveryOrder = 'whatever-comes-next')), + 'spec.messageDeliveryOrder' + ], + + // The val-or-ref wrappers. + ['a wrapper with no discriminant', corrupt((item) => (item.spec.startFrom = {})), 'spec.startFrom.type'], + ['a value wrapper with no val', corrupt((item) => (item.spec.startFrom = { type: 'value' })), 'spec.startFrom.val'], + [ + 'a reference wrapper with an empty ref', + corrupt((item) => (item.spec.startFrom = { type: 'reference', ref: '' })), + 'spec.startFrom.ref' + ], + ['a NULL nested wrapper', corrupt((item) => (item.spec.coloringRuleChain = null)), 'spec.coloringRuleChain'], + [ + 'a val-PLUS-reference hybrid', + corrupt((item) => (item.spec.valueProjectionList = { ...item.spec.valueProjectionList, ref: 'some-other-item' })), + 'spec.valueProjectionList' + ], + [ + 'a chain whose item is of the wrong managed type', + corrupt((item) => (item.spec.messageFilterChain.val.metadata.type = 'coloring-rule-chain')), + 'spec.messageFilterChain.val.metadata.type' + ], + [ + 'a chain spec missing its own list', + corrupt((item) => delete item.spec.messageFilterChain.val.spec.filters), + 'spec.messageFilterChain.val.spec.filters' + ], + + // Into the targets. + ['a NULL target', withTwoTargets((item) => (item.spec.targets[1] = null)), 'spec.targets[1]'], + ['an empty target object', withTwoTargets((item) => (item.spec.targets[1] = {})), 'spec.targets[1].type'], + [ + 'a target wrapper whose val has no metadata', + withTwoTargets((item) => delete item.spec.targets[1].val.metadata), + 'spec.targets[1].val.metadata' + ], + [ + 'a target holding some other managed item', + withTwoTargets((item) => (item.spec.targets[1].val.metadata.type = 'message-filter')), + 'spec.targets[1].val.metadata.type' + ], + [ + 'a target flag of the wrong type', + withTwoTargets((item) => (targetSpec(item, 1).isEnabled = 'yes')), + 'spec.targets[1].val.spec.isEnabled' + ], + + // Inside one target: consumption mode, deserializer, topic selector, and the three chains. + [ + 'a target with no consumption mode', + withTwoTargets((item) => delete targetSpec(item, 1).consumptionMode), + 'spec.targets[1].val.spec.consumptionMode' + ], + [ + 'a consumption mode this build cannot run', + withTwoTargets((item) => (targetSpec(item, 1).consumptionMode.mode = { type: 'read-backwards' })), + 'spec.targets[1].val.spec.consumptionMode.mode.type' + ], + [ + 'a deserializer this build cannot run', + withTwoTargets((item) => (targetSpec(item, 1).messageValueDeserializer.val.spec.deserializer.deserializer = { type: 'avro' })), + 'spec.targets[1].val.spec.messageValueDeserializer.val.spec.deserializer.deserializer.type' + ], + [ + 'a topic selector that is a bare object', + withTwoTargets((item) => (targetSpec(item, 1).topicSelector = {})), + 'spec.targets[1].val.spec.topicSelector.type' + ], + [ + 'a topic selector kind this build does not have', + withTwoTargets((item) => (targetSpec(item, 1).topicSelector.val.spec.topicSelector = { type: 'all-topics' })), + 'spec.targets[1].val.spec.topicSelector.val.spec.topicSelector.type' + ], + [ + 'a topic list holding something that is not a topic name', + withTwoTargets( + (item) => + (targetSpec(item, 1).topicSelector.val.spec.topicSelector = { + type: 'multi-topic-selector', + topicFqns: ['persistent://t/n/a', 7] + }) + ), + 'spec.targets[1].val.spec.topicSelector.val.spec.topicSelector.topicFqns[1]' + ], + [ + "a target's coloring rule list holding a null", + withTwoTargets((item) => (targetSpec(item, 1).coloringRuleChain.val.spec.coloringRules = [null])), + 'spec.targets[1].val.spec.coloringRuleChain.val.spec.coloringRules[0]' + ], + [ + "a target's filter chain mode this build does not have", + withTwoTargets((item) => (targetSpec(item, 1).messageFilterChain.val.spec.mode = 'most')), + 'spec.targets[1].val.spec.messageFilterChain.val.spec.mode' + ], + [ + "a target's value projection missing its short name", + withTwoTargets((item) => { + const projection = clone(getDefaultManagedItem('value-projection', libraryContext)) as any; + delete projection.spec.shortName; + targetSpec(item, 1).valueProjectionList.val.spec.projections = [{ type: 'value', val: projection }]; + }), + 'spec.targets[1].val.spec.valueProjectionList.val.spec.projections[0].val.spec.shortName' + ] + ]; + + it.each(cases)('refuses %s and degrades instead of crashing', (_name, item, path) => { + // The decoder refuses it, and names the level it refused at. + const decoded = decodeConsumerSessionConfig(item); + expect(decoded.ok).toBe(false); + expect((decoded as any).problem.path).toBe(path); + + // And the editor degrades through the SAME `cs-invalid-config` path a malformed config + // already used - no crash, no endless spinner, no third mechanism. + expect(() => renderConfig(item)).not.toThrow(); + + const shown = screen.getByTestId('cs-invalid-config'); + expect(shown.textContent).toContain('not a valid Consumer Session configuration'); + // The path is the point: "something is wrong somewhere" is not actionable on a config with + // several targets and a chain inside each of them. + expect(shown.textContent).toContain(path); + }); + + /** + * The other half of a decoder: what it must NOT refuse. A shape check that rejects legitimate + * saved items is a worse bug than the one it fixes, because it locks people out of their own + * library instead of showing them one broken screen. + */ + const accepted: [string, () => any][] = [ + ['the item a new session starts from', () => base()], + [ + 'a saved config with several targets', + () => + withTwoTargets(() => { + /* two sound targets, nothing corrupted */ + }) + ], + [ + 'chains stored as library REFERENCES, not resolved yet', + () => + corrupt((item) => { + item.spec.messageFilterChain = { type: 'reference', ref: 'shared-filter-chain' }; + item.spec.coloringRuleChain = { type: 'reference', ref: 'shared-coloring' }; + item.spec.targets[0] = { type: 'reference', ref: 'shared-target' }; + }) + ], + [ + 'a reference carrying an unsaved in-browser edit', + () => + corrupt((item) => { + item.spec.valueProjectionList = { type: 'reference', ref: 'shared-projections', val: item.spec.valueProjectionList.val }; + }) + ], + [ + 'a start position that names a nested library item', + () => + corrupt((item) => { + item.spec.startFrom.val.spec.startFrom = { + type: 'relativeDateTime', + relativeDateTime: { + type: 'value', + val: { + metadata: { id: 'rel-1', name: '', descriptionMarkdown: '', type: 'relative-date-time' }, + spec: { value: 15, unit: 'minute', isRoundedToUnitStart: false } + } + } + }; + }) + ], + [ + 'an older spec with no delivery order and no display limit', + () => + corrupt((item) => { + delete item.spec.messageDeliveryOrder; + delete item.spec.numDisplayItems; + }) + ], + [ + 'a regex target with a populated topic list beside it', + () => + withTwoTargets((item) => { + targetSpec(item, 0).topicSelector.val.spec.topicSelector = { + type: 'multi-topic-selector', + topicFqns: ['persistent://t/n/a', 'persistent://t/n/b'] + }; + targetSpec(item, 1).topicSelector.val.spec.topicSelector = { + type: 'namespaced-regex-topic-selector', + namespaceFqn: 't/n', + pattern: '.*', + regexSubscriptionMode: 'persistent-only' + }; + }) + ] + ]; + + it.each(accepted)('accepts %s', (_name, build) => { + const decoded = decodeConsumerSessionConfig(build()); + + expect(decoded.ok ? undefined : describeProblem((decoded as any).problem)).toBeUndefined(); + }); + + it('will not let the editor delete its way into the shape it just refused', () => { + // The decoder refuses an empty target list, so the editor must not be able to produce one - + // otherwise removing the last target replaces the editor with its own error page, for good. + renderConfig(base()); + + expect((screen.getByTestId('cs-target-remove') as HTMLButtonElement).disabled).toBe(true); + }); +}); +/** + * "Limit num. display messages" is the only thing standing between a long session and a tab that + * runs out of memory: the session keeps `messages.slice(-limit)` and nothing else bounds it. The + * field committed `Number(v)` of whatever was on screen, and every non-positive answer it produced + * turns that slice into "keep everything" - `slice(-0)` IS `slice(0)`. Clearing the field to retype + * the number is the ordinary way to reach it. + */ +describe('the display-message limit', () => { + const configWithLimit = (numDisplayItems: unknown) => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + return { ...item, spec: { ...(item as any).spec, numDisplayItems } }; + }; + + /** The editor with a parent that applies what it is handed, as the session does. */ + const renderControlled = (numDisplayItems: unknown) => { + const onChange = jest.fn(); + const Controlled = () => { + const [value, setValue] = React.useState<unknown>(() => ({ type: 'value', val: configWithLimit(numDisplayItems) })); + return ( + <SessionConfiguration + value={value as never} + onChange={(v) => { + setValue(v); + onChange(v); + }} + libraryContext={libraryContext as never} + /> + ); + }; + + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <Controlled /> + </SWRConfig> + ); + return onChange; + }; + + const limitInput = () => screen.getByTestId('cs-num-display-items') as HTMLInputElement; + const error = () => screen.queryByTestId('cs-num-display-items-error'); + const lastLimit = (onChange: jest.Mock) => + onChange.mock.calls[onChange.mock.calls.length - 1][0].val.spec.numDisplayItems; + + it('commits a typed limit', () => { + const onChange = renderControlled(1000); + + fireEvent.change(limitInput(), { target: { value: '250' } }); + + expect(error()).toBeNull(); + expect(lastLimit(onChange)).toBe(250); + }); + + it('refuses an emptied field rather than committing a limit that keeps everything', () => { + const onChange = renderControlled(1000); + + fireEvent.change(limitInput(), { target: { value: '' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it.each([['0'], ['-5'], ['2.5'], ['1e3'], ['abc']])('refuses a limit of %p', (text) => { + const onChange = renderControlled(1000); + + fireEvent.change(limitInput(), { target: { value: text } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('keeps the refused text on screen so it can be corrected, and recovers', () => { + const onChange = renderControlled(1000); + + fireEvent.change(limitInput(), { target: { value: '0' } }); + expect(limitInput().value).toBe('0'); + + fireEvent.change(limitInput(), { target: { value: '25' } }); + + expect(error()).toBeNull(); + expect(lastLimit(onChange)).toBe(25); + }); + + it('says which limit is still in effect while the entry is refused', () => { + renderControlled(1000); + + fireEvent.change(limitInput(), { target: { value: '0' } }); + + expect(error()?.textContent).toMatch(/\b1000\b/); + }); + + it('cannot be edited in a read-only (library-owned) configuration', () => { + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <SessionConfiguration + value={{ type: 'value', val: configWithLimit(1000) } as never} + onChange={() => undefined} + libraryContext={libraryContext as never} + isReadOnly + /> + </SWRConfig> + ); + + expect(limitInput().disabled).toBe(true); + }); + + /** + * A persisted spec is JSON on disk - written by an older build, hand-edited, or committed by a + * field that did not validate - so the conversion that turns it into a runtime config is the + * boundary that has to hold, whatever the editor does. + */ + describe('a limit read back from a persisted config', () => { + const limitOf = (numDisplayItems: unknown) => + consumerSessionConfigFromValOrRef({ type: 'value', val: configWithLimit(numDisplayItems) } as never, undefined) + .numDisplayItems; + + it.each([[0], [-5], [2.5], [Number.NaN], [Number.POSITIVE_INFINITY]])( + 'does not let a stored limit of %p disable retention', + (stored) => { + // Every one of these makes `slice(-limit)` keep the whole array, or drop from the wrong end. + const limit = limitOf(stored); + + expect(Number.isSafeInteger(limit)).toBe(true); + expect(limit).toBeGreaterThan(0); + } + ); + + it('falls back to the default when nothing is stored', () => { + expect(limitOf(undefined)).toBe(defaultNumDisplayItems); + }); + + it('keeps a stored limit that is usable', () => { + expect(limitOf(250)).toBe(250); + }); + }); +}); + +describe('start-from is told what the targets retain', () => { + it('disables the history modes when the session sits on a non-persistent topic', () => { + const nonPersistent = contextForTopic('non-persistent'); + const item = getDefaultManagedItem('consumer-session-config', nonPersistent); + + renderConfig(item, nonPersistent); + + const options = Array.from(screen.getByTestId('cs-start-from').querySelectorAll('option')); + expect(options.find((o) => o.value === 'latestMessage')?.disabled).toBe(false); + expect(options.find((o) => o.value === 'earliestMessage')?.disabled).toBe(true); + expect(screen.getByTestId('cs-start-from-non-persistent-note')).toBeTruthy(); + }); + + it('leaves them alone on a persistent topic', () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + + renderConfig(item); + + const options = Array.from(screen.getByTestId('cs-start-from').querySelectorAll('option')); + expect(options.every((o) => !o.disabled)).toBe(true); + expect(screen.queryByTestId('cs-start-from-non-persistent-note')).toBeNull(); + }); +}); + +// Owner decision (2026-08-11, direct instruction): the default is Guaranteed - third move of +// this default (the plan file's decision log is the record), superseding the 2026-08-09 +// Best effort default. +describe('delivery order', () => { + const pbModule = require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'); + const { consumerSessionConfigToPb, messageDeliveryOrderToPb } = require('../conversions/conversions'); + + const renderControlledOrder = (initial = getDefaultManagedItem('consumer-session-config', libraryContext)) => { + const Controlled = () => { + const [value, setValue] = React.useState<unknown>(() => ({ type: 'value', val: initial })); + return ( + <SessionConfiguration + value={value as never} + onChange={setValue as never} + libraryContext={libraryContext as never} + /> + ); + }; + + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <Controlled /> + </SWRConfig> + ); + }; + + const orderSelect = () => screen.getByTestId('cs-delivery-order') as HTMLSelectElement; + // The help moved into the label's question-mark circle (2026-08-11), which renders through the + // app-wide tooltip - so what the reader gets is the data-tooltip-html payload, and that is what + // these read. + const helpHtml = (labelTestId: string) => + screen.getByTestId(labelTestId).querySelector('[data-tooltip-html]')?.getAttribute('data-tooltip-html') ?? ''; + const orderHelp = () => helpHtml('cs-delivery-order-help'); + const timeSelect = () => screen.getByTestId('cs-delivery-order-key') as HTMLSelectElement; + const timeHelp = () => helpHtml('cs-delivery-order-key-help'); + + it('a new spec resolves to the Guaranteed default and serializes it explicitly', () => { + // The product default, by owner decision (2026-08-11; the third move of this default - the + // plan file's decision log is the record): a new session replays recorded history exactly and + // auto-pauses when caught up. Best effort remains the explicit choice for live following. + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const resolved = consumerSessionConfigFromValOrRef({ type: 'value', val: item } as never, undefined); + + expect(resolved.messageDeliveryOrder).toBe('guaranteed'); + expect(consumerSessionConfigToPb(resolved).getMessageDeliveryOrder()) + .toBe(pbModule.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + }); + + it('an older spec without the field resolves and serializes as Guaranteed, like every other absence', () => { + // Pre-branch saved sessions carry no field. They inherit the same product default as a new + // one - a session that never named an order runs the default, not something it chose. + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const legacy = { ...item, spec: { ...item.spec, messageDeliveryOrder: undefined } }; + const resolved = consumerSessionConfigFromValOrRef({ type: 'value', val: legacy } as never, undefined); + + expect(resolved.messageDeliveryOrder).toBe('guaranteed'); + expect(consumerSessionConfigToPb(resolved).getMessageDeliveryOrder()) + .toBe(pbModule.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + }); + + it('an absent value at the wire boundary itself serializes as Guaranteed, the default', () => { + // The one mapping Play and the live switch share. Its catch-all branch is what an + // unresolved absent value falls into, so the branch has to name the default - and explicit + // Best effort has to stay untouched next to it. + expect(messageDeliveryOrderToPb(undefined)) + .toBe(pbModule.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + expect(messageDeliveryOrderToPb('best-effort')) + .toBe(pbModule.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + }); + + it('an explicit Guaranteed choice survives spec -> resolved config -> protobuf, undisturbed by the default', () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const merged = { ...item, spec: { ...item.spec, messageDeliveryOrder: 'guaranteed' } }; + const resolved = consumerSessionConfigFromValOrRef({ type: 'value', val: merged } as never, undefined); + + expect(resolved.messageDeliveryOrder).toBe('guaranteed'); + expect(consumerSessionConfigToPb(resolved).getMessageDeliveryOrder()) + .toBe(pbModule.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + }); + + it('shows all session settings immediately with Guaranteed order selected by default', () => { + renderConfig(getDefaultManagedItem('consumer-session-config', libraryContext)); + + expect(screen.queryByTestId('cs-advanced-toggle')).toBeNull(); + expect(orderSelect().value).toBe('guaranteed'); + expect(screen.getByText('Limit num. display messages')).toBeTruthy(); + expect(screen.getByTestId('cs-session-filters')).toBeTruthy(); + expect(screen.getByTestId('cs-session-projections')).toBeTruthy(); + expect(screen.getByTestId('cs-session-coloring')).toBeTruthy(); + }); + + it('orders and names the modes as Guaranteed, Best effort, Fastest', () => { + renderConfig(getDefaultManagedItem('consumer-session-config', libraryContext)); + + expect(Array.from(orderSelect().options).map((option) => option.textContent)).toEqual([ + 'Guaranteed', + 'Best effort', + 'Fastest', + ]); + }); + + it('documents ALL modes in the help circle, and hides Order by only for Fastest', () => { + // Re-aimed 2026-08-11: the help became a chooser's aid in the label's circle, describing every + // mode at once - a reader deciding between modes should not have to select each one to learn + // what it does. The mode contracts themselves are unchanged. + renderControlledOrder(); + + expect(orderHelp()).toContain('Guaranteed'); + expect(orderHelp()).toContain('exact timestamp order'); + expect(orderHelp()).toContain('Best effort'); + expect(orderHelp()).toContain('~0.75 s'); + expect(orderHelp()).toContain('nothing is dropped'); + expect(orderHelp()).toContain('Fastest'); + expect(orderHelp()).toContain('no ordering across topics'); + // The old hold-forever framing must never come back. + expect(orderHelp()).not.toContain('Waits indefinitely'); + expect(timeSelect()).toBeTruthy(); + + // The help is selection-independent; the Order by row is not - Fastest sorts nothing. + fireEvent.change(orderSelect(), { target: { value: 'guaranteed' } }); + expect(timeSelect()).toBeTruthy(); + fireEvent.change(orderSelect(), { target: { value: 'as-received' } }); + expect(orderHelp()).toContain('Guaranteed'); // still the full catalogue + expect(screen.queryByTestId('cs-delivery-order-key')).toBeNull(); + }); + + it('uses Pulsar timestamp names and explains their fallbacks', () => { + renderControlledOrder(); + + expect(Array.from(timeSelect().options).map((option) => option.textContent)).toEqual([ + 'Publish time', + 'Broker publish time', + 'Event time', + ]); + // One circle documents all three timestamps, whatever is selected. + expect(timeHelp()).toContain('Publish time'); + expect(timeHelp()).toContain('stamped by the producer'); + expect(timeHelp()).toContain('Broker publish time'); + expect(timeHelp()).toContain('broker entry metadata'); + expect(timeHelp()).toContain('cannot start'); + expect(timeHelp()).toContain('Event time'); + expect(timeHelp()).toContain('fall back to'); + }); + + it('shows a spec without the field as the Guaranteed default', () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + renderConfig({ ...item, spec: { ...item.spec, messageDeliveryOrder: undefined } }); + + expect(orderSelect().value).toBe('guaranteed'); + expect(screen.getByTestId('cs-delivery-order-key')).toBeTruthy(); + }); + + it('shows a saved Guaranteed choice as itself, not as the Best effort default', () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext); + const merged = { ...item, spec: { ...item.spec, messageDeliveryOrder: 'guaranteed' } }; + renderConfig(merged); + + const select = screen.getByTestId('cs-delivery-order') as HTMLSelectElement; + expect(select.value).toBe('guaranteed'); + }); +}); + +/** + * Latest x Guaranteed, approached from the ORDER side: the start-from already says Latest and the + * user picks Guaranteed in the delivery-order select. The combination gets the same one-line note + * the start-from side shows - and neither field is rewritten from under the user (the M4 lesson): + * the order becomes what was just chosen, the start-from stays Latest, and Play on the combo + * yields the server's instant caught-up answer. + */ +describe('choosing Guaranteed while the start-from is Latest', () => { + const renderControlledOrder = (initial: unknown) => { + const Controlled = () => { + const [value, setValue] = React.useState<unknown>(() => ({ type: 'value', val: initial })); + return ( + <SessionConfiguration + value={value as never} + onChange={setValue as never} + libraryContext={libraryContext as never} + /> + ); + }; + + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <Controlled /> + </SWRConfig> + ); + }; + + const withLatestStartFrom = () => { + const item = getDefaultManagedItem('consumer-session-config', libraryContext) as { + spec: { startFrom: { val: { spec: { startFrom: unknown } } } }; + }; + item.spec.startFrom.val.spec.startFrom = { type: 'latestMessage' }; + return item; + }; + + it('rewrites neither field, and renders no note (removed 2026-08-11, owner instruction)', () => { + renderControlledOrder(withLatestStartFrom()); + + fireEvent.change(screen.getByTestId('cs-delivery-order'), { target: { value: 'guaranteed' } }); + + // The order is what the user just chose; the start-from is untouched - the gate is the + // disabled option alone, with the why in the delivery-order help circle. + expect((screen.getByTestId('cs-delivery-order') as HTMLSelectElement).value).toBe('guaranteed'); + expect((screen.getByTestId('cs-start-from') as HTMLSelectElement).value).toBe('latestMessage'); + expect(screen.queryByTestId('cs-start-from-latest-guaranteed-note')).toBeNull(); + + fireEvent.change(screen.getByTestId('cs-delivery-order'), { target: { value: 'best-effort' } }); + expect((screen.getByTestId('cs-start-from') as HTMLSelectElement).value).toBe('latestMessage'); + }); +}); diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.tsx index b81007226..6a9771c38 100644 --- a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.tsx +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionConfiguration.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; import FilterChainEditor from './FilterChainEditor/FilterChainEditor'; import s from './SessionConfiguration.module.css' @@ -6,8 +6,10 @@ import LibraryBrowserPanel, { LibraryBrowserPanelProps } from '../../LibraryBrow import { useHover } from '../../../app/hooks/use-hover'; import { ManagedConsumerSessionConfig, ManagedConsumerSessionConfigSpec, ManagedConsumerSessionConfigValOrRef, ManagedConsumerSessionTarget, ManagedConsumerSessionTargetValOrRef } from '../../LibraryBrowser/model/user-managed-items'; import { UseManagedItemValueSpinner, useManagedItemValue } from '../../LibraryBrowser/useManagedItemValue'; +import NothingToShow from '../../NothingToShow/NothingToShow'; import { LibraryContext } from '../../LibraryBrowser/model/library-context'; import StartFromInput from './StartFromInput/StartFromInput'; +import { targetTopicsPersistency } from './StartFromInput/target-topics-persistency'; import SessionTargetInput from './SessionTargetInput/SessionTargetInput'; import AddButton from '../../AddButton/AddButton'; import DeleteButton from '../../DeleteButton/DeleteButton'; @@ -19,10 +21,13 @@ import SmallButton from '../../SmallButton/SmallButton'; import { arrayMove } from './array-move'; import moveLeftIcon from './icons/move-left.svg'; import moveRightIcon from './icons/move-right.svg'; -import Input from '../../Input/Input'; import FormItem from '../../ConfigurationTable/FormItem/FormItem'; - -export const defaultNumDisplayItems = 10_000; +import HelpIcon from '../../ConfigurationTable/HelpIcon/HelpIcon'; +import NumDisplayItemsInput from './NumDisplayItemsInput'; +import { defaultNumDisplayItems } from './display-items'; +import Select from '../../Select/Select'; +import { DeliveryOrderKey, MessageDeliveryOrder } from '../types'; +import { decodeConsumerSessionConfig, describeProblem } from './decode-session-config'; export type SessionConfigurationProps = { value: ManagedConsumerSessionConfigValOrRef, @@ -33,37 +38,9 @@ export type SessionConfigurationProps = { libraryBrowserPanel?: Partial<LibraryBrowserPanelProps> }; -function detectAdvancedConfig(value: ManagedConsumerSessionConfigValOrRef): boolean { - if (value.val?.spec.coloringRuleChain.val?.spec.coloringRules.length) { - return true; - } - - if (value.val?.spec.messageFilterChain.val?.spec.filters.length) { - return true; - } - - if (value.val?.spec.valueProjectionList.val?.spec.projections.length) { - return true; - } - - if (value.val?.spec.numDisplayItems !== undefined) { - return true; - } - - return false; -} - const SessionConfiguration: React.FC<SessionConfigurationProps> = (props) => { const [hoverRef, isHovered] = useHover(); const ref = React.useRef<HTMLDivElement>(null); - const [isShowAdvanced, setIsShowAdvanced] = useState(detectAdvancedConfig(props.value)); - const isAdvancedConfig = detectAdvancedConfig(props.value); - - useEffect(() => { - if (isAdvancedConfig && !isShowAdvanced) { - setIsShowAdvanced(true); - } - }, [isAdvancedConfig, isShowAdvanced]); const resolveResult = useManagedItemValue<ManagedConsumerSessionConfig>(props.value); @@ -82,7 +59,68 @@ const SessionConfiguration: React.FC<SessionConfigurationProps> = (props) => { } const item = resolveResult.value; - const itemSpec = item.spec; + + // `/consumer-session?id=` accepts the id of ANY persisted library item, so what arrives here is + // not guaranteed to be a consumer session config - it can be a foreign item type, a spec written + // by another build, or one that was hand-edited. Reaching into such a spec used to throw during + // render and, with no error boundary above the route, take the whole app down with it. So the + // whole document is decoded before anything is dereferenced, and a failure says WHERE. + const decoded = decodeConsumerSessionConfig(item); + + if (!decoded.ok) { + return ( + <div style={{ margin: '12rem', flex: '1' }}> + <NothingToShow + reason="error" + content={( + <div data-testid="cs-invalid-config"> + The library item with id: {item?.metadata?.id ?? (props.value.type === 'reference' ? props.value.ref : 'unknown')} + {item?.metadata?.type === undefined ? '' : ` (type: ${item.metadata.type})`} +  is not a valid Consumer Session configuration. + <br /> + <code data-testid="cs-invalid-config-problem">{describeProblem(decoded.problem)}</code> + <br /> + Open a Consumer Session configuration item, or start a new session instead. + </div> + )} + /> + </div> + ); + } + + const itemSpec = decoded.item.spec; + + // A spec saved before the field existed names no order, and an absent value means the product + // default - Guaranteed (owner decision 2026-08-11) - here exactly as it does on the wire and + // on the server. + const deliveryOrder = itemSpec.messageDeliveryOrder ?? 'guaranteed'; + const deliveryOrderTime = itemSpec.deliveryOrderKey ?? 'publish-time'; + + // ALL modes described at once, behind the label's help circle (the app-wide pattern): the reader + // opens this to CHOOSE, so showing only the selected mode's contract - as the always-visible + // paragraph this replaced did - answered the one question they were not asking. + const deliveryOrderHelp = ( + <div> + <p><strong>Guaranteed</strong> (default) - replays already-recorded messages in exact timestamp + order, then pauses. Pressing Play again loads what arrived since. Multi-topic sessions need + persistent topics.</p> + <p><strong>Best effort</strong> - follows live traffic, sorting by timestamp within + ~0.75 s. A late message can appear out of order; nothing is dropped.</p> + <p><strong>Fastest</strong> - shows messages as they arrive, with no ordering across topics or + partitions.</p> + </div> + ); + + const deliveryOrderTimeHelp = ( + <div> + <p><strong>Publish time</strong> - stamped by the producer when sending. Always present.</p> + <p><strong>Broker publish time</strong> - stamped by the broker on arrival, so producer clock skew + cannot disorder it. Needs broker entry metadata enabled on the cluster; without it the session + cannot start.</p> + <p><strong>Event time</strong> - set by your application. Messages without one fall back to + publish time.</p> + </div> + ); const onSpecChange = (spec: ManagedConsumerSessionConfigSpec) => { const newValue: ManagedConsumerSessionConfigValOrRef = { ...props.value, val: { ...item, spec } }; @@ -112,10 +150,6 @@ const SessionConfiguration: React.FC<SessionConfigurationProps> = (props) => { type: 'value', val: item as ManagedConsumerSessionConfig }; - - const isAdvancedConfig = detectAdvancedConfig(newValue); - setIsShowAdvanced(isAdvancedConfig); - props.onChange(newValue); }} onSave={(item) => props.onChange({ @@ -141,62 +175,115 @@ const SessionConfiguration: React.FC<SessionConfigurationProps> = (props) => { onChange={(v) => onSpecChange({ ...itemSpec, startFrom: v })} libraryContext={props.libraryContext} isReadOnly={props.isReadOnly} + // The start-from modes depend on what the selected topics retain, and the targets that + // decide that live here. + targetTopicsPersistency={targetTopicsPersistency(itemSpec.targets, props.libraryContext)} /> - {!isAdvancedConfig && <Toggle - testId="cs-advanced-toggle" - value={isShowAdvanced} - onChange={v => setIsShowAdvanced(v)} - label='Show advanced settings' - isReadOnly={props.isReadOnly} - />} - {isShowAdvanced && (<> - <FormItem> - <div style={{ display: 'flex', alignItems: 'center', gap: '12rem' }}> - <Toggle - value={itemSpec.numDisplayItems !== undefined} + <FormItem> + <label htmlFor="cs-delivery-order-select" style={{ display: 'flex', alignItems: 'center', gap: '12rem' }}> + <span style={{ display: 'flex', alignItems: 'center', gap: '6rem' }} data-testid="cs-delivery-order-help"> + Delivery order + <HelpIcon help={deliveryOrderHelp} /> + </span> + <Select<MessageDeliveryOrder> + testId="cs-delivery-order" + id="cs-delivery-order-select" + value={deliveryOrder} + onChange={(v) => onSpecChange({ + ...itemSpec, + messageDeliveryOrder: v + })} + list={[ + { type: 'item', value: 'guaranteed', title: 'Guaranteed' }, + { type: 'item', value: 'best-effort', title: 'Best effort' }, + { type: 'item', value: 'as-received', title: 'Fastest' } + ]} + isReadOnly={props.isReadOnly} + /> + </label> + {deliveryOrder !== 'as-received' && (<> + <label + htmlFor="cs-delivery-order-key-select" + style={{ display: 'flex', alignItems: 'center', gap: '12rem', marginTop: '8rem' }} + > + <span style={{ display: 'flex', alignItems: 'center', gap: '6rem' }} data-testid="cs-delivery-order-key-help"> + Order by + <HelpIcon help={deliveryOrderTimeHelp} /> + </span> + <Select<DeliveryOrderKey> + testId="cs-delivery-order-key" + id="cs-delivery-order-key-select" + value={deliveryOrderTime} onChange={(v) => onSpecChange({ ...itemSpec, - numDisplayItems: v ? defaultNumDisplayItems : undefined + deliveryOrderKey: v === 'publish-time' ? undefined : v })} - label='Limit num. display messages' + list={[ + { type: 'item', value: 'publish-time', title: 'Publish time' }, + { type: 'item', value: 'broker-publish-time', title: 'Broker publish time' }, + { type: 'item', value: 'event-time', title: 'Event time' } + ]} + isReadOnly={props.isReadOnly} + /> + </label> + </>)} + </FormItem> + + <FormItem> + <div style={{ display: 'flex', alignItems: 'center', gap: '12rem' }}> + <Toggle + testId="cs-limit-display-items" + value={itemSpec.numDisplayItems !== undefined} + onChange={(v) => onSpecChange({ + ...itemSpec, + numDisplayItems: v ? defaultNumDisplayItems : undefined + })} + label='Limit num. display messages' + isReadOnly={props.isReadOnly} + /> + <span data-testid="cs-num-display-items-help" style={{ display: 'flex', alignItems: 'center' }}> + <HelpIcon + help={ + 'Limits how many messages are kept in the browser. Choose it based on the average ' + + 'message size in the consumed topics - if the browser hangs or runs out of memory, ' + + 'lower the limit for this consumer session.' + } + /> + </span> + <div style={{ visibility: itemSpec.numDisplayItems === undefined ? 'hidden' : 'visible' }}> + <NumDisplayItemsInput + value={itemSpec.numDisplayItems ?? defaultNumDisplayItems} + onChange={(numDisplayItems) => onSpecChange({ ...itemSpec, numDisplayItems })} isReadOnly={props.isReadOnly} /> - <div style={{ visibility: itemSpec.numDisplayItems === undefined ? 'hidden' : 'visible' }}> - <Input - type="number" - value={String(itemSpec.numDisplayItems)} - size='small' - onChange={v => onSpecChange({ ...itemSpec, numDisplayItems: Number(v) })} - /> - </div> </div> - </FormItem> + </div> + </FormItem> - <FilterChainEditor - testId="cs-session-filters" - value={itemSpec.messageFilterChain} - onChange={(v) => onSpecChange({ ...itemSpec, messageFilterChain: v })} - libraryContext={props.libraryContext} - isReadOnly={props.isReadOnly} - /> + <FilterChainEditor + testId="cs-session-filters" + value={itemSpec.messageFilterChain} + onChange={(v) => onSpecChange({ ...itemSpec, messageFilterChain: v })} + libraryContext={props.libraryContext} + isReadOnly={props.isReadOnly} + /> - <ValueProjectionListInput - testId="cs-session-projections" - value={itemSpec.valueProjectionList} - onChange={(v) => onSpecChange({ ...itemSpec, valueProjectionList: v })} - libraryContext={props.libraryContext} - isReadOnly={props.isReadOnly} - /> + <ValueProjectionListInput + testId="cs-session-projections" + value={itemSpec.valueProjectionList} + onChange={(v) => onSpecChange({ ...itemSpec, valueProjectionList: v })} + libraryContext={props.libraryContext} + isReadOnly={props.isReadOnly} + /> - <ColoringRuleChainInput - testId="cs-session-coloring" - value={itemSpec.coloringRuleChain} - onChange={(v) => onSpecChange({ ...itemSpec, coloringRuleChain: v })} - libraryContext={props.libraryContext} - isReadOnly={props.isReadOnly} - /> - </>)} + <ColoringRuleChainInput + testId="cs-session-coloring" + value={itemSpec.coloringRuleChain} + onChange={(v) => onSpecChange({ ...itemSpec, coloringRuleChain: v })} + libraryContext={props.libraryContext} + isReadOnly={props.isReadOnly} + /> {/* <FormLabel @@ -261,7 +348,13 @@ const SessionConfiguration: React.FC<SessionConfigurationProps> = (props) => { )} <DeleteButton testId="cs-target-remove" - title='Remove this Consumer Session Target' + title={itemSpec.targets.length > 1 + ? 'Remove this Consumer Session Target' + : 'A session needs at least one target'} + // Removing the last one leaves a session with nothing to consume from. The + // server refuses that config and so does the decoder above, which would swap + // this editor for the invalid-config error with no way back. + disabled={itemSpec.targets.length === 1} onClick={() => { const newTargets = [...itemSpec.targets]; newTargets.splice(i, 1); diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionTargetInput/SessionTargetInput.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionTargetInput/SessionTargetInput.tsx index 8411c5121..2d48bffa5 100644 --- a/ui/components/ui/ConsumerSession/SessionConfiguration/SessionTargetInput/SessionTargetInput.tsx +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/SessionTargetInput/SessionTargetInput.tsx @@ -108,6 +108,7 @@ const SessionTargetInput: React.FC<SessionTargetInputProps> = (props) => { <div style={{ marginTop: '-28rem' }}> <FormItem> <IconToggle<boolean> + testId="cs-target-compacted" items={[ { type: 'item', value: true, help: readCompactedHelp, foregroundColor: '#fff', backgroundColor: 'var(--accent-color-blue)', label: 'Compacted' }, { type: 'item', value: false, help: readCompactedHelp, foregroundColor: 'var(--background-color)', backgroundColor: '#aaa', label: 'Compacted' } diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/ApproximateFractionInput.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/ApproximateFractionInput.tsx new file mode 100644 index 000000000..d99c60903 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/ApproximateFractionInput.tsx @@ -0,0 +1,118 @@ +import React, { useEffect, useState } from 'react'; +import s from './StartFromInput.module.css'; +import Input from '../../../Input/Input'; +import { fractionFromPercent, percentFromFraction } from './approximate-fraction'; + +export type ApproximateFractionInputProps = { + /** The stored proportion, in [0, 1]. What it is a proportion OF is the mode's business, not this + * control's - it edits a percentage either way. */ + fraction: number; + onChange: (fraction: number) => void; + /** + * Test-id prefix identifying which mode this instance is editing. Both modes render the same + * control, so one shared id would let a test drive one and assert the other. + */ + testIdPrefix: string; + /** Plain-language name shared by the slider and exact percentage field. */ + accessibleName: string; + /** Labels shown below the two ends of the slider. */ + startLabel: string; + endLabel: string; + disabled?: boolean; + isReadOnly?: boolean; +}; + +/** + * Percentage editor shared by the two approximate start-from modes: a slider for the coarse move + * and a number field for an exact value. + * + * The typed text is kept in local state rather than derived from `fraction` on every render, so an + * in-progress or invalid entry stays on screen (and stays correctable) without ever being committed + * to the session config. + */ +const ApproximateFractionInput: React.FC<ApproximateFractionInputProps> = (props) => { + const [draft, setDraft] = useState<string>(() => percentFromFraction(props.fraction)); + + // Adopt a fraction that changed elsewhere (the slider, or a library item that resolved), but leave + // a draft that already means the same value alone - re-deriving it would eat a trailing '.' and + // make "60.5" untypeable. + useEffect(() => { + if (fractionFromPercent(draft) !== props.fraction) { + setDraft(percentFromFraction(props.fraction)); + } + }, [props.fraction]); + + const onDraftChange = (v: string) => { + setDraft(v); + + const fraction = fractionFromPercent(v); + if (fraction !== undefined) { + props.onChange(fraction); + } + }; + + const isInvalid = fractionFromPercent(draft) === undefined; + const sliderPercent = Number(percentFromFraction(props.fraction)) || 0; + const endpointsId = `${props.testIdPrefix}-endpoints`; + const errorId = `${props.testIdPrefix}-fraction-error`; + const describedBy = isInvalid ? `${endpointsId} ${errorId}` : endpointsId; + + return ( + <div className={s.ApproximateFraction}> + <input + className={s.ApproximateFractionSlider} + data-testid={`${props.testIdPrefix}-fraction-slider`} + type="range" + min={0} + max={100} + step={1} + value={sliderPercent} + aria-label={`${props.accessibleName} slider`} + aria-describedby={describedBy} + aria-invalid={isInvalid} + disabled={props.disabled || props.isReadOnly} + onChange={(e) => props.onChange(fractionFromPercent(e.target.value) ?? props.fraction)} + /> + <div id={endpointsId} className={s.ApproximateFractionEndpoints}> + <span>0% · {props.startLabel}</span> + <span>100% · {props.endLabel}</span> + </div> + <div className={s.ApproximateFractionValue}> + <Input + testId={`${props.testIdPrefix}-fraction`} + value={draft} + type="number" + onChange={onDraftChange} + isError={isInvalid} + inputProps={{ + disabled: props.disabled, + min: 0, + max: 100, + step: 'any', + 'aria-label': `${props.accessibleName} percentage`, + 'aria-describedby': describedBy, + 'aria-invalid': isInvalid + }} + placeholder="0-100" + isReadOnly={props.isReadOnly} + /> + <div className={s.ApproximateFractionUnit}>%</div> + </div> + {isInvalid && ( + <div + id={errorId} + className={s.ApproximateFractionError} + data-testid={errorId} + role="alert" + > + {/* A refused entry does not undo the last valid one, so Play would start from THAT while + the box shows something else. Saying which percentage that is turns a silent + difference into a visible one. */} + Enter a percentage between 0 and 100. The session still uses {percentFromFraction(props.fraction)}%. + </div> + )} + </div> + ); +}; + +export default ApproximateFractionInput; diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/MessageCountInput.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/MessageCountInput.tsx new file mode 100644 index 000000000..139e25c94 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/MessageCountInput.tsx @@ -0,0 +1,72 @@ +import React, { useEffect, useState } from 'react'; +import s from './StartFromInput.module.css'; +import Input from '../../../Input/Input'; +import { messageCountFromText } from './message-count'; + +export type MessageCountInputProps = { + /** The committed count - what a session started right now would use. */ + n: number; + onChange: (n: number) => void; + /** The largest count THIS mode's server side can answer, where it has one. */ + max?: number; + disabled?: boolean; + isReadOnly?: boolean; +}; + +/** + * The message count shared by "Skip first n messages" and "Latest n messages". + * + * The typed text is kept in local state rather than derived from `n` on every render, so an + * in-progress or invalid entry stays on screen (and stays correctable) without ever being committed + * to the session config - clearing the field to retype it is the ordinary case, and it used to + * commit NaN on the way through. + */ +const MessageCountInput: React.FC<MessageCountInputProps> = (props) => { + const [draft, setDraft] = useState<string>(() => String(props.n)); + + // Adopt a count that changed elsewhere (a library item that resolved, a mode switch), but leave a + // draft that already means the same number alone. + useEffect(() => { + if (messageCountFromText(draft, props.max) !== props.n) { + setDraft(String(props.n)); + } + }, [props.n]); + + const onDraftChange = (v: string) => { + setDraft(v); + + const count = messageCountFromText(v, props.max); + if (count !== undefined) { + props.onChange(count); + } + }; + + const isInvalid = messageCountFromText(draft, props.max) === undefined; + + return ( + <div className={s.MessageCount}> + <Input + testId="cs-start-from-n" + value={draft} + type="number" + onChange={onDraftChange} + isError={isInvalid} + inputProps={{ disabled: props.disabled, min: 0, max: props.max, step: 1 }} + placeholder="n" + isReadOnly={props.isReadOnly} + /> + {isInvalid && ( + <div className={s.ApproximateFractionError} data-testid="cs-start-from-n-error"> + {/* A refused entry does not undo the last valid one, so Play would start from THAT. Saying + which number that is turns a silent difference into a visible one - and where the mode + has a ceiling, saying what it is turns "wrong" into something correctable. */} + {props.max === undefined + ? <>Enter a whole number of messages, 0 or more. The session still uses {props.n}.</> + : <>Enter a whole number of messages from 0 to {props.max}. The session still uses {props.n}.</>} + </div> + )} + </div> + ); +}; + +export default MessageCountInput; diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.module.css b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.module.css index e7d2edc5d..09137b2fa 100644 --- a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.module.css +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.module.css @@ -10,3 +10,53 @@ .AdditionalControls { margin-top: 8rem; } + +.PersistencyNote { + padding: 12rem; + border-radius: 8rem; + margin-top: 8rem; + background: var(--surface-color); +} + +.ApproximateFraction { + display: flex; + flex-direction: column; + gap: 8rem; +} + +.MessageCount { + display: flex; + flex-direction: column; + gap: 8rem; +} + +.ApproximateFractionSlider { + width: 100%; + margin: 0; + /* Without this the native range control paints in the BROWSER'S own blue, which sits visibly + off the app's palette next to every other control. `accent-color` tints the thumb and the + filled track while keeping the native widget (focus ring, keyboard behavior). */ + accent-color: var(--accent-color-blue); +} + +.ApproximateFractionEndpoints { + display: flex; + justify-content: space-between; + color: grey; + font-size: x-small; +} + +.ApproximateFractionValue { + display: flex; + align-items: center; + gap: 6rem; +} + +.ApproximateFractionUnit { + color: grey; +} + +.ApproximateFractionError { + color: var(--accent-color-red); + font-size: x-small; +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.test.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.test.tsx new file mode 100644 index 000000000..969682a33 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.test.tsx @@ -0,0 +1,883 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The two approximate start-from modes, their selector branches, and the percent control each one + * edits. + * + * The percent control is the piece Playwright can drive but cannot judge: the model stores a + * FRACTION in [0, 1] while the user edits a PERCENT in [0, 100], so every keystroke crosses a + * conversion, and an out-of-range percent must be refused BEFORE it reaches the model (the server + * rejects a fraction outside [0.0, 1.0] outright). + * + * Both modes render the SAME control with different test ids, so the per-mode suite below runs + * twice. The ids are what keep them apart: a shared one would let a test drive the entry control and + * assert the publish-time one without noticing. + * + * mermaid/nanoid are ESM-only and jest does not transform node_modules; both are pulled in far away + * through the library browser panel that every managed-item editor renders. + */ +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); + +import React from 'react'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import StartFromInput from './StartFromInput'; +import { fractionFromPercent, percentFromFraction } from './approximate-fraction'; +import { latestMessageCountMax } from './message-count'; + +const libraryContext = { + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent' as const, + topic: 'a-topic', + }, +}; + +const startFromItem = (startFrom: unknown) => ({ + type: 'value' as const, + val: { + metadata: { id: 'sf-1', name: '', descriptionMarkdown: '', type: 'consumer-session-start-from' as const }, + spec: { startFrom }, + }, +}); + +// The same SWR settings the app installs - without them the data hooks in the library panel keep a +// retry timer alive past the jsdom teardown. +const renderInput = ( + startFrom: unknown, + targetTopicsPersistency?: { hasPersistent: boolean; hasNonPersistent: boolean } +) => { + const onChange = jest.fn(); + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <StartFromInput + value={startFromItem(startFrom) as never} + onChange={onChange} + libraryContext={libraryContext} + targetTopicsPersistency={targetTopicsPersistency} + /> + </SWRConfig> + ); + return onChange; +}; + +/** The same editor as it renders a REFERENCED library item: shown, never edited in place. */ +const renderReadOnly = (startFrom: unknown) => { + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <StartFromInput + value={startFromItem(startFrom) as never} + onChange={() => undefined} + libraryContext={libraryContext} + isReadOnly + /> + </SWRConfig> + ); +}; + +/** + * Same component, but with a parent that actually applies what it is handed - which is what the app + * does. Anything about how the control reacts to its own committed value needs this: with an inert + * `onChange` the props never move, so half the behaviour never runs. + */ +const renderControlled = (startFrom: unknown) => { + const onChange = jest.fn(); + const Controlled = () => { + const [value, setValue] = React.useState<unknown>(() => startFromItem(startFrom)); + return ( + <StartFromInput + value={value as never} + onChange={(v) => { + setValue(v); + onChange(v); + }} + libraryContext={libraryContext} + /> + ); + }; + + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <Controlled /> + </SWRConfig> + ); + return onChange; +}; + +const nonPersistentOnly = { hasPersistent: false, hasNonPersistent: true }; +const mixed = { hasPersistent: true, hasNonPersistent: true }; + +/** The start-from modes offered by the selector, mapped to whether the option is selectable. */ +const modeOptions = () => + Array.from(screen.getByTestId('cs-start-from').querySelectorAll('option')).reduce<Record<string, boolean>>( + (acc, o) => ({ ...acc, [o.value]: !o.disabled }), + {} + ); + +/** The start-from the component handed back to its parent on the last onChange. */ +const lastStartFrom = (onChange: jest.Mock) => onChange.mock.calls[onChange.mock.calls.length - 1][0].val.spec.startFrom; + +describe('percent <-> fraction', () => { + it('renders a fraction as a clean percent, without binary-float debris', () => { + // In IEEE-754 `0.07 * 100` is 7.000000000000001 and `0.29 * 100` is 28.999999999999996. + // Putting either of those in a form field would be absurd. + expect(percentFromFraction(0.07)).toBe('7'); + expect(percentFromFraction(0.29)).toBe('29'); + expect(percentFromFraction(0)).toBe('0'); + expect(percentFromFraction(1)).toBe('100'); + expect(percentFromFraction(0.605)).toBe('60.5'); + }); + + it('accepts a valid percent and converts it back to a fraction', () => { + expect(fractionFromPercent('0')).toBe(0); + expect(fractionFromPercent('60')).toBe(0.6); + expect(fractionFromPercent('100')).toBe(1); + expect(fractionFromPercent('12.5')).toBe(0.125); + }); + + it('rejects out-of-range percents - the server refuses a fraction outside [0.0, 1.0]', () => { + expect(fractionFromPercent('-1')).toBeUndefined(); + expect(fractionFromPercent('101')).toBeUndefined(); + expect(fractionFromPercent('1000')).toBeUndefined(); + }); + + it('rejects non-numeric input rather than turning it into NaN', () => { + expect(fractionFromPercent('')).toBeUndefined(); + expect(fractionFromPercent(' ')).toBeUndefined(); + expect(fractionFromPercent('abc')).toBeUndefined(); + expect(fractionFromPercent('50%')).toBeUndefined(); + expect(fractionFromPercent('1e2')).toBeUndefined(); + expect(fractionFromPercent('0x10')).toBeUndefined(); + }); +}); + +describe('the start-from mode selector', () => { + it('offers both approximate modes alongside the pre-existing ones', () => { + renderInput({ type: 'latestMessage' }); + + const values = Array.from(screen.getByTestId('cs-start-from').querySelectorAll('option')).map((o) => o.value); + expect(values).toContain('approximateEntryPosition'); + expect(values).toContain('approximatePublishTimePosition'); + // The seven pre-existing modes must survive the addition. + expect(values).toEqual( + expect.arrayContaining([ + 'earliestMessage', + 'latestMessage', + 'messageId', + 'dateTime', + 'relativeDateTime', + 'nthMessageAfterEarliest', + 'nthMessageBeforeLatest', + ]) + ); + }); + + it('labels the two approximate modes by what the percentage is OF', () => { + // The labels are the whole point of the split: "Approximate position" meant either of these and + // could not say which. They are also what the e2e specs select by. + renderInput({ type: 'latestMessage' }); + + const byValue = Object.fromEntries( + Array.from(screen.getByTestId('cs-start-from').querySelectorAll('option')).map((o) => [o.value, o.textContent]) + ); + expect(byValue.approximateEntryPosition).toBe('Approximate position (% of data)'); + expect(byValue.approximatePublishTimePosition).toBe('Approximate position (% of time)'); + }); + + it.each([ + ['approximateEntryPosition'], + ['approximatePublishTimePosition'], + ])('builds a %s spec when the mode is picked', (mode) => { + const onChange = renderInput({ type: 'latestMessage' }); + + fireEvent.change(screen.getByTestId('cs-start-from'), { target: { value: mode } }); + + // A silently-ignored branch would leave the spec untouched; a missing branch would leave + // onChange uncalled entirely. The two share one switch, so each needs its own case. + expect(onChange).toHaveBeenCalled(); + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.5 }); + }); + + it('still builds the skip-n spec, which shares the same switch', () => { + const onChange = renderInput({ type: 'latestMessage' }); + + fireEvent.change(screen.getByTestId('cs-start-from'), { target: { value: 'nthMessageAfterEarliest' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: 'nthMessageAfterEarliest', n: 5 }); + }); +}); + +/** The two modes and the test-id prefix each one's controls carry. */ +const approximateModes = [ + ['approximateEntryPosition', 'cs-start-from-entry'], + ['approximatePublishTimePosition', 'cs-start-from-publish-time'], +] as const; + +const approximateModeUi = { + approximateEntryPosition: { + accessibleName: 'Approximate data position', + endLabel: 'New messages', + }, + approximatePublishTimePosition: { + accessibleName: 'Approximate publish-time position', + endLabel: 'Last publish time', + }, +} as const; + +describe.each(approximateModes)('the percent control for %s', (mode, prefix) => { + const fractionInput = () => screen.getByTestId(`${prefix}-fraction`); + const slider = () => screen.getByTestId(`${prefix}-fraction-slider`); + const error = () => screen.queryByTestId(`${prefix}-fraction-error`); + const ui = approximateModeUi[mode]; + + it('shows the stored fraction as a percent in both the number input and the slider', () => { + renderInput({ type: mode, fraction: 0.6 }); + + expect((fractionInput() as HTMLInputElement).value).toBe('60'); + expect((slider() as HTMLInputElement).value).toBe('60'); + expect(error()).toBeNull(); + }); + + it('gives both controls semantic names and associates the endpoint labels as their description', () => { + // The endpoints line ("0% · Earliest / 100% · ...") is the only helper text these modes carry - + // the explanatory notes were removed 2026-08-12 (owner instruction), so it must not be + // reintroduced into aria-describedby as a dangling id. + renderInput({ type: mode, fraction: 0.6 }); + + const exactPercentage = screen.getByRole('spinbutton', { name: `${ui.accessibleName} percentage` }); + const coarseSlider = screen.getByRole('slider', { name: `${ui.accessibleName} slider` }); + const endpoints = document.getElementById(`${prefix}-endpoints`)!; + + expect(exactPercentage.getAttribute('aria-describedby')).toBe(endpoints.id); + expect(coarseSlider.getAttribute('aria-describedby')).toBe(endpoints.id); + expect(exactPercentage.getAttribute('aria-invalid')).toBe('false'); + expect(coarseSlider.getAttribute('aria-invalid')).toBe('false'); + }); + + it('shows the endpoint meanings without repeating the old explanatory paragraphs', () => { + renderInput({ type: mode, fraction: 0.6 }); + + expect(screen.getByText('0% · Earliest')).toBeTruthy(); + expect(screen.getByText(`100% · ${ui.endLabel}`)).toBeTruthy(); + }); + + it('commits a typed percent as a fraction', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '40' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.4 }); + }); + + it('commits a slider move as a fraction', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(slider(), { target: { value: '25' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.25 }); + }); + + it('refuses a percent above 100 and says so, instead of sending a fraction the server rejects', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '150' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + // The refused text stays visible so the user can correct it. + expect((fractionInput() as HTMLInputElement).value).toBe('150'); + }); + + it('exposes an invalid value and its error to assistive technology', () => { + renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '150' } }); + + const exactPercentage = screen.getByRole('spinbutton', { name: `${ui.accessibleName} percentage` }); + const coarseSlider = screen.getByRole('slider', { name: `${ui.accessibleName} slider` }); + const alert = screen.getByRole('alert'); + expect(exactPercentage.getAttribute('aria-invalid')).toBe('true'); + expect(coarseSlider.getAttribute('aria-invalid')).toBe('true'); + expect(exactPercentage.getAttribute('aria-describedby')).toContain(alert.id); + expect(coarseSlider.getAttribute('aria-describedby')).toContain(alert.id); + }); + + it('refuses a negative percent', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '-5' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses an emptied field rather than committing NaN', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('recovers once the value is corrected', () => { + const onChange = renderInput({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '150' } }); + fireEvent.change(fractionInput(), { target: { value: '15' } }); + + expect(error()).toBeNull(); + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.15 }); + }); + + it('is not rendered for the other modes', () => { + renderInput({ type: 'nthMessageAfterEarliest', n: 5 }); + + expect(screen.queryByTestId(`${prefix}-fraction`)).toBeNull(); + expect(screen.queryByTestId(`${prefix}-note`)).toBeNull(); + expect(screen.getByTestId('cs-start-from-n')).toBeTruthy(); + }); + + it('does not rewrite what the user typed just because the model rounded it', () => { + // The stored fraction keeps 6 decimals, so "12.34567" commits 0.123457 - which renders back as + // "12.3457". Re-deriving the field from the model would edit the text from under the user + // mid-entry; the field is only re-derived when the model means something ELSE. + const onChange = renderControlled({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '12.34567' } }); + + expect((fractionInput() as HTMLInputElement).value).toBe('12.34567'); + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.123457 }); + }); + + it('says which percentage is still in effect while the entry is refused', () => { + // The refused entry does not undo the last valid one, so Play starts from THAT - 60%, while the + // box shows 150 and an error. Either the control blocks Play or it says what will happen; it + // must not do neither. + renderControlled({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '150' } }); + + expect(error()?.textContent).toMatch(/60\s*%/); + }); + + it('adopts a value changed elsewhere, discarding a rejected entry', () => { + const onChange = renderControlled({ type: mode, fraction: 0.6 }); + + fireEvent.change(fractionInput(), { target: { value: '150' } }); + expect(error()).toBeTruthy(); + + fireEvent.change(slider(), { target: { value: '30' } }); + + expect((fractionInput() as HTMLInputElement).value).toBe('30'); + expect(error()).toBeNull(); + expect(lastStartFrom(onChange)).toEqual({ type: mode, fraction: 0.3 }); + }); +}); + +describe('the two percent controls are not the same control', () => { + // They render identically and store the same shape, so the ONLY thing keeping them apart in the + // DOM is the test-id prefix. If both instances shared one id, every per-mode test above would + // still pass while a Playwright spec silently drove the wrong mode. + it('renders only the entry control for the entry mode', () => { + renderInput({ type: 'approximateEntryPosition', fraction: 0.6 }); + + expect(screen.getByTestId('cs-start-from-entry-fraction')).toBeTruthy(); + expect(screen.queryByTestId('cs-start-from-publish-time-fraction')).toBeNull(); + }); + + it('renders only the publish-time control for the publish-time mode', () => { + renderInput({ type: 'approximatePublishTimePosition', fraction: 0.6 }); + + expect(screen.getByTestId('cs-start-from-publish-time-fraction')).toBeTruthy(); + expect(screen.queryByTestId('cs-start-from-entry-fraction')).toBeNull(); + }); + + it('keeps the mode when the percentage is edited, rather than falling back to the other one', () => { + // The onChange handler rebuilds the whole start-from, so it has to re-state its own `type`. + const onChange = renderInput({ type: 'approximatePublishTimePosition', fraction: 0.6 }); + + fireEvent.change(screen.getByTestId('cs-start-from-publish-time-fraction'), { target: { value: '20' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: 'approximatePublishTimePosition', fraction: 0.2 }); + }); +}); + +/** + * "Skip first n messages" and "Latest n messages" both edit a COUNT OF MESSAGES: a whole number, at + * least zero, that the server has to be able to act on. The field used to commit `parseInt` of + * whatever was on screen after every keystroke, and `parseInt` answers something for nearly + * anything - which is how a blank field became NaN, `min=0` became decoration, and `2.7`, `1e3` and + * a number past 2^53 all became a different count than the one that was typed. + */ +const countModes = [ + ['nthMessageAfterEarliest'], + ['nthMessageBeforeLatest'], +] as const; + +describe.each(countModes)('the message-count field for %s', (mode) => { + const countInput = () => screen.getByTestId('cs-start-from-n') as HTMLInputElement; + const error = () => screen.queryByTestId('cs-start-from-n-error'); + + it('commits a valid count', () => { + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '12' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: mode, n: 12 }); + expect(error()).toBeNull(); + }); + + it('accepts zero - "skip nothing" and "the latest 0" are legal counts', () => { + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '0' } }); + + expect(lastStartFrom(onChange)).toEqual({ type: mode, n: 0 }); + }); + + it('refuses an emptied field rather than committing NaN', () => { + // Clearing the field to retype it is the most ordinary thing a user does here, and `parseInt('')` + // is NaN - which then serializes into the request as a count nobody asked for. + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses a negative count - `min=0` is advice to the browser, not a guard', () => { + // Defence in depth: the UI refuses a negative here AND the server now refuses it at the trust + // boundary (server startFromCountValidationTest). `min=0` is only advice to the browser, so this + // client-side check still earns its place even though the server guards the same thing. + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '-1' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses a fractional count instead of silently truncating it', () => { + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '2.7' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses an exponent instead of reading only its mantissa', () => { + // `parseInt('1e3')` is 1: a thousand becomes one, with nothing on screen to say so. + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '1e3' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses a count past the safe integer range instead of rounding it', () => { + // 2^53 + 1 is not representable: the committed value would differ from the typed one. + const onChange = renderInput({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '9007199254740993' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('keeps the refused text on screen so it can be corrected, and recovers', () => { + const onChange = renderControlled({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '-1' } }); + expect(countInput().value).toBe('-1'); + + fireEvent.change(countInput(), { target: { value: '7' } }); + + expect(error()).toBeNull(); + expect(lastStartFrom(onChange)).toEqual({ type: mode, n: 7 }); + }); + + it('says which count is still in effect while the entry is refused', () => { + // The refused entry does not undo the last valid one, so Play would start from THAT. Saying so + // is the difference between a rejected keystroke and a session that silently starts somewhere + // the screen does not show. + renderControlled({ type: mode, n: 5 }); + + fireEvent.change(countInput(), { target: { value: '-1' } }); + + expect(error()?.textContent).toMatch(/\b5\b/); + }); +}); + +/** + * The two count modes look identical but are NOT bounded the same way. "Latest n messages" is + * resolved by a synchronous backward metadata walk - roughly one broker lookup per entry - so the + * server caps it (`latestNMaxAccepted`, ten million); "Skip first n messages" streams past its + * messages and has no such ceiling, deliberately. A UI that accepts both alike shows a count as + * valid and then fails only after Play, with the field it came from long out of sight - which is + * exactly what happened when this file pinned the stale `Int.MaxValue` after the server tightened. + */ +describe('the mode-specific maximum count', () => { + const countInput = () => screen.getByTestId('cs-start-from-n') as HTMLInputElement; + const error = () => screen.queryByTestId('cs-start-from-n-error'); + + it('mirrors the server ceiling exactly - the parity pin', () => { + // The server side of this pin is startFromCountValidationTest, which fixes + // `latestNMaxAccepted` at the same number. If either side moves alone, one of the two suites + // goes red - that asymmetric window (UI accepts, server refuses after Play) is the bug. + expect(latestMessageCountMax).toBe(10_000_000); + }); + + it('accepts exactly the ceiling for "Latest n messages"', () => { + const onChange = renderInput({ type: 'nthMessageBeforeLatest', n: 5 }); + + fireEvent.change(countInput(), { target: { value: String(latestMessageCountMax) } }); + + expect(error()).toBeNull(); + expect(lastStartFrom(onChange)).toEqual({ type: 'nthMessageBeforeLatest', n: latestMessageCountMax }); + }); + + it('refuses one more than the ceiling for "Latest n messages"', () => { + const onChange = renderInput({ type: 'nthMessageBeforeLatest', n: 5 }); + + fireEvent.change(countInput(), { target: { value: String(latestMessageCountMax + 1) } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('says what the ceiling is, not merely that the number is wrong', () => { + renderControlled({ type: 'nthMessageBeforeLatest', n: 5 }); + + fireEvent.change(countInput(), { target: { value: String(latestMessageCountMax + 1) } }); + + expect(error()?.textContent).toContain(String(latestMessageCountMax)); + }); + + it('does not impose that ceiling on "Skip first n messages"', () => { + // Skip-N has no server-side maximum, so borrowing Latest-N's would refuse a count the server + // would happily serve. + const onChange = renderInput({ type: 'nthMessageAfterEarliest', n: 5 }); + + fireEvent.change(countInput(), { target: { value: String(latestMessageCountMax + 1) } }); + + expect(error()).toBeNull(); + expect(lastStartFrom(onChange)).toEqual({ type: 'nthMessageAfterEarliest', n: latestMessageCountMax + 1 }); + }); + + it('tells the browser about the ceiling too, for the spinner and the native step', () => { + renderInput({ type: 'nthMessageBeforeLatest', n: 5 }); + expect(countInput().getAttribute('max')).toBe(String(latestMessageCountMax)); + + cleanup(); + + renderInput({ type: 'nthMessageAfterEarliest', n: 5 }); + expect(countInput().getAttribute('max')).toBeNull(); + }); +}); + +describe('the message-id field', () => { + const idInput = () => screen.getByTestId('cs-start-from-message-id') as HTMLInputElement; + const error = () => screen.queryByTestId('cs-start-from-message-id-error'); + + const messageIdStartFrom = (hexString: string) => ({ + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString }, + }, + }, + }); + + it('accepts a message id as the placeholder renders one', () => { + renderControlled(messageIdStartFrom('')); + + fireEvent.change(idInput(), { target: { value: '08 c3 03 10 cd 04 20 00 30 01' } }); + + expect(error()).toBeNull(); + }); + + it('says so when the text is not hex, instead of failing later inside Play', () => { + // The parser throws on malformed hex, and it is called while the create request is being built - + // after the click, outside any catch. The user sees a session that never starts. + renderControlled(messageIdStartFrom('')); + + fireEvent.change(idInput(), { target: { value: 'zz' } }); + + expect(error()).toBeTruthy(); + expect(idInput().value).toBe('zz'); + }); + + it('says so when a byte is left half-written', () => { + renderControlled(messageIdStartFrom('')); + + fireEvent.change(idInput(), { target: { value: '08 c' } }); + + expect(error()).toBeTruthy(); + }); + + // The shared hex parser deliberately reads "" as an empty byte array, because a byte payload + // really can be empty - a message id cannot. Sending zero bytes as a start position is refused by + // the server (`MessageId.messageId` is parsed as a real id), and the refusal arrives long after + // the click, as a create failure with nothing on screen pointing back at this field. + it.each([[''], [' '], ['\t']])('says an id of %p is missing, rather than sending zero bytes', (text) => { + renderControlled(messageIdStartFrom('08 c3')); + + fireEvent.change(idInput(), { target: { value: text } }); + + expect(error()).toBeTruthy(); + expect(error()?.textContent).toMatch(/message id/i); + }); + + it('flags a message-id mode that starts out empty, before anything is typed', () => { + // Picking the mode creates an empty id, so this is the state the user is dropped into. + renderControlled(messageIdStartFrom('')); + + expect(error()).toBeTruthy(); + }); +}); + +/** + * A referenced (library-owned) start-from is shown, never edited in place - the value belongs to the + * stored item. Both number fields forward `inputProps` alongside that, which is exactly the + * combination that used to hand them back to the user. + */ +describe('a read-only start-from', () => { + it('does not let the message count be edited', () => { + renderReadOnly({ type: 'nthMessageAfterEarliest', n: 5 }); + + expect((screen.getByTestId('cs-start-from-n') as HTMLInputElement).disabled).toBe(true); + }); + + it.each([ + ['approximateEntryPosition', 'cs-start-from-entry'], + ['approximatePublishTimePosition', 'cs-start-from-publish-time'], + ])('does not let the %s percentage be edited', (mode, prefix) => { + renderReadOnly({ type: mode, fraction: 0.6 }); + + expect((screen.getByTestId(`${prefix}-fraction`) as HTMLInputElement).disabled).toBe(true); + expect((screen.getByTestId(`${prefix}-fraction-slider`) as HTMLInputElement).disabled).toBe(true); + }); + + it('does not let the message id be edited', () => { + renderReadOnly({ + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString: '08 c3' }, + }, + }, + }); + + expect((screen.getByTestId('cs-start-from-message-id') as HTMLInputElement).disabled).toBe(true); + }); +}); + +describe('non-persistent targets', () => { + it('offers only Latest when nothing in the selection retains history', () => { + renderInput({ type: 'latestMessage' }, nonPersistentOnly); + + expect(modeOptions()).toEqual({ + earliestMessage: false, + latestMessage: true, + messageId: false, + dateTime: false, + relativeDateTime: false, + nthMessageAfterEarliest: false, + nthMessageBeforeLatest: false, + // Both approximate modes are history-dependent: one needs the topic's entry count, the other + // its first and last publish times, and a non-persistent topic answers neither. + approximateEntryPosition: false, + approximatePublishTimePosition: false, + }); + }); + + it('disables Earliest too, because on a non-persistent topic it would quietly mean "from now"', () => { + renderInput({ type: 'latestMessage' }, nonPersistentOnly); + + expect(modeOptions().earliestMessage).toBe(false); + }); + + it('explains why, in the reader\'s terms', () => { + renderInput({ type: 'latestMessage' }, nonPersistentOnly); + + const note = screen.getByTestId('cs-start-from-non-persistent-note'); + expect(note.textContent).toMatch(/non-persistent/i); + expect(note.textContent).toMatch(/no history|retain nothing|keep no history/i); + // Plain language, not Pulsar internals. + expect(note.textContent).not.toMatch(/examineMessage|405|broker/i); + }); + + it('does not rewrite an unusable mode - it says the session will run from latest instead', () => { + // The user had picked a history mode, then pointed the session at a non-persistent topic. The + // old behaviour overwrote the stored start-from with the live tail, with no notice and no + // undo - flipping the target back re-enabled the modes but the parameters were gone. The + // stored value now belongs to the user: nothing is written, the substitution happens at + // session-conversion time (effectiveStartFrom), and this note names it. + const onChange = renderInput({ type: 'nthMessageAfterEarliest', n: 5 }, nonPersistentOnly); + + expect(onChange).not.toHaveBeenCalled(); + // The selector still shows the configured mode (disabled, with the general note saying why). + expect((screen.getByTestId('cs-start-from') as HTMLSelectElement).value).toBe('nthMessageAfterEarliest'); + + const note = screen.getByTestId('cs-start-from-inapplicable-note'); + expect(note.textContent).toMatch(/latest message/i); + expect(note.textContent).toMatch(/unchanged/i); + }); + + it('never mutates a REFERENCED item into a local val-plus-reference state', () => { + // The old rewrite spread `{...props.value, val: ...}`, which for a reference kept + // `type: 'reference'` while planting a diverging local `val` - the session then disagreed + // with the library item it named, and not even durably (references persist by ref alone). + // The reference shape below is exactly what the app holds after resolving a library item. + const onChange = jest.fn(); + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, revalidateOnFocus: false }}> + <StartFromInput + value={{ + type: 'reference', + ref: 'sf-1', + val: { + metadata: { id: 'sf-1', name: 'saved', descriptionMarkdown: '', type: 'consumer-session-start-from' }, + spec: { startFrom: { type: 'nthMessageAfterEarliest', n: 5 } }, + }, + } as never} + onChange={onChange} + libraryContext={libraryContext} + targetTopicsPersistency={nonPersistentOnly} + /> + </SWRConfig> + ); + + expect(onChange).not.toHaveBeenCalled(); + expect(screen.getByTestId('cs-start-from-inapplicable-note')).toBeTruthy(); + }); + + it('leaves an already-valid selection alone, with no substitution note', () => { + const onChange = renderInput({ type: 'latestMessage' }, nonPersistentOnly); + + expect(onChange).not.toHaveBeenCalled(); + expect(screen.queryByTestId('cs-start-from-inapplicable-note')).toBeNull(); + }); + + it('keeps every mode when the selection is mixed, and says the non-persistent parts start from now', () => { + const onChange = renderInput({ type: 'nthMessageAfterEarliest', n: 5 }, mixed); + + expect(Object.values(modeOptions()).every((enabled) => enabled)).toBe(true); + expect(screen.getByTestId('cs-start-from-mixed-persistency-note').textContent).toMatch(/non-persistent/i); + expect(screen.queryByTestId('cs-start-from-non-persistent-note')).toBeNull(); + // A mixed selection is legal - nothing may be rewritten under the user, and no substitution + // is claimed either: history modes really run on the persistent part. + expect(onChange).not.toHaveBeenCalled(); + expect(screen.queryByTestId('cs-start-from-inapplicable-note')).toBeNull(); + }); + + it('says nothing at all when every selected topic retains history', () => { + renderInput({ type: 'nthMessageAfterEarliest', n: 5 }, { hasPersistent: true, hasNonPersistent: false }); + + expect(Object.values(modeOptions()).every((enabled) => enabled)).toBe(true); + expect(screen.queryByTestId('cs-start-from-non-persistent-note')).toBeNull(); + expect(screen.queryByTestId('cs-start-from-mixed-persistency-note')).toBeNull(); + }); + + it('says nothing when there is no target context at all, as in the library item editor', () => { + renderInput({ type: 'nthMessageAfterEarliest', n: 5 }); + + expect(Object.values(modeOptions()).every((enabled) => enabled)).toBe(true); + expect(screen.queryByTestId('cs-start-from-non-persistent-note')).toBeNull(); + expect(screen.queryByTestId('cs-start-from-mixed-persistency-note')).toBeNull(); + }); +}); + +/** + * Latest x Guaranteed is NOT gated (owner decision 2026-08-11, reversing the 2026-08-09 + * disabled-option gate and its note). The combination is legal and now freely selectable: Play + * answers with an instant caught-up, and the caught-up panel's two actions are where the user + * decides what happens next. What remains pinned here: no option is disabled for it, no note + * renders for it, and the editor never rewrites a stored value (the M4 lesson) - while the + * NON-PERSISTENT gate (a different mechanism for a different reason) stays intact. + */ +describe('Latest under a Guaranteed delivery order', () => { + const note = () => screen.queryByTestId('cs-start-from-latest-guaranteed-note'); + + it('keeps every mode selectable - the delivery order disables nothing', () => { + renderInput({ type: 'earliestMessage' }, { hasPersistent: true, hasNonPersistent: false }); + + const options = modeOptions(); + expect(options.latestMessage).toBe(true); + expect(options.earliestMessage).toBe(true); + expect(options.nthMessageAfterEarliest).toBe(true); + expect(options.dateTime).toBe(true); + }); + + it('renders no note and rewrites nothing for a stored Latest value', () => { + const onChange = renderInput({ type: 'latestMessage' }, { hasPersistent: true, hasNonPersistent: false }); + + expect(onChange).not.toHaveBeenCalled(); + expect((screen.getByTestId('cs-start-from') as HTMLSelectElement).value).toBe('latestMessage'); + expect(note()).toBeNull(); + }); + + it('keeps Latest selectable when nothing retains history - the one mode such topics have', () => { + // Non-persistent topics: Latest is the ONLY selectable mode there, gate or no gate. + renderInput({ type: 'latestMessage' }, nonPersistentOnly); + + expect(modeOptions().latestMessage).toBe(true); + }); +}); + +describe('counted modes carry no ordering note', () => { + // The note that disclosed the counted modes' ordering basis (exactly n, but WHICH n depends on + // producer clocks) was removed 2026-08-12 by owner instruction - the contract lives in + // MessageOrderKey's scaladoc and docs/consume instead. This pin keeps the removal deliberate: a + // reappearing note is a regression of that instruction, not a restoration. + it.each([ + ['nthMessageAfterEarliest', { type: 'nthMessageAfterEarliest', n: 5 }], + ['nthMessageBeforeLatest', { type: 'nthMessageBeforeLatest', n: 5 }], + ])('%s renders without the retired note', (_name, startFrom) => { + renderInput(startFrom); + expect(screen.queryByTestId('cs-start-from-counted-note')).toBeNull(); + }); +}); + +describe('the "single non-partitioned topic" hint sits only where it is true', () => { + // A message id names one entry in one topic's log, so it genuinely wants a single non-partitioned + // topic. The counted modes are exact-count ACROSS partitions, so the same hint there undersells + // them - it used to appear on all three, pre-rework. + const hint = () => screen.queryByText(/works best with a single non-partitioned topic/i); + + const messageIdStartFrom = { + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString: '08 c3' }, + }, + }, + }; + + it('shows it for a message id, where a single-topic position is the accurate advice', () => { + renderInput(messageIdStartFrom); + expect(hint()).toBeTruthy(); + }); + + it.each([ + ['nthMessageAfterEarliest', { type: 'nthMessageAfterEarliest', n: 5 }], + ['nthMessageBeforeLatest', { type: 'nthMessageBeforeLatest', n: 5 }], + ])('does not show it for %s, whose count is exact across partitions', (_name, startFrom) => { + renderInput(startFrom); + expect(hint()).toBeNull(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.tsx b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.tsx index cbf5722ee..3ba809da9 100644 --- a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.tsx +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/StartFromInput.tsx @@ -12,6 +12,12 @@ import { UseManagedItemValueSpinner, useManagedItemValue } from '../../../Librar import LibraryBrowserPanel, { LibraryBrowserPanelProps } from '../../../LibraryBrowser/LibraryBrowserPanel/LibraryBrowserPanel'; import { LibraryContext } from '../../../LibraryBrowser/model/library-context'; import { cloneDeep } from 'lodash'; +import ApproximateFractionInput from './ApproximateFractionInput'; +import MessageCountInput from './MessageCountInput'; +import { defaultApproximateFraction } from './approximate-fraction'; +import { TargetTopicsPersistency, historyDependentStartFromTypes, startFromPersistencyAdvice } from './target-topics-persistency'; +import { messageIdError } from './message-id'; +import { latestMessageCountMax } from './message-count'; export type StartFromInputProps = { value: ManagedConsumerSessionStartFromValOrRef; @@ -20,6 +26,11 @@ export type StartFromInputProps = { disabled?: boolean; isReadOnly?: boolean; libraryBrowserPanel?: Partial<LibraryBrowserPanelProps> + /** + * What the session's selected topics can retain. Omitted where there are no targets to speak of + * (the standalone library item editor), which leaves every mode available. + */ + targetTopicsPersistency?: TargetTopicsPersistency; }; type StartFromType = ConsumerSessionStartFrom['type']; @@ -35,20 +46,65 @@ const list: List<StartFromType> = [ { type: 'item', title: 'Specific time', value: 'dateTime' }, { type: 'item', title: 'Relative time ago', value: 'relativeDateTime' }, { type: 'item', title: 'Skip first n messages', value: 'nthMessageAfterEarliest' }, - { type: 'item', title: 'Latest n messages', value: 'nthMessageBeforeLatest' } + { type: 'item', title: 'Latest n messages', value: 'nthMessageBeforeLatest' }, + // One estimates position from retained entries; the other interpolates over publish time. + { type: 'item', title: 'Approximate position (% of data)', value: 'approximateEntryPosition' }, + { type: 'item', title: 'Approximate position (% of time)', value: 'approximatePublishTimePosition' } ]; +// Which modes need retained history - and why - lives in target-topics-persistency.ts, shared +// with the session-local fallback so the note below and what Play actually sends cannot disagree. +// +// Latest x Guaranteed is deliberately NOT gated here (owner decision 2026-08-11, reversing the +// 2026-08-09 disabled-option gate): the combination is legal - Play answers with an instant +// caught-up and raises the caught-up panel, whose "Load new messages up to now" and "Switch to +// Best effort and follow live" are the decision the user then makes. A selectable option plus +// that notification beats a greyed-out entry needing an explanation. +function startFromList(isHistoryUnavailable: boolean): List<StartFromType> { + if (!isHistoryUnavailable) { + return list; + } + + return list.map((item) => { + if (item.type !== 'item') { + return item; + } + + if (historyDependentStartFromTypes.includes(item.value)) { + return { ...item, disabled: true }; + } + + return item; + }); +} + const StartFromInput: React.FC<StartFromInputProps> = (props) => { const [hoverRef, isHovered] = useHover(); const resolveResult = useManagedItemValue<ManagedConsumerSessionStartFrom>(props.value); + const persistencyAdvice = props.targetTopicsPersistency === undefined + ? 'none' + : startFromPersistencyAdvice(props.targetTopicsPersistency); + const isHistoryUnavailable = persistencyAdvice === 'history-unavailable'; + const resolvedStartFromType = resolveResult.type === 'success' ? resolveResult.value?.spec?.startFrom?.type : undefined; + useEffect(() => { if (props.value.val === undefined && resolveResult.type === 'success') { props.onChange({ ...props.value, val: resolveResult.value }); } }, [resolveResult]); + // A configured mode the selected topics can no longer honour is NOT rewritten. This effect used + // to overwrite it with the live tail, which silently destroyed the configured message id / time + // / count - and, for a referenced library item, wrote a local `val` over a kept + // `type: 'reference'`, so the session disagreed with the item it named. The stored value now + // stays exactly as the user wrote it; the session applies the live tail as a session-local + // substitution at conversion time (effectiveStartFrom), and the note below says so. + const isStartFromInapplicable = isHistoryUnavailable + && resolvedStartFromType !== undefined + && historyDependentStartFromTypes.includes(resolvedStartFromType); + if (resolveResult.type !== 'success') { return <UseManagedItemValueSpinner item={props.value} result={resolveResult} /> } @@ -66,7 +122,20 @@ const StartFromInput: React.FC<StartFromInputProps> = (props) => { props.onChange(newValue); }; - const worksBestWithNonPartitionedTopic = <div style={{ padding: '12rem', borderRadius: '8rem', marginTop: '8rem', background: 'var(--surface-color)' }}>Works best with a single non-partitioned topic.</div>; + // messageId only: a message id names one entry in one topic's log, so it has no meaning across a + // partitioned topic or several topics. The counted modes deliberately do NOT carry this - they are + // exact-count across partitions, and this line would undersell them. (Their ordering contract + // lives in MessageOrderKey's scaladoc and docs/consume; the inline note that restated it was + // removed 2026-08-12, owner instruction.) + const messageIdWorksBestNote = <div style={{ padding: '12rem', borderRadius: '8rem', marginTop: '8rem', background: 'var(--surface-color)' }}>Works best with a single non-partitioned topic.</div>; + + const messageIdHexString = itemSpec.startFrom.type === 'messageId' + ? (itemSpec.startFrom.messageId.val?.spec.hexString || '') + : undefined; + // The same check the request conversion makes, run while the user can still see the field: the + // conversion happens after the Play click, so without this the only symptom is a create that the + // server refuses for a reason that names no field. + const startFromMessageIdError = messageIdHexString === undefined ? undefined : messageIdError(messageIdHexString); return ( <div className={s.StartFromInput} ref={hoverRef}> @@ -96,7 +165,7 @@ const StartFromInput: React.FC<StartFromInputProps> = (props) => { <div className={s.TypeSelect}> <Select<ManagedConsumerSessionStartFromSpec['startFrom']['type']> testId="cs-start-from" - list={list} + list={startFromList(isHistoryUnavailable)} value={itemSpec.startFrom.type} onChange={(v) => { switch (v as StartFromType) { @@ -116,6 +185,14 @@ const StartFromInput: React.FC<StartFromInputProps> = (props) => { onSpecChange({ startFrom: { type: 'nthMessageBeforeLatest', n: 5 } }); return; } + case 'approximateEntryPosition': { + onSpecChange({ startFrom: { type: 'approximateEntryPosition', fraction: defaultApproximateFraction } }); + return; + } + case 'approximatePublishTimePosition': { + onSpecChange({ startFrom: { type: 'approximatePublishTimePosition', fraction: defaultApproximateFraction } }); + return; + } case 'messageId': { const messageId: ManagedMessageIdValOrRef = { type: 'value', @@ -159,32 +236,75 @@ const StartFromInput: React.FC<StartFromInputProps> = (props) => { isReadOnly={props.isReadOnly} /> </div> + {persistencyAdvice === 'history-unavailable' && ( + <div className={s.PersistencyNote} data-testid="cs-start-from-non-persistent-note"> + These topics are non-persistent: they keep no history, so a session can only show messages published from + now on. + </div> + )} + {isStartFromInapplicable && ( + <div className={s.PersistencyNote} data-testid="cs-start-from-inapplicable-note" role="status"> + The configured start position needs history these topics do not keep, so this session will start from the + latest message instead. The configuration itself is left unchanged. + </div> + )} + {persistencyAdvice === 'history-partial' && ( + <div className={s.PersistencyNote} data-testid="cs-start-from-mixed-persistency-note"> + Some of the selected topics are non-persistent and keep no history. Those start from now, whatever is chosen + here. + </div> + )} {itemSpec.startFrom.type === 'nthMessageAfterEarliest' && ( <div className={s.AdditionalControls}> - <Input - testId="cs-start-from-n" - value={itemSpec.startFrom.n.toString()} - type='number' - onChange={(v) => onSpecChange({ startFrom: { type: 'nthMessageAfterEarliest', n: parseInt(v) } })} - inputProps={{ disabled: props.disabled, min: 0 }} - placeholder='n' + <MessageCountInput + n={itemSpec.startFrom.n} + onChange={(n) => onSpecChange({ startFrom: { type: 'nthMessageAfterEarliest', n } })} + disabled={props.disabled} isReadOnly={props.isReadOnly} /> - {worksBestWithNonPartitionedTopic} </div> )} {itemSpec.startFrom.type === 'nthMessageBeforeLatest' && ( <div className={s.AdditionalControls}> - <Input - testId="cs-start-from-n" - value={itemSpec.startFrom.n.toString()} - type='number' - onChange={(v) => onSpecChange({ startFrom: { type: 'nthMessageBeforeLatest', n: parseInt(v) } })} - inputProps={{ disabled: props.disabled, min: 0 }} - placeholder='n' + <MessageCountInput + n={itemSpec.startFrom.n} + onChange={(n) => onSpecChange({ startFrom: { type: 'nthMessageBeforeLatest', n } })} + // Only this mode has a ceiling: the last n are resolved by a backward walk over entry + // metadata, one broker lookup per entry with no progress reporting, so a huge n is a + // request the server would grind on for hours. Skip-N streams past its messages + // instead, reports progress while it does, and deliberately has no maximum. + max={latestMessageCountMax} + disabled={props.disabled} + isReadOnly={props.isReadOnly} + /> + </div> + )} + {itemSpec.startFrom.type === 'approximateEntryPosition' && ( + <div className={s.AdditionalControls}> + <ApproximateFractionInput + testIdPrefix="cs-start-from-entry" + accessibleName="Approximate data position" + startLabel="Earliest" + endLabel="New messages" + fraction={itemSpec.startFrom.fraction} + onChange={(fraction) => onSpecChange({ startFrom: { type: 'approximateEntryPosition', fraction } })} + disabled={props.disabled} + isReadOnly={props.isReadOnly} + /> + </div> + )} + {itemSpec.startFrom.type === 'approximatePublishTimePosition' && ( + <div className={s.AdditionalControls}> + <ApproximateFractionInput + testIdPrefix="cs-start-from-publish-time" + accessibleName="Approximate publish-time position" + startLabel="Earliest" + endLabel="Last publish time" + fraction={itemSpec.startFrom.fraction} + onChange={(fraction) => onSpecChange({ startFrom: { type: 'approximatePublishTimePosition', fraction } })} + disabled={props.disabled} isReadOnly={props.isReadOnly} /> - {worksBestWithNonPartitionedTopic} </div> )} {itemSpec.startFrom.type === 'messageId' && ( @@ -207,9 +327,15 @@ const StartFromInput: React.FC<StartFromInputProps> = (props) => { newItemSpec.startFrom.messageId.val.spec.hexString = v; onSpecChange(newItemSpec); }} + isError={startFromMessageIdError !== undefined} isReadOnly={props.isReadOnly} /> - {worksBestWithNonPartitionedTopic} + {startFromMessageIdError !== undefined && ( + <div className={s.ApproximateFractionError} data-testid="cs-start-from-message-id-error"> + {startFromMessageIdError} + </div> + )} + {messageIdWorksBestNote} </div> )} {itemSpec.startFrom.type === 'dateTime' && ( diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/approximate-fraction.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/approximate-fraction.ts new file mode 100644 index 000000000..9ab6f6349 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/approximate-fraction.ts @@ -0,0 +1,52 @@ +/** + * Both approximate position modes store a FRACTION in [0, 1] + * (the protobuf contract), but a percentage is the natural thing to put in front of a user. These + * two functions are the whole conversion, and the only place the 0-100 range is enforced on the way + * in - the server rejects a fraction outside [0.0, 1.0] outright, so an invalid percent must never + * reach the model. + * + * Shared deliberately: the two modes differ in what the percentage is OF, never in how it is typed, + * so a divergence here would be a defect in one of them rather than a feature. + */ + +/** Default for a freshly-picked approximate mode: the middle of whatever it is a proportion of. */ +export const defaultApproximateFraction = 0.5; + +/** Digits kept when converting back and forth - 4 decimal places of a percent, i.e. 1 part in 10^6. */ +const percentDecimals = 4; + +/** + * A stored fraction rendered as the percent shown in the input. + * + * `Number(...)` after `toFixed` strips both the trailing zeros and the binary-float debris: + * `0.33 * 100` is `33.000000000000004`, which nobody wants to see in a form field. + */ +export function percentFromFraction(fraction: number): string { + if (!Number.isFinite(fraction)) { + return ''; + } + + return String(Number((fraction * 100).toFixed(percentDecimals))); +} + +/** + * A percent as typed, converted to the fraction the model stores - or `undefined` when the text is + * not a percentage in [0, 100]. `undefined` means "refuse this", never "use zero". + * + * The regexp is deliberately stricter than `Number()`, which happily accepts `''` (0), `'0x10'` (16) + * and `'1e2'` (100) - none of which a user meant to type into a percentage field. + */ +export function fractionFromPercent(raw: string): number | undefined { + const trimmed = raw.trim(); + + if (!/^[+-]?(\d+(\.\d*)?|\.\d+)$/.test(trimmed)) { + return undefined; + } + + const percent = Number(trimmed); + if (!Number.isFinite(percent) || percent < 0 || percent > 100) { + return undefined; + } + + return Number((percent / 100).toFixed(percentDecimals + 2)); +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-count.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-count.ts new file mode 100644 index 000000000..50462fa46 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-count.ts @@ -0,0 +1,55 @@ +/** + * The count of messages behind "Skip first n messages" and "Latest n messages". + * + * Both are a whole number of messages, at least zero, that the server has to be able to act on. The + * field used to commit `parseInt` of whatever was on screen after every keystroke, and `parseInt` + * answers something for nearly anything: `''` is NaN, `'2.7'` is 2, `'1e3'` is 1, and anything past + * 2^53 comes back as a different number than was typed. None of those are refusals, so each one + * became a start position the user never asked for. The server now refuses negatives too + * (startFromCountRejectionReason), so this field is the first line of defence, not the only one. + */ + +/** + * The largest "Latest n messages" the server accepts - the mirror of `latestNMaxAccepted` in + * `server/.../handleStartFrom.scala`, enforced there by `startFromCountRejectionReason`. + * + * Not a memory bound: the last n are resolved by a backward walk over entry METADATA (nothing is + * retained), but that walk costs one broker lookup per entry, runs while session creation is + * blocked, and reports no progress - so a larger n is a request the server refuses outright. This + * constant MUST track the server's: when it lagged behind (the server tightened to ten million + * while this stayed at Int.MaxValue), every value in between validated here, was committed, and + * failed only after Play - with the field it came from long out of sight. The parity is pinned by + * test on both sides. + * + * "Skip first n messages" has no such ceiling on purpose - it streams past its messages rather + * than holding them, and it reports progress while it does - so this is a per-MODE limit, not a + * limit on counts. + */ +export const latestMessageCountMax = 10_000_000; + +/** + * The count a text field means, or `undefined` when it does not mean one. `undefined` is "refuse + * this", never "use zero". + * + * Deliberately stricter than `Number()`: no sign, no decimal point, no exponent, nothing outside the + * range where an integer survives the round trip through a double, and nothing above the mode's own + * maximum where it has one. + */ +export function messageCountFromText(raw: string, max?: number): number | undefined { + const trimmed = raw.trim(); + + if (!/^\d+$/.test(trimmed)) { + return undefined; + } + + const count = Number(trimmed); + if (!Number.isSafeInteger(count)) { + return undefined; + } + + if (max !== undefined && count > max) { + return undefined; + } + + return count; +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-id.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-id.ts new file mode 100644 index 000000000..f41e217ba --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/message-id.ts @@ -0,0 +1,27 @@ +/** + * The message id behind "Message with specific ID", and why some text is not one. + * + * The shared hex parser reads blank text as an EMPTY byte array, and that is right for what it is + * for: a byte payload really can be empty. A start POSITION cannot. The server parses this field as + * a real message id and refuses zero bytes, so an empty field bought a full create round trip whose + * only outcome was an error naming no field at all. + * + * Kept out of the component so every serialization path can refuse the same text with one check. + * There is more than one such path and no single "last" place: the Play request conversion runs + * after the click, and the library-save conversion runs on save - an unguarded parse in either + * throws where nothing catches it, so the check belongs at each sink. + */ +import { hexStringToByteArray } from '../../../../conversions/conversions'; + +export function messageIdError(hexString: string): string | undefined { + if (hexString.trim() === '') { + return 'Enter the message id to start from, as hex bytes - for example 08 c3 03 10 cd 04 20 00 30 01.'; + } + + try { + hexStringToByteArray(hexString); + return undefined; + } catch (err) { + return (err as Error).message; + } +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.spec.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.spec.ts new file mode 100644 index 000000000..7878bf6fc --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.spec.ts @@ -0,0 +1,223 @@ +/** + * Which of a session's selected topics can retain history at all. + * + * A non-persistent topic stores NOTHING: a consumer only ever sees messages published after it + * subscribed, and the admin `examineMessage` call the history-based start-from modes are built on + * refuses non-persistent topics with a 405. So the start-from selector has to know, and the only + * signal it needs is client-side: `non-persistent://` versus `persistent://`. + * + * The bias throughout is "never disable on a guess": anything this cannot resolve counts as + * possibly-persistent, so an unresolved reference or an open-ended regex leaves every mode enabled. + */ +import { + effectiveStartFrom, + historyDependentStartFromTypes, + isNonPersistentTopicFqn, + startFromPersistencyAdvice, + targetTopicsPersistency, +} from './target-topics-persistency'; + +const topicContext = (topicPersistency: 'persistent' | 'non-persistent') => ({ + pulsarResource: { + type: 'topic' as const, + tenant: 'public', + namespace: 'default', + topicPersistency, + topic: 'a-topic', + }, +}); + +const namespaceContext = { pulsarResource: { type: 'namespace' as const, tenant: 'public', namespace: 'default' } }; + +/** A target whose topic selector is `selector`; `isEnabled` defaults to true. */ +const target = (selector: unknown, isEnabled = true) => ({ + type: 'value' as const, + val: { + metadata: { id: 'tg', name: '', descriptionMarkdown: '', type: 'consumer-session-target' as const }, + spec: { + isEnabled, + topicSelector: { + type: 'value' as const, + val: { + metadata: { id: 'ts', name: '', descriptionMarkdown: '', type: 'topic-selector' as const }, + spec: { topicSelector: selector }, + }, + }, + }, + }, +}); + +const multi = (...topicFqns: string[]) => ({ type: 'multi-topic-selector', topicFqns }); +const regex = (regexSubscriptionMode: string) => ({ + type: 'namespaced-regex-topic-selector', + namespaceFqn: 'public/default', + pattern: '.*', + regexSubscriptionMode, +}); + +const persistency = (targets: unknown[], context: unknown = topicContext('persistent')) => + targetTopicsPersistency(targets as never, context as never); + +describe('isNonPersistentTopicFqn', () => { + it('splits on the FQN scheme', () => { + expect(isNonPersistentTopicFqn('non-persistent://public/default/t')).toBe(true); + expect(isNonPersistentTopicFqn('persistent://public/default/t')).toBe(false); + }); + + it('does not mistake a persistent topic merely NAMED like one', () => { + // The scheme is a prefix, not a substring - a topic called "non-persistent-audit" is persistent. + expect(isNonPersistentTopicFqn('persistent://public/default/non-persistent-audit')).toBe(false); + }); +}); + +describe('targetTopicsPersistency', () => { + it('reports a lone non-persistent target', () => { + expect(persistency([target(multi('non-persistent://public/default/t'))])).toEqual({ + hasPersistent: false, + hasNonPersistent: true, + }); + }); + + it('reports a lone persistent target', () => { + expect(persistency([target(multi('persistent://public/default/t'))])).toEqual({ + hasPersistent: true, + hasNonPersistent: false, + }); + }); + + it('reports a mix, whether it comes from two targets or one multi-topic target', () => { + const acrossTargets = persistency([ + target(multi('persistent://public/default/a')), + target(multi('non-persistent://public/default/b')), + ]); + expect(acrossTargets).toEqual({ hasPersistent: true, hasNonPersistent: true }); + + const withinOneTarget = persistency([ + target(multi('persistent://public/default/a', 'non-persistent://public/default/b')), + ]); + expect(withinOneTarget).toEqual({ hasPersistent: true, hasNonPersistent: true }); + }); + + it('ignores a disabled target - it is not consumed from', () => { + expect( + persistency([ + target(multi('persistent://public/default/a')), + target(multi('non-persistent://public/default/b'), false), + ]) + ).toEqual({ hasPersistent: true, hasNonPersistent: false }); + }); + + it('reads the current topic from the page the session is mounted on', () => { + const current = [target({ type: 'current-topic' })]; + + expect(persistency(current, topicContext('non-persistent'))).toEqual({ + hasPersistent: false, + hasNonPersistent: true, + }); + expect(persistency(current, topicContext('persistent'))).toEqual({ + hasPersistent: true, + hasNonPersistent: false, + }); + // Mounted on a namespace: "current topic" pins nothing down. + expect(persistency(current, namespaceContext)).toEqual({ hasPersistent: true, hasNonPersistent: false }); + }); + + it('only treats a regex selector as non-persistent when it excludes persistent topics', () => { + expect(persistency([target(regex('non-persistent-only'))])).toEqual({ + hasPersistent: false, + hasNonPersistent: true, + }); + expect(persistency([target(regex('persistent-only'))])).toEqual({ + hasPersistent: true, + hasNonPersistent: false, + }); + }); + + it('treats an "all topics" regex as reaching BOTH domains, because the server makes it', () => { + // NamespacedRegexTopicSelector lists the persistent and the non-persistent topics of the + // namespace and concatenates them before matching the pattern, so a matching non-persistent + // topic IS consumed. Claiming persistent-only here suppressed the one warning that says those + // topics start from now whatever the start-from asks for - and nothing gets disabled by this, + // since a mixed selection keeps every mode available. + expect(persistency([target(regex('all-topics'))])).toEqual({ hasPersistent: true, hasNonPersistent: true }); + expect(startFromPersistencyAdvice(persistency([target(regex('all-topics'))]))).toBe('history-partial'); + }); + + it('never concludes "non-persistent" from something it cannot resolve', () => { + // No targets at all, an unresolved target reference, and an empty topic list: each is unknown, + // and unknown must leave every mode available. + expect(persistency([])).toEqual({ hasPersistent: true, hasNonPersistent: false }); + expect(persistency([{ type: 'reference', ref: 'some-id' }])).toEqual({ + hasPersistent: true, + hasNonPersistent: false, + }); + expect(persistency([target(multi())])).toEqual({ hasPersistent: true, hasNonPersistent: false }); + }); + + it('lets an empty selector alongside a non-persistent one stay non-persistent', () => { + // An empty selector picks no topics, so it must not dilute what the other target proved. + expect(persistency([target(multi()), target(multi('non-persistent://public/default/b'))])).toEqual({ + hasPersistent: false, + hasNonPersistent: true, + }); + }); +}); + +describe('startFromPersistencyAdvice', () => { + it('classifies the three cases the selector reacts to', () => { + expect(startFromPersistencyAdvice({ hasPersistent: false, hasNonPersistent: true })).toBe('history-unavailable'); + expect(startFromPersistencyAdvice({ hasPersistent: true, hasNonPersistent: true })).toBe('history-partial'); + expect(startFromPersistencyAdvice({ hasPersistent: true, hasNonPersistent: false })).toBe('none'); + }); +}); + +/** + * The session-local substitution that replaced the old in-place rewrite: when nothing selected + * retains history and the configured mode needs one, the RUNTIME config runs from the live tail + * while the stored configuration keeps exactly what the user wrote. These tests pin the decision; + * the non-rewriting itself is pinned in StartFromInput.test.tsx, and the substituted request in + * ConsumerSession.test.ts. + */ +describe('effectiveStartFrom', () => { + const nonPersistentOnly = [target(multi('non-persistent://public/default/t'))]; + const persistentOnly = [target(multi('persistent://public/default/t'))]; + const mixed = [target(multi('persistent://public/default/a', 'non-persistent://public/default/b'))]; + const context = topicContext('persistent'); + const substitute = (startFrom: unknown, targets: unknown[], ctx: unknown = context) => + effectiveStartFrom(startFrom as never, targets as never, ctx as never); + + it('substitutes the live tail when a history mode meets a history-less selection', () => { + expect(substitute({ type: 'nthMessageAfterEarliest', n: 5 }, nonPersistentOnly)).toEqual({ type: 'latestMessage' }); + }); + + it('substitutes for EVERY history-dependent mode - the same set the selector disables', () => { + // One shared list drives both the disabled options and this substitution; if a future mode + // joins the selector without joining the list, this loop will not catch it - the selector + // tests will - but a mode in the list must never slip through here. + historyDependentStartFromTypes.forEach((type) => { + expect(substitute({ type }, nonPersistentOnly)).toEqual({ type: 'latestMessage' }); + }); + }); + + it('returns the very same object when the mode is the live tail already', () => { + // Identity, not just equality: the caller uses it to keep the converted config untouched. + const startFrom = { type: 'latestMessage' }; + expect(substitute(startFrom, nonPersistentOnly)).toBe(startFrom); + }); + + it('leaves a history mode alone when the selection retains history', () => { + const startFrom = { type: 'nthMessageAfterEarliest', n: 5 }; + expect(substitute(startFrom, persistentOnly)).toBe(startFrom); + }); + + it('leaves a mixed selection alone - history modes are legal there', () => { + const startFrom = { type: 'earliestMessage' }; + expect(substitute(startFrom, mixed)).toBe(startFrom); + }); + + it('never substitutes on a guess - the unresolvable counts as possibly-persistent', () => { + const startFrom = { type: 'dateTime', dateTime: new Date(0) }; + expect(substitute(startFrom, [])).toBe(startFrom); + expect(substitute(startFrom, [{ type: 'reference', ref: 'some-id' }])).toBe(startFrom); + }); +}); diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.ts new file mode 100644 index 000000000..1408881d3 --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/StartFromInput/target-topics-persistency.ts @@ -0,0 +1,186 @@ +import { LibraryContext } from '../../../LibraryBrowser/model/library-context'; +import { ManagedConsumerSessionTargetValOrRef } from '../../../LibraryBrowser/model/user-managed-items'; +import { ConsumerSessionStartFrom } from '../../types'; + +/** + * What the session's selected topics can offer the start-from selector. + * + * A non-persistent topic retains nothing: a consumer only ever receives messages published after it + * subscribed, and every history-based start-from mode is built on an admin call that refuses + * non-persistent topics outright. So the selector needs to know whether ANY selected topic can + * retain history - not whether all of them can. + */ +export type TargetTopicsPersistency = { + /** At least one selected topic can retain history. Anything unresolvable counts here. */ + hasPersistent: boolean; + /** At least one selected topic is non-persistent and therefore retains nothing. */ + hasNonPersistent: boolean; +}; + +/** Pulsar FQNs carry the persistency in the scheme: `non-persistent://tenant/namespace/topic`. */ +export function isNonPersistentTopicFqn(topicFqn: string): boolean { + return topicFqn.trim().startsWith('non-persistent://'); +} + +/** + * Classify the topics the session's enabled targets point at. + * + * Every branch that cannot resolve a topic reports `hasPersistent` instead of staying silent: this + * result disables controls, and disabling a control on a guess is worse than leaving a useless one + * enabled. An empty selector is the one exception - it selects no topics, so it contributes nothing + * either way and must not dilute what a sibling target proved. + */ +export function targetTopicsPersistency( + targets: ManagedConsumerSessionTargetValOrRef[], + libraryContext: LibraryContext +): TargetTopicsPersistency { + let hasPersistent = false; + let hasNonPersistent = false; + + const observeTopicFqn = (topicFqn: string) => { + if (isNonPersistentTopicFqn(topicFqn)) { + hasNonPersistent = true; + return; + } + + hasPersistent = true; + }; + + targets.forEach((target) => { + const targetSpec = target.val?.spec; + + // An unresolved target reference - what it points at is unknown. + if (targetSpec === undefined) { + hasPersistent = true; + return; + } + + if (targetSpec.isEnabled === false) { + return; + } + + const topicSelector = targetSpec.topicSelector?.val?.spec?.topicSelector; + if (topicSelector === undefined) { + hasPersistent = true; + return; + } + + switch (topicSelector.type) { + case 'current-topic': { + const pulsarResource = libraryContext.pulsarResource; + if (pulsarResource.type !== 'topic') { + // Mounted on a namespace: "the current topic" pins nothing down yet. + hasPersistent = true; + return; + } + + if (pulsarResource.topicPersistency === 'non-persistent') { + hasNonPersistent = true; + return; + } + + hasPersistent = true; + return; + } + case 'multi-topic-selector': { + topicSelector.topicFqns.forEach(observeTopicFqn); + return; + } + case 'namespaced-regex-topic-selector': { + if (topicSelector.regexSubscriptionMode === 'non-persistent-only') { + hasNonPersistent = true; + return; + } + + // "all-topics" really does mean both domains: the server lists the namespace's persistent + // AND non-persistent topics and concatenates them before matching the pattern + // (NamespacedRegexTopicSelector), so a matching non-persistent topic is consumed from. + // Reporting it as possibly-mixed costs nothing - a mixed selection disables no mode - and + // it restores the note saying those topics start from now whatever is chosen here. + if (topicSelector.regexSubscriptionMode === 'all-topics') { + hasPersistent = true; + hasNonPersistent = true; + return; + } + + hasPersistent = true; + return; + } + } + }); + + // Nothing resolvable at all (no targets, or only empty selectors). + if (!hasPersistent && !hasNonPersistent) { + return { hasPersistent: true, hasNonPersistent: false }; + } + + return { hasPersistent, hasNonPersistent }; +} + +export type StartFromPersistencyAdvice = + /** Nothing selected retains history: only a live tail is possible. */ + | 'history-unavailable' + /** Some selected topics retain history and some do not. */ + | 'history-partial' + | 'none'; + +export function startFromPersistencyAdvice(persistency: TargetTopicsPersistency): StartFromPersistencyAdvice { + if (!persistency.hasPersistent && persistency.hasNonPersistent) { + return 'history-unavailable'; + } + + if (persistency.hasPersistent && persistency.hasNonPersistent) { + return 'history-partial'; + } + + return 'none'; +} + +/** + * Every start-from mode except the live tail needs the topic to have kept something. On a + * non-persistent topic they are all unusable - including "Earliest message", the misleading one: + * nothing is retained, so it quietly behaves as "from now" instead of failing. Both approximate + * modes are proportions OF a history, so neither can be computed where there is none: one needs + * the topic's entry count, the other its first and last publish times. + * + * Shared by the selector (which disables these modes and says why) and the session-local fallback + * below (which decides what a Play actually sends): one list, so the two cannot disagree. + */ +export const historyDependentStartFromTypes: ConsumerSessionStartFrom['type'][] = [ + 'earliestMessage', + 'messageId', + 'dateTime', + 'relativeDateTime', + 'nthMessageAfterEarliest', + 'nthMessageBeforeLatest', + 'approximateEntryPosition', + 'approximatePublishTimePosition' +]; + +/** + * The start-from a session should ACTUALLY RUN with, given what its selected topics retain. + * + * When nothing selected retains history and the configured mode needs one, the session runs from + * the live tail instead - the only position a non-persistent topic has. This is a SESSION-LOCAL + * substitution, applied where the stored configuration is converted into a runtime one. The + * stored configuration is deliberately left as the user wrote it: rewriting it in place (the old + * behaviour) silently destroyed the configured message id / time / count with no undo, and for a + * REFERENCED library item it wrote a local `val` while keeping `type: 'reference'` - a session + * that disagreed with the item it named, and not even durably, since references persist by ref + * alone. The selector shows a note naming this exact substitution while it is in effect. + */ +export function effectiveStartFrom( + startFrom: ConsumerSessionStartFrom, + targets: ManagedConsumerSessionTargetValOrRef[], + libraryContext: LibraryContext +): ConsumerSessionStartFrom { + if (!historyDependentStartFromTypes.includes(startFrom.type)) { + return startFrom; + } + + if (startFromPersistencyAdvice(targetTopicsPersistency(targets, libraryContext)) !== 'history-unavailable') { + return startFrom; + } + + return { type: 'latestMessage' }; +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/decode-session-config.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/decode-session-config.ts new file mode 100644 index 000000000..d9192aa6c --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/decode-session-config.ts @@ -0,0 +1,390 @@ +import { ManagedConsumerSessionConfig, ManagedItemType } from '../../LibraryBrowser/model/user-managed-items'; + +/** + * A RUNTIME shape decoder for a persisted consumer session config. + * + * `/consumer-session?id=` accepts the id of ANY persisted library item, and a persisted item is + * JSON on disk: written by an older build, hand-edited, truncated, or saved by a build whose model + * differed. TypeScript says nothing about it at runtime, so the editor needs an actual decoder + * before it starts dereferencing. + * + * The check this replaced went one level deep - `targets` had to be an array and four wrappers had + * to be objects - which let `{ targets: [{}] }` through to crash on `topic.val.metadata.id`, let a + * `{}` chain sit in `useManagedItemValue` forever as neither a value nor a reference, and let an + * unknown topic-selector kind reach an effect that throws outside React's reach. So this walks the + * WHOLE document: every val-or-ref wrapper, every managed item's metadata and spec, every target, + * and the chains, selector, deserializer, and projections inside each target. + * + * It reports the FIRST problem with a path in the document's own field names + * (`spec.targets[1].val.spec.consumptionMode.mode.type`), the way the server reports + * `targets[1]: consumption_mode: ...`. On a config with several targets and a chain inside each, + * "something is wrong somewhere" is not an actionable answer. + * + * What it deliberately does NOT do: resolve references. A `{ type: 'reference', ref }` wrapper is + * checked for a usable id and nothing more - the item behind it is fetched later, by the component + * that needs it, and is that component's problem. + */ + +/** Where the corruption is, in the saved document's own field names, and what is wrong with it. */ +export type ConfigProblem = { + path: string; + problem: string; +}; + +export type ConfigDecodeResult = + | { ok: true; item: ManagedConsumerSessionConfig } + | { ok: false; problem: ConfigProblem }; + +/** The one-line rendering, e.g. `spec.targets[1].val.spec.isEnabled: expected true or false, found a string`. */ +export function describeProblem(problem: ConfigProblem): string { + return problem.path === '' ? problem.problem : `${problem.path}: ${problem.problem}`; +} + +/** `undefined` means "no problem found here" - so checks compose with `??`, which short-circuits. */ +type Problem = ConfigProblem | undefined; + +type SpecCheck = (spec: Record<string, unknown>, path: string) => Problem; + +const chainModes = ['all', 'any'] as const; +const dateTimeUnits = ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'] as const; +const deliveryOrders = ['as-received', 'best-effort', 'guaranteed'] as const; +const deliveryOrderKeys = ['publish-time', 'broker-publish-time', 'event-time'] as const; +const regexSubscriptionModes = ['all-topics', 'persistent-only', 'non-persistent-only'] as const; +const consumptionModes = ['regular-consumption-mode', 'read-compacted-consumption-mode'] as const; +const deserializerKinds = ['use-latest-topic-schema', 'treat-bytes-as-json'] as const; +const filterKinds = ['BasicMessageFilter', 'JsMessageFilter'] as const; + +const typeName = (v: unknown): string => { + if (v === null) { + return 'null'; + } + if (Array.isArray(v)) { + return 'a list'; + } + if (v instanceof Date) { + return 'a date'; + } + return `a ${typeof v}`; +}; + +/** Short and readable in a message: a quoted string, or what kind of thing it is. */ +const shown = (v: unknown): string => (typeof v === 'string' ? JSON.stringify(v) : typeName(v)); + +const at = (path: string, problem: string): ConfigProblem => ({ path, problem }); + +/** The root of the document has no name, so its fields must not be prefixed with a stray dot. */ +const child = (path: string, field: string): string => (path === '' ? field : `${path}.${field}`); + +const isRecord = (v: unknown): v is Record<string, unknown> => + typeof v === 'object' && v !== null && !Array.isArray(v) && !(v instanceof Date); + +const recordAt = (v: unknown, path: string): Problem => + isRecord(v) ? undefined : at(path, `expected an object, found ${typeName(v)}`); + +const stringAt = (v: unknown, path: string): Problem => + typeof v === 'string' ? undefined : at(path, `expected text, found ${typeName(v)}`); + +const nonEmptyStringAt = (v: unknown, path: string): Problem => + typeof v === 'string' && v !== '' ? undefined : at(path, `expected non-empty text, found ${shown(v)}`); + +const booleanAt = (v: unknown, path: string): Problem => + typeof v === 'boolean' ? undefined : at(path, `expected true or false, found ${typeName(v)}`); + +const numberAt = (v: unknown, path: string): Problem => + typeof v === 'number' && Number.isFinite(v) ? undefined : at(path, `expected a number, found ${shown(v)}`); + +const oneOfAt = (v: unknown, path: string, allowed: readonly string[]): Problem => + typeof v === 'string' && allowed.includes(v) + ? undefined + : at(path, `expected one of ${allowed.join(', ')}, found ${shown(v)}`); + +const listAt = (v: unknown, path: string, checkItem: (item: unknown, itemPath: string) => Problem): Problem => { + if (!Array.isArray(v)) { + return at(path, `expected a list, found ${typeName(v)}`); + } + for (let i = 0; i < v.length; i += 1) { + const problem = checkItem(v[i], `${path}[${i}]`); + if (problem !== undefined) { + return problem; + } + } + return undefined; +}; + +/** An absent optional field is fine; a present one still has to be the right shape. */ +const optionalAt = (v: unknown, check: () => Problem): Problem => (v === undefined ? undefined : check()); + +/** + * A managed library item: metadata that identifies it, and a spec of the kind that metadata claims. + * + * `metadata.type` is checked against what the position expects, so a message filter saved into a + * target slot is named as the wrong item rather than half-rendered. + */ +const managedItem = (v: unknown, path: string, itemType: ManagedItemType, checkSpec: SpecCheck): Problem => { + const notAnItem = recordAt(v, path); + if (notAnItem !== undefined) { + return notAnItem; + } + const item = v as Record<string, unknown>; + + const metadataPath = child(path, 'metadata'); + const notMetadata = recordAt(item.metadata, metadataPath); + if (notMetadata !== undefined) { + return notMetadata; + } + const metadata = item.metadata as Record<string, unknown>; + + const badId = nonEmptyStringAt(metadata.id, `${metadataPath}.id`); + if (badId !== undefined) { + return badId; + } + if (metadata.type !== itemType) { + return at(`${metadataPath}.type`, `expected a ${itemType}, found ${shown(metadata.type)}`); + } + + const specPath = child(path, 'spec'); + return recordAt(item.spec, specPath) ?? checkSpec(item.spec as Record<string, unknown>, specPath); +}; + +/** + * A val-or-ref wrapper: an inline value, or a reference to a library item by id. + * + * It must be DISCRIMINATED, not merely an object. A wrapper with no `type` resolves to neither a + * value to render nor a reference to fetch, and the component holding it shows a spinner that + * never ends. A `value` wrapper that also carries a `ref` says two contradictory things about + * where the item lives, and nothing in the app produces one. + * + * A `reference` MAY carry a `val`: that is an edit made in the browser and not yet saved. If it is + * there it has to be sound, because that is the copy the editor renders. + */ +const valOrRef = (v: unknown, path: string, itemType: ManagedItemType, checkSpec: SpecCheck): Problem => { + const notAWrapper = recordAt(v, path); + if (notAWrapper !== undefined) { + return notAWrapper; + } + const wrapper = v as Record<string, unknown>; + + if (wrapper.type === 'value') { + if (wrapper.ref !== undefined) { + return at(path, 'a stored value also carries a library reference; it cannot be both'); + } + return managedItem(wrapper.val, child(path, 'val'), itemType, checkSpec); + } + + if (wrapper.type === 'reference') { + return ( + nonEmptyStringAt(wrapper.ref, child(path, 'ref')) ?? + optionalAt(wrapper.val, () => managedItem(wrapper.val, child(path, 'val'), itemType, checkSpec)) + ); + } + + return at(child(path, 'type'), `expected "value" or "reference", found ${shown(wrapper.type)}`); +}; + +const messageIdSpec: SpecCheck = (spec, path) => stringAt(spec.hexString, `${path}.hexString`); + +const dateTimeSpec: SpecCheck = (spec, path) => + spec.dateTime instanceof Date && Number.isFinite(spec.dateTime.getTime()) + ? undefined + : at(`${path}.dateTime`, `expected a date, found ${typeName(spec.dateTime)}`); + +const relativeDateTimeSpec: SpecCheck = (spec, path) => + numberAt(spec.value, `${path}.value`) ?? + oneOfAt(spec.unit, `${path}.unit`, dateTimeUnits) ?? + booleanAt(spec.isRoundedToUnitStart, `${path}.isRoundedToUnitStart`); + +const startFromSpec: SpecCheck = (spec, path) => { + const startFromPath = `${path}.startFrom`; + const notAnObject = recordAt(spec.startFrom, startFromPath); + if (notAnObject !== undefined) { + return notAnObject; + } + const startFrom = spec.startFrom as Record<string, unknown>; + + switch (startFrom.type) { + case 'earliestMessage': + case 'latestMessage': + return undefined; + case 'nthMessageAfterEarliest': + case 'nthMessageBeforeLatest': + return numberAt(startFrom.n, `${startFromPath}.n`); + case 'approximateEntryPosition': + case 'approximatePublishTimePosition': + return numberAt(startFrom.fraction, `${startFromPath}.fraction`); + case 'messageId': + return valOrRef(startFrom.messageId, `${startFromPath}.messageId`, 'message-id', messageIdSpec); + case 'dateTime': + return valOrRef(startFrom.dateTime, `${startFromPath}.dateTime`, 'date-time', dateTimeSpec); + case 'relativeDateTime': + return valOrRef( + startFrom.relativeDateTime, + `${startFromPath}.relativeDateTime`, + 'relative-date-time', + relativeDateTimeSpec + ); + default: + return at(`${startFromPath}.type`, `expected a start position this build supports, found ${shown(startFrom.type)}`); + } +}; + +/** The filter operator tree is its own editor's contract; here the kind of filter is the shape. */ +const basicMessageFilterTargetSpec: SpecCheck = (spec, path) => + recordAt(spec.target, `${path}.target`) ?? + nonEmptyStringAt((spec.target as Record<string, unknown>).type, `${path}.target.type`); + +const messageFilterSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + booleanAt(spec.isNegated, `${path}.isNegated`) ?? + valOrRef(spec.targetField, `${path}.targetField`, 'basic-message-filter-target', basicMessageFilterTargetSpec) ?? + recordAt(spec.filter, `${path}.filter`) ?? + oneOfAt((spec.filter as Record<string, unknown>).type, `${path}.filter.type`, filterKinds); + +const messageFilterChainSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + booleanAt(spec.isNegated, `${path}.isNegated`) ?? + oneOfAt(spec.mode, `${path}.mode`, chainModes) ?? + listAt(spec.filters, `${path}.filters`, (filter, filterPath) => + valOrRef(filter, filterPath, 'message-filter', messageFilterSpec) + ); + +const coloringRuleSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + stringAt(spec.foregroundColor, `${path}.foregroundColor`) ?? + stringAt(spec.backgroundColor, `${path}.backgroundColor`) ?? + valOrRef(spec.messageFilterChain, `${path}.messageFilterChain`, 'message-filter-chain', messageFilterChainSpec); + +const coloringRuleChainSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + listAt(spec.coloringRules, `${path}.coloringRules`, (rule, rulePath) => + valOrRef(rule, rulePath, 'coloring-rule', coloringRuleSpec) + ); + +const valueProjectionSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + stringAt(spec.shortName, `${path}.shortName`) ?? + optionalAt(spec.width, () => numberAt(spec.width, `${path}.width`)) ?? + valOrRef(spec.target, `${path}.target`, 'basic-message-filter-target', basicMessageFilterTargetSpec); + +const valueProjectionListSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + listAt(spec.projections, `${path}.projections`, (projection, projectionPath) => + valOrRef(projection, projectionPath, 'value-projection', valueProjectionSpec) + ); + +const consumerSessionEventSpec: SpecCheck = (spec, path) => + recordAt(spec.event, `${path}.event`) ?? + nonEmptyStringAt((spec.event as Record<string, unknown>).type, `${path}.event.type`); + +const pauseTriggerChainSpec: SpecCheck = (spec, path) => + oneOfAt(spec.mode, `${path}.mode`, chainModes) ?? + listAt(spec.events, `${path}.events`, (event, eventPath) => + valOrRef(event, eventPath, 'consumer-session-event', consumerSessionEventSpec) + ); + +const topicSelectorSpec: SpecCheck = (spec, path) => { + const selectorPath = `${path}.topicSelector`; + const notAnObject = recordAt(spec.topicSelector, selectorPath); + if (notAnObject !== undefined) { + return notAnObject; + } + const selector = spec.topicSelector as Record<string, unknown>; + + switch (selector.type) { + case 'current-topic': + return undefined; + case 'multi-topic-selector': + return listAt(selector.topicFqns, `${selectorPath}.topicFqns`, nonEmptyStringAt); + case 'namespaced-regex-topic-selector': + return ( + nonEmptyStringAt(selector.namespaceFqn, `${selectorPath}.namespaceFqn`) ?? + stringAt(selector.pattern, `${selectorPath}.pattern`) ?? + oneOfAt(selector.regexSubscriptionMode, `${selectorPath}.regexSubscriptionMode`, regexSubscriptionModes) + ); + default: + return at(`${selectorPath}.type`, `expected a topic selector this build supports, found ${shown(selector.type)}`); + } +}; + +const deserializerSpec: SpecCheck = (spec, path) => { + const deserializerPath = `${path}.deserializer`; + const notAnObject = recordAt(spec.deserializer, deserializerPath); + if (notAnObject !== undefined) { + return notAnObject; + } + const deserializer = spec.deserializer as Record<string, unknown>; + + return ( + oneOfAt(deserializer.type, `${deserializerPath}.type`, ['deserializer']) ?? + recordAt(deserializer.deserializer, `${deserializerPath}.deserializer`) ?? + oneOfAt( + (deserializer.deserializer as Record<string, unknown>).type, + `${deserializerPath}.deserializer.type`, + deserializerKinds + ) + ); +}; + +/** Not a library item of its own - the consumption mode is stored inline in the target spec. */ +const consumptionModeAt = (v: unknown, path: string): Problem => { + const notAnObject = recordAt(v, path); + if (notAnObject !== undefined) { + return notAnObject; + } + const mode = v as Record<string, unknown>; + + return ( + oneOfAt(mode.type, `${path}.type`, ['consumer-session-target-consumption-mode']) ?? + recordAt(mode.mode, `${path}.mode`) ?? + oneOfAt((mode.mode as Record<string, unknown>).type, `${path}.mode.type`, consumptionModes) + ); +}; + +const targetSpec: SpecCheck = (spec, path) => + booleanAt(spec.isEnabled, `${path}.isEnabled`) ?? + consumptionModeAt(spec.consumptionMode, `${path}.consumptionMode`) ?? + valOrRef(spec.messageValueDeserializer, `${path}.messageValueDeserializer`, 'deserializer', deserializerSpec) ?? + valOrRef(spec.topicSelector, `${path}.topicSelector`, 'topic-selector', topicSelectorSpec) ?? + valOrRef(spec.messageFilterChain, `${path}.messageFilterChain`, 'message-filter-chain', messageFilterChainSpec) ?? + valOrRef(spec.coloringRuleChain, `${path}.coloringRuleChain`, 'coloring-rule-chain', coloringRuleChainSpec) ?? + valOrRef(spec.valueProjectionList, `${path}.valueProjectionList`, 'value-projection-list', valueProjectionListSpec); + +const configSpec: SpecCheck = (spec, path) => { + const targetsPath = `${path}.targets`; + + return ( + listAt(spec.targets, targetsPath, (target, targetPath) => + valOrRef(target, targetPath, 'consumer-session-target', targetSpec) + ) ?? + // A session with no target consumes from nothing. The server refuses it outright, so accepting + // it here only buys a Play button that starts a session which can never deliver a message. + ((spec.targets as unknown[]).length === 0 + ? at(targetsPath, 'expected at least one target; a session with no target has nothing to consume') + : undefined) ?? + valOrRef(spec.startFrom, `${path}.startFrom`, 'consumer-session-start-from', startFromSpec) ?? + valOrRef(spec.messageFilterChain, `${path}.messageFilterChain`, 'message-filter-chain', messageFilterChainSpec) ?? + valOrRef(spec.coloringRuleChain, `${path}.coloringRuleChain`, 'coloring-rule-chain', coloringRuleChainSpec) ?? + valOrRef(spec.valueProjectionList, `${path}.valueProjectionList`, 'value-projection-list', valueProjectionListSpec) ?? + valOrRef( + spec.pauseTriggerChain, + `${path}.pauseTriggerChain`, + 'consumer-session-pause-trigger-chain', + pauseTriggerChainSpec + ) ?? + // The VALUE range is sanitized downstream by displayItemLimit - only a non-number is a shape + // error here, because that is what no reader of this field can recover from. + optionalAt(spec.numDisplayItems, () => numberAt(spec.numDisplayItems, `${path}.numDisplayItems`)) ?? + optionalAt(spec.messageDeliveryOrder, () => + oneOfAt(spec.messageDeliveryOrder, `${path}.messageDeliveryOrder`, deliveryOrders) + ) ?? + optionalAt(spec.deliveryOrderKey, () => oneOfAt(spec.deliveryOrderKey, `${path}.deliveryOrderKey`, deliveryOrderKeys)) + ); +}; + +/** Decode a saved library item as a consumer session configuration, or say exactly why it is not one. */ +export function decodeConsumerSessionConfig(item: unknown): ConfigDecodeResult { + const problem = managedItem(item, '', 'consumer-session-config', configSpec); + + return problem === undefined + ? { ok: true, item: item as ManagedConsumerSessionConfig } + : { ok: false, problem }; +} diff --git a/ui/components/ui/ConsumerSession/SessionConfiguration/display-items.ts b/ui/components/ui/ConsumerSession/SessionConfiguration/display-items.ts new file mode 100644 index 000000000..baab7b61b --- /dev/null +++ b/ui/components/ui/ConsumerSession/SessionConfiguration/display-items.ts @@ -0,0 +1,56 @@ +/** + * How many messages a session keeps on screen. + * + * The limit is what stops a long-running session from growing until the tab dies, and it is applied + * as `messages.slice(-limit)`. That expression turns EVERY non-positive limit into "keep + * everything": `slice(-0)` is `slice(0)`, i.e. the whole array, and so is `slice(-NaN)`. So a limit + * of zero - which is what an emptied number field commits, and what a saved session can carry - + * silently removes the very limit it configures, and the browser buffer grows without bound. + * + * A fractional or negative limit is no better: `slice(-2.5)` drops a different number of messages + * than either 2 or 3 would, and `slice(5)` (from a limit of -5) drops the OLDEST messages from the + * front while keeping everything after them. + * + * Hence one definition of the domain, used by the field that edits it, by the conversion that reads + * a persisted config, and by the retention itself. + */ + +// 1,000,000 by owner instruction (2026-08-11; was 10,000). This is BOTH the prefill when the +// limit toggle is switched on AND - via `displayItemLimit` - the bound applied when no limit is +// configured, so it is the real ceiling of every session's browser buffer. The help beside the +// toggle tells the reader to lower it per session when message sizes make it too much. +export const defaultNumDisplayItems = 1_000_000; + +/** + * The limit a text field means, or `undefined` when it does not mean one. `undefined` is "refuse + * this" - there is no sensible "0 messages on screen", so zero is refused rather than reinterpreted. + */ +export function numDisplayItemsFromText(raw: string): number | undefined { + const trimmed = raw.trim(); + + if (!/^\d+$/.test(trimmed)) { + return undefined; + } + + const count = Number(trimmed); + if (!Number.isSafeInteger(count) || count < 1) { + return undefined; + } + + return count; +} + +/** + * The limit to actually apply, given whatever a persisted config carries. + * + * A stored spec is a TRUST BOUNDARY: it is JSON on disk, written by an older build, hand-edited, or + * committed by a field that did not validate. Anything that is not a usable limit falls back to the + * default rather than disabling retention. + */ +export function displayItemLimit(stored: number | undefined): number { + if (stored === undefined || !Number.isSafeInteger(stored) || stored < 1) { + return defaultNumDisplayItems; + } + + return stored; +} diff --git a/ui/components/ui/ConsumerSession/StartFromDegradedBanner.tsx b/ui/components/ui/ConsumerSession/StartFromDegradedBanner.tsx new file mode 100644 index 000000000..981b72bd1 --- /dev/null +++ b/ui/components/ui/ConsumerSession/StartFromDegradedBanner.tsx @@ -0,0 +1,103 @@ +import React, { FC, useState } from 'react'; +import s from './ConsumerSession.module.css'; +import * as Notifications from '../../app/contexts/Notifications'; +import { copyToClipboard, copyFailureMessage } from '../../app/clipboard'; + +/** + * How many topic names the disclosed banner puts in the DOM. + * + * A topic selector admits up to 2,000 streams, and one `<li>` per abandoned stream built a + * ~2,000-node subtree by default - on a session that is already degraded and under load. The count + * is still stated in full, and the remaining names are one click (or one copy) away, so the bound + * costs no information. + */ +export const startFromDegradedPreviewLimit = 5; + +export type StartFromDegradedBannerProps = { + /** The streams the start-from resolution gave up waiting for, as `<consumer>@<topic fqn>`. */ + abandonedStreams: string[], +}; + +/** + * The sticky disclosure for a start position that could not be resolved against every topic. It + * stays expanded by default on purpose - the result really is degraded - and is collapsible to a + * compact badge. Its own disclosure state is deliberately local: the record is cleared by + * unmounting the banner, which is also what re-arms a new session's disclosure. + */ +const StartFromDegradedBanner: FC<StartFromDegradedBannerProps> = (props) => { + const { notifySuccess, notifyWarn } = Notifications.useContext(); + const [isCollapsed, setIsCollapsed] = useState(false); + const [isShowAllTopics, setIsShowAllTopics] = useState(false); + + // The stream ids are internal (`<consumer>@<topic fqn>`); the person reading the banner + // owns TOPICS, so that is what is shown. + const topics = props.abandonedStreams.map(id => (id.includes('@') ? id.slice(id.indexOf('@') + 1) : id)); + const plural = topics.length !== 1; + const shownTopics = isShowAllTopics ? topics : topics.slice(0, startFromDegradedPreviewLimit); + const numHiddenTopics = topics.length - shownTopics.length; + + const copyTopics = () => { + // The WHOLE list, never what happens to be on screen - the bound above must not become the + // limit of what can be got out of here. + void copyToClipboard(topics.join('\n')).then((ok) => { + if (ok) notifySuccess(`${topics.length} topic${plural ? 's' : ''} copied to clipboard.`); + else notifyWarn(copyFailureMessage()); + }); + }; + + return ( + <div className={s.StartFromDegraded} data-testid="cs-start-from-degraded" role="status"> + {isCollapsed ? ( + <button + type="button" + className={s.StartFromDegradedToggle} + data-testid="cs-start-from-degraded-expand" + onClick={() => setIsCollapsed(false)} + > + Best effort: {topics.length} silent topic{plural ? 's' : ''} skipped - details + </button> + ) : ( + <> + <div> + Best effort: {topics.length} topic{plural ? 's' : ''} stayed silent while the start position was being + resolved and {plural ? 'were' : 'was'} skipped past. The start cut across topics is approximate + {plural ? ' for these topics' : ' for this topic'}: + </div> + <ul className={s.StartFromDegradedTopics} data-testid="cs-start-from-degraded-topics"> + {shownTopics.map(topic => <li key={topic}>{topic}</li>)} + </ul> + <div className={s.StartFromDegradedActions}> + {topics.length > startFromDegradedPreviewLimit && ( + <button + type="button" + className={s.StartFromDegradedToggle} + data-testid="cs-start-from-degraded-show-all" + onClick={() => setIsShowAllTopics(!isShowAllTopics)} + > + {isShowAllTopics ? `Show first ${startFromDegradedPreviewLimit}` : `and ${numHiddenTopics} more`} + </button> + )} + <button + type="button" + className={s.StartFromDegradedToggle} + data-testid="cs-start-from-degraded-copy" + onClick={copyTopics} + > + Copy all topics + </button> + <button + type="button" + className={s.StartFromDegradedToggle} + data-testid="cs-start-from-degraded-collapse" + onClick={() => setIsCollapsed(true)} + > + Collapse + </button> + </div> + </> + )} + </div> + ); +}; + +export default StartFromDegradedBanner; diff --git a/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.module.css b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.module.css new file mode 100644 index 000000000..40a5447db --- /dev/null +++ b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.module.css @@ -0,0 +1,20 @@ +.StartFromProgress { + display: flex; + flex-direction: column; + gap: 8rem; + min-width: 320rem; +} + +.Bar { + width: 100%; + height: 8rem; +} + +.Counts { + font-variant-numeric: tabular-nums; +} + +.Hint { + font-size: x-small; + color: grey; +} diff --git a/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.test.tsx b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.test.tsx new file mode 100644 index 000000000..7aa7d0383 --- /dev/null +++ b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.test.tsx @@ -0,0 +1,63 @@ +/** + * @jest-environment jsdom + * + * The panel shown while a large "skip first n messages" is being resolved. Skipping n messages + * exactly costs O(n) - Pulsar keeps no message-ordinal index - so a very large n genuinely takes + * time, and without this the session looks frozen on "Awaiting for new messages...". + */ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import StartFromProgress from './StartFromProgress'; + +describe('StartFromProgress', () => { + it('says what is happening and how far along it is', () => { + render(<StartFromProgress progress={{ messagesSkipped: 2_500_000, messagesToSkip: 10_000_000 }} />); + + const panel = screen.getByTestId('cs-start-from-progress'); + expect(panel.textContent).toMatch(/skipping/i); + // Both counts must be legible, and the reader must be able to see it is moving. + expect(panel.textContent).toContain('2,500,000'); + expect(panel.textContent).toContain('10,000,000'); + expect(panel.textContent).toContain('25%'); + }); + + it('drives a real progress indicator, not just text', () => { + render(<StartFromProgress progress={{ messagesSkipped: 2_500_000, messagesToSkip: 10_000_000 }} />); + + const bar = screen.getByTestId('cs-start-from-progress-bar'); + expect(bar.getAttribute('value')).toBe('2500000'); + expect(bar.getAttribute('max')).toBe('10000000'); + }); + + it('does not present entry position as an equivalent message-count position', () => { + render(<StartFromProgress progress={{ messagesSkipped: 2_500_000, messagesToSkip: 10_000_000 }} />); + + const panel = screen.getByTestId('cs-start-from-progress'); + expect(panel.textContent).toContain('based on retained entries, not message count'); + expect(panel.textContent).not.toContain('roughly the same place'); + }); + + it('exposes the raw counts for e2e, so a test does not have to parse prose', () => { + render(<StartFromProgress progress={{ messagesSkipped: 3_000_000, messagesToSkip: 4_000_000 }} />); + + const panel = screen.getByTestId('cs-start-from-progress'); + expect(panel.getAttribute('data-cs-skipped')).toBe('3000000'); + expect(panel.getAttribute('data-cs-to-skip')).toBe('4000000'); + expect(panel.getAttribute('data-cs-percent')).toBe('75'); + }); + + it('survives a zero total instead of rendering NaN%', () => { + // The server should never send this, but a divide-by-zero here would put "NaN%" on screen. + render(<StartFromProgress progress={{ messagesSkipped: 0, messagesToSkip: 0 }} />); + + const panel = screen.getByTestId('cs-start-from-progress'); + expect(panel.getAttribute('data-cs-percent')).toBe('0'); + expect(panel.textContent).not.toContain('NaN'); + }); + + it('never claims more than 100%', () => { + render(<StartFromProgress progress={{ messagesSkipped: 12, messagesToSkip: 10 }} />); + + expect(screen.getByTestId('cs-start-from-progress').getAttribute('data-cs-percent')).toBe('100'); + }); +}); diff --git a/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.tsx b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.tsx new file mode 100644 index 000000000..62a61616b --- /dev/null +++ b/ui/components/ui/ConsumerSession/StartFromProgress/StartFromProgress.tsx @@ -0,0 +1,55 @@ +import React from 'react'; +import s from './StartFromProgress.module.css'; + +/** How far a "skip first n messages" start-from has got, as the UI needs it. */ +export type StartFromSkipProgress = { + messagesSkipped: number; + messagesToSkip: number; +}; + +const formatCount = (n: number) => n.toLocaleString('en-US'); + +export type StartFromProgressProps = { + progress: StartFromSkipProgress; +}; + +/** + * Shown while a large "skip first n messages" start-from is still being resolved. + * + * Skipping n messages exactly costs O(n): Pulsar keeps no message-ordinal index, so the only way to + * land on message n is to count n messages. This panel exists so a big skip reads as "working" rather + * than "hung". + */ +const StartFromProgress: React.FC<StartFromProgressProps> = ({ progress }) => { + const { messagesSkipped, messagesToSkip } = progress; + const percent = messagesToSkip > 0 + ? Math.min(100, Math.floor((messagesSkipped / messagesToSkip) * 100)) + : 0; + + return ( + <div + className={s.StartFromProgress} + data-testid="cs-start-from-progress" + data-cs-skipped={messagesSkipped} + data-cs-to-skip={messagesToSkip} + data-cs-percent={percent} + > + <div>Skipping {formatCount(messagesToSkip)} messages before the first one is shown...</div> + <progress + className={s.Bar} + data-testid="cs-start-from-progress-bar" + value={messagesSkipped} + max={messagesToSkip} + /> + <div className={s.Counts}> + {formatCount(messagesSkipped)} of {formatCount(messagesToSkip)} skipped ({percent}%) + </div> + <div className={s.Hint}> + Skipping an exact number of messages means counting them one by one, so a large skip takes a while. + For an immediate estimate, use Approximate position (% of data); it is based on retained entries, not message count. + </div> + </div> + ); +}; + +export default StartFromProgress; diff --git a/ui/components/ui/ConsumerSession/Th.tsx b/ui/components/ui/ConsumerSession/Th.tsx index 4c862d06a..320b4bada 100644 --- a/ui/components/ui/ConsumerSession/Th.tsx +++ b/ui/components/ui/ConsumerSession/Th.tsx @@ -4,7 +4,7 @@ import { Sort, SortKey } from "./sort"; import arrowDownIcon from '../../ui/ChildrenTable/arrow-down.svg'; import arrowUpIcon from '../../ui/ChildrenTable/arrow-up.svg'; import SvgIcon from '../SvgIcon/SvgIcon'; -import { FC, MutableRefObject } from "react"; +import React, { FC, MutableRefObject } from "react"; import s from './ConsumerSession.module.css' import cts from "../../ui/ChildrenTable/ChildrenTable.module.css"; import { isEqual } from "lodash"; @@ -20,7 +20,10 @@ export type ThProps = { width?: number, onResizeStart?: (startClientX: number) => void, suppressSortClickRef?: MutableRefObject<boolean>, - testId?: string + testId?: string, + /** Native-drag column reorder, provided by the table that owns the order. */ + dragProps?: React.ThHTMLAttributes<HTMLTableCellElement>, + isDragOver?: boolean }; export const Th: FC<ThProps> = (props: ThProps) => { @@ -43,7 +46,13 @@ export const Th: FC<ThProps> = (props: ThProps) => { } return ( - <th className={`${cts.Th} ${s.Th}`} data-testid={props.testId} style={props.style} onClick={handleColumnHeaderClick}> + <th + className={`${cts.Th} ${s.Th} ${props.isDragOver ? s.ThDragOver : ''}`} + data-testid={props.testId} + style={props.style} + onClick={handleColumnHeaderClick} + {...(props.dragProps ?? {})} + > <div className={props.sortKey === undefined ? '' : cts.SortableTh} style={props.width === undefined ? undefined : { width: props.width, overflow: 'hidden' }} diff --git a/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.test.tsx b/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.test.tsx new file mode 100644 index 000000000..b107a6956 --- /dev/null +++ b/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.test.tsx @@ -0,0 +1,73 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The `customExportConditions` override keeps jsdom off its default "browser" export condition, + * which resolves `uuid` (pulled in transitively by the exporters) to its untranspiled ESM build. + * + * Regression: MessagesExporter recovers from an ExportConfig poisoned in local storage by + * resetting it to the default from ErrorBoundary#onError. It used to only bump the child's `key`, + * but react-error-boundary keeps its own `error` state until the boundary is reset (or its + * `resetKeys` change) - and the fallback here is empty. So the export modal went permanently blank + * instead of coming back with a working, default configuration. + * + * CS-EXPORT (e2e) asserts the "Resetting to default" toast only, so it passes on a blank modal. + */ +import React from 'react'; +import { render, screen, waitFor } from '@testing-library/react'; +import MessagesExporter from './MessagesExporter'; +import { defaultExportConfig } from './defaults'; +import { localStorageKeys } from '../../../../../local-storage-keys'; + +const key = localStorageKeys.messageExportConfig; + +// `fields` is missing, so `_MessagesExporter` throws on `props.config.fields.fields` while rendering +// the "Message fields N of M" label - exactly the class of local-storage rot the boundary exists for. +const poisonedConfig = { format: { type: 'json-message-per-entry' } }; + +function readStoredConfig(): unknown { + const raw = window.localStorage.getItem(key); + return raw === null ? null : JSON.parse(raw); +} + +describe('MessagesExporter recovers from a poisoned export config', () => { + let consoleError: jest.SpyInstance; + + beforeEach(() => { + window.localStorage.clear(); + // React logs the caught render error; keep the suite output readable. + consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + consoleError.mockRestore(); + window.localStorage.clear(); + }); + + it('brings the controls back and persists a usable default config', async () => { + window.localStorage.setItem(key, JSON.stringify(poisonedConfig)); + + render(<MessagesExporter messages={[]} sessionState="paused" />); + + // The modal is usable again: format picker, Export and Reset buttons are all back. + await waitFor(() => expect(screen.getByTestId('cs-export-run')).toBeTruthy()); + expect(screen.getByTestId('cs-export-format')).toBeTruthy(); + expect(screen.getByTestId('cs-export-reset')).toBeTruthy(); + + // The very section that threw renders again, from the recovered config. + const activeFields = defaultExportConfig.fields.fields.filter((f) => f.isActive).length; + expect(screen.getByText(`Message fields ${activeFields} of ${defaultExportConfig.fields.fields.length}`)).toBeTruthy(); + + // A subsequent export would run against the default config, not the poisoned one. + expect(readStoredConfig()).toEqual(defaultExportConfig); + }); + + it('renders the controls normally when the stored config is valid', () => { + window.localStorage.setItem(key, JSON.stringify(defaultExportConfig)); + + render(<MessagesExporter messages={[]} sessionState="paused" />); + + expect(screen.getByTestId('cs-export-run')).toBeTruthy(); + expect(consoleError).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.tsx b/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.tsx index edc87e6e5..73a04df75 100644 --- a/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.tsx +++ b/ui/components/ui/ConsumerSession/Toolbar/ExportMessagesButton/MessagesExporter/MessagesExporter.tsx @@ -169,6 +169,10 @@ const MessagesExporter = (props: MessagesExporterProps) => { setErrorKey(errorKey + 1); notifyInfo("Invalid export config. Resetting to default. Try to reload the page if the problem persists."); }} + // ErrorBoundary keeps its own error state (and keeps rendering the empty fallback) until it is + // reset. Re-keying the child alone left the user with a blank modal, so hand the bumped key to + // resetKeys - that clears the error and re-renders the controls with the default config. + resetKeys={[errorKey]} fallback={<></>} > <_MessagesExporter key={errorKey} {...props} config={config} onConfigChange={setConfig} /> diff --git a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.module.css b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.module.css index 8c17dc03e..c8b5f9aa5 100644 --- a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.module.css +++ b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.module.css @@ -19,6 +19,32 @@ justify-content: flex-end; } +/* Sits in the ToolbarLeft button row, between Play and Stop: spaced like a .Control, captions + left-aligned under the inputs so they read as labels rather than as right-aligned stats. */ +.DeliveryControl { + display: flex; + flex-direction: column; + align-items: flex-start; + margin-right: 8rem; +} + +.DeliveryControlInput { + width: 72rem; + font-size: 12rem; + padding: 1rem 6rem; + text-align: right; + border: 1rem solid var(--border-color, #d0d0d0); + border-radius: 4rem; + background: var(--surface-color, transparent); + color: inherit; +} + +.DeliveryControlCaption { + font-size: 12rem; + color: var(--text-color-secondary, #666); + white-space: nowrap; +} + .MessagesLoadedStats { display: flex; flex-direction: column; @@ -78,3 +104,24 @@ .NoData { color: #aaa; } + +/* The out-of-order warning beside the loaded count: the app's warning palette in a small round + badge, carrying its count and cause in the tooltip. */ +.OrderWarning { + display: inline-flex; + align-items: center; + justify-content: center; + align-self: center; + width: 22rem; + height: 22rem; + margin-left: 4rem; + /* Breathing room against whatever sits to the right (the Export messages button). */ + margin-right: 12rem; + border-radius: 50%; + background: var(--warning-background-color, #fff7e0); + color: var(--warning-text-color, #7a5b00); + border: 1rem solid var(--warning-text-color, #7a5b00); + font-size: 14rem; + font-weight: 700; + cursor: help; +} diff --git a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.test.tsx b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.test.tsx new file mode 100644 index 000000000..243af82ff --- /dev/null +++ b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.test.tsx @@ -0,0 +1,278 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The play/pause/resume button's disabled logic. An unconvertible stored config (no runtime config) + * must block only the transition that BUILDS a session from it - Play from `new`. Pause (`running` -> + * `pausing`) and Resume (`paused` -> `running`) are name-only RPCs that never touch the runtime + * config, so an unusable config must NOT strand a live session with Stop (which throws away the + * loaded messages) as its only move. + */ +import React from 'react'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import * as Modals from '../../../app/contexts/Modals/Modals'; +import Toolbar from './Toolbar'; +import { SessionState, ConsumerSessionConfig } from '../types'; + +const renderToolbar = ( + sessionState: SessionState, + config: ConsumerSessionConfig | undefined, + onToggleConsoleClick = jest.fn(), + overrides: Partial<React.ComponentProps<typeof Toolbar>> = {} +) => { + render( + <MemoryRouter> + <Modals.DefaultProvider> + <Toolbar + sessionState={sessionState} + config={config} + messages={[]} + onSessionStateChange={jest.fn()} + onStopSession={jest.fn()} + messagesLoaded={0} + messagesProcessed={0} + messagesLoadedPerSecond={{ prev: 0, now: 0 }} + messagesProcessedPerSecond={{ prev: 0, now: 0 }} + orderingLateDeliveries={0} + orderingActive={false} + orderingWaitingStreams={0} + orderKeyFallbacks={0} + replaySeamViolations={0} + onToggleConsoleClick={onToggleConsoleClick} + searchInResults="" + onSearchInResultsChange={jest.fn()} + numFoundInResults={0} + {...overrides} + /> + </Modals.DefaultProvider> + </MemoryRouter> + ); +}; + +const playButton = () => screen.getByTestId('cs-play') as HTMLButtonElement; + +// A defined config is "usable"; Toolbar only reads whether it is undefined. +const usableConfig = {} as ConsumerSessionConfig; + +describe('an unusable (undefined) config disables only the build-from-config transition', () => { + afterEach(cleanup); + + it('disables Play from `new`, which is the only state that builds a session from config', () => { + renderToolbar('new', undefined); + expect(playButton().disabled).toBe(true); + }); + + it('keeps Pause enabled while running - pausing does not consume the config', () => { + renderToolbar('running', undefined); + expect(playButton().disabled).toBe(false); + }); + + it('keeps Resume enabled while paused - resuming does not consume the config', () => { + renderToolbar('paused', undefined); + expect(playButton().disabled).toBe(false); + }); +}); + +describe('a usable config leaves the ordinary transitions enabled', () => { + afterEach(cleanup); + + it('enables Play from `new`', () => { + renderToolbar('new', usableConfig); + expect(playButton().disabled).toBe(false); + }); + + it('enables Pause while running', () => { + renderToolbar('running', usableConfig); + expect(playButton().disabled).toBe(false); + }); + + it('enables Resume while paused', () => { + renderToolbar('paused', usableConfig); + expect(playButton().disabled).toBe(false); + }); +}); + +describe('the transient states have no action, config aside', () => { + afterEach(cleanup); + + it.each([['initializing'], ['pausing']] as const)('disables the button in %s', (state) => { + renderToolbar(state, usableConfig); + expect(playButton().disabled).toBe(true); + }); +}); + +describe('the More tools control', () => { + afterEach(cleanup); + + it('uses the user-facing name and invokes the panel toggle', () => { + const onToggle = jest.fn(); + renderToolbar('new', usableConfig, onToggle); + + const button = screen.getByTestId('cs-tools'); + expect(button.textContent).toContain('More tools'); + fireEvent.click(button); + + expect(onToggle).toHaveBeenCalledTimes(1); + }); +}); + +describe('the delivery-order status', () => { + afterEach(cleanup); + + const orderedConfig = ( + messageDeliveryOrder: 'guaranteed' | 'best-effort', + deliveryOrderKey?: 'publish-time' | 'broker-publish-time' | 'event-time' + ) => ({ messageDeliveryOrder, deliveryOrderKey } as ConsumerSessionConfig); + + // 2026-08-11 (owner instruction): the always-visible mode label ("Best effort · Publish time") + // and its contract tooltip were REMOVED from the bar - the mode is configuration, documented at + // the config's help circles. What the bar still owes a session: the stall disclosure with its + // escape, the order-key fallbacks, and the out-of-order WARNING MARK beside the loaded count. + + it('shows no ordering status at all while an ordered run has nothing to disclose', () => { + renderToolbar('running', orderedConfig('guaranteed'), jest.fn(), { orderingActive: true }); + + expect(screen.queryByTestId('cs-order-chip')).toBeNull(); + expect(screen.queryByTestId('cs-order-warning')).toBeNull(); + }); + + it('marks out-of-order deliveries beside the loaded count under Best effort, counting the LATE counter', () => { + renderToolbar('running', orderedConfig('best-effort', 'event-time'), jest.fn(), { + orderingActive: true, + messagesLoaded: 120, + orderingLateDeliveries: 3, + }); + + const warning = screen.getByTestId('cs-order-warning'); + const html = warning.getAttribute('data-tooltip-html') ?? ''; + // The count leads, in bold; the cause in plain words; the window named. + expect(html).toContain('<strong>3</strong>'); + expect(html).toContain('arrived late'); + expect(html).toContain('Nothing is lost'); + expect(html).toContain('~0.75 s'); + }); + + it('marks out-of-order rows under Guaranteed, counting the SEAM counter - never both counters', () => { + // The server ticks lateEmissions and seamViolations on the identical guaranteed commit-phase + // condition; the mark must count each violation once, and say causes delivery did not add. + renderToolbar('running', orderedConfig('guaranteed'), jest.fn(), { + orderingActive: true, + orderingLateDeliveries: 3, + replaySeamViolations: 2, + }); + + const warning = screen.getByTestId('cs-order-warning'); + const html = warning.getAttribute('data-tooltip-html') ?? ''; + expect(html).toContain('<strong>2</strong>'); + expect(html).toContain('did not reorder them'); + // The common real-world cause is NAMED, not alluded to: several producers, own clocks. + expect(html).toContain('several producers'); + expect(html).toContain('stamps its own clock'); + expect(html).toContain('marked with'); + }); + + it('claims nothing at zero - the mark exists only when a violation happened', () => { + renderToolbar('running', orderedConfig('guaranteed'), jest.fn(), { + orderingActive: true, + replaySeamViolations: 0, + orderingLateDeliveries: 0, + }); + + expect(screen.queryByTestId('cs-order-warning')).toBeNull(); + }); + + it('shows a sustained wait as the status area, without any mode label around it', () => { + // The waiting disclosure survives the label removal for its documented corner: a stream a + // time seek positioned past its end can still hold the merge, and the bar must say so. + renderToolbar('running', orderedConfig('guaranteed'), jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 2 + }); + + const chip = screen.getByTestId('cs-order-chip'); + expect(screen.getByTestId('cs-order-waiting').textContent).toContain('waiting for 2 topics/partitions'); + expect(chip.textContent).not.toContain('Replaying history'); + expect(chip.textContent).not.toContain('Best effort'); + }); + + it('shows publish-time fallbacks in plain language', () => { + renderToolbar('running', orderedConfig('best-effort', 'event-time'), jest.fn(), { + orderingActive: true, + orderKeyFallbacks: 2, + }); + + expect(screen.getByTestId('cs-order-key-fallbacks').textContent).toContain('2 used publish time'); + }); + + it('shows no status for fastest delivery, which discloses nothing and warns of nothing', () => { + renderToolbar('running', { messageDeliveryOrder: 'as-received' } as ConsumerSessionConfig, jest.fn(), { + orderingActive: true + }); + + expect(screen.queryByTestId('cs-order-chip')).toBeNull(); + expect(screen.queryByTestId('cs-order-warning')).toBeNull(); + }); +}); + +describe('a disclosed Guaranteed stall is actionable from the chip itself', () => { + afterEach(cleanup); + + const stalled = { messageDeliveryOrder: 'guaranteed' } as ConsumerSessionConfig; + + it('switches the session to Best effort in one click, without opening configuration', () => { + const onDeliveryOrderChange = jest.fn(); + renderToolbar('running', stalled, jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 3, + onDeliveryOrderChange + }); + + fireEvent.click(screen.getByTestId('cs-order-switch-best-effort')); + + expect(onDeliveryOrderChange).toHaveBeenCalledTimes(1); + expect(onDeliveryOrderChange).toHaveBeenCalledWith('best-effort'); + }); + + it('names the escape in the user\'s terms, next to the wait it ends', () => { + renderToolbar('running', stalled, jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 3, + onDeliveryOrderChange: jest.fn() + }); + + const chip = screen.getByTestId('cs-order-chip'); + expect(chip.textContent).toContain('waiting for 3 topics/partitions'); + expect(screen.getByTestId('cs-order-switch-best-effort').textContent).toContain('Continue with Best effort'); + }); + + it('offers nothing while Guaranteed is merging normally - there is no stall to escape', () => { + renderToolbar('running', stalled, jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 0, + onDeliveryOrderChange: jest.fn() + }); + + expect(screen.queryByTestId('cs-order-switch-best-effort')).toBeNull(); + }); + + it('offers nothing in Best effort, which is already the mode the switch selects', () => { + renderToolbar('running', { messageDeliveryOrder: 'best-effort' } as ConsumerSessionConfig, jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 3, + onDeliveryOrderChange: jest.fn() + }); + + expect(screen.queryByTestId('cs-order-switch-best-effort')).toBeNull(); + }); + + it('offers nothing when no caller can apply the change - a dead control is worse than none', () => { + renderToolbar('running', stalled, jest.fn(), { + orderingActive: true, + orderingWaitingStreams: 3 + }); + + expect(screen.getByTestId('cs-order-waiting')).toBeTruthy(); + expect(screen.queryByTestId('cs-order-switch-best-effort')).toBeNull(); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.throttle.test.tsx b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.throttle.test.tsx new file mode 100644 index 000000000..5a8e9176a --- /dev/null +++ b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.throttle.test.tsx @@ -0,0 +1,107 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The two browser-wide delivery controls in the toolbar: "msg/s limit" and "pause after". Both are + * DRAFT-COMMITTED localStorage values - keystrokes edit a draft, only blur/Enter commits - so what + * these tests pin is the commit boundary: what reaches storage, and what can never reach it. + */ +import React from 'react'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import * as Modals from '../../../app/contexts/Modals/Modals'; +import Toolbar from './Toolbar'; +import { ConsumerSessionConfig, SessionState } from '../types'; +import { localStorageKeys } from '../../../local-storage-keys'; + +const renderToolbar = (sessionState: SessionState = 'new') => { + render( + <MemoryRouter> + <Modals.DefaultProvider> + <Toolbar + sessionState={sessionState} + config={{} as ConsumerSessionConfig} + messages={[]} + onSessionStateChange={jest.fn()} + onStopSession={jest.fn()} + messagesLoaded={0} + messagesProcessed={0} + messagesLoadedPerSecond={{ prev: 0, now: 0 }} + messagesProcessedPerSecond={{ prev: 0, now: 0 }} + orderingLateDeliveries={0} + orderingActive={false} + orderingWaitingStreams={0} + orderKeyFallbacks={0} + replaySeamViolations={0} + onToggleConsoleClick={jest.fn()} + searchInResults="" + onSearchInResultsChange={jest.fn()} + numFoundInResults={0} + /> + </Modals.DefaultProvider> + </MemoryRouter> + ); +}; + +const rateInput = () => screen.getByTestId('cs-rate-limit') as HTMLInputElement; +const pauseInput = () => screen.getByTestId('cs-pause-after') as HTMLInputElement; +const stored = (key: string) => window.localStorage.getItem(key); + +afterEach(() => { + cleanup(); + window.localStorage.clear(); +}); + +describe('the delivery controls commit to localStorage', () => { + it('typing a number and leaving the field commits it', () => { + renderToolbar(); + fireEvent.change(rateInput(), { target: { value: '250' } }); + // Storage still holds the mounted default mid-edit - a half-typed "2" must never become a + // live 2 msg/s limit. (The hook persists its default on mount, so "unset" reads as '0'.) + expect(stored(localStorageKeys.consumerSessionRateLimit)).toBe('0'); + fireEvent.blur(rateInput()); + expect(stored(localStorageKeys.consumerSessionRateLimit)).toBe('250'); + }); + + it('Enter commits too', () => { + renderToolbar(); + fireEvent.change(pauseInput(), { target: { value: '100' } }); + fireEvent.keyDown(pauseInput(), { key: 'Enter' }); + fireEvent.blur(pauseInput()); + expect(stored(localStorageKeys.consumerSessionPauseAfterLoaded)).toBe('100'); + }); + + it('invalid text is REJECTED wholesale, never repaired into a different number', () => { + // Stripping used to turn a pasted "1.5" into 15 and "1e3" into 13 - a number the user never + // typed. Rejection keeps whatever was there before. + renderToolbar(); + fireEvent.change(rateInput(), { target: { value: '250' } }); + fireEvent.change(rateInput(), { target: { value: '1e5-2.7' } }); + expect(rateInput().value).toBe('250'); + fireEvent.change(rateInput(), { target: { value: '1.5' } }); + expect(rateInput().value).toBe('250'); + }); + + it('a committed value is capped at the operational ceiling', () => { + renderToolbar(); + fireEvent.change(rateInput(), { target: { value: '9999999999' } }); + fireEvent.blur(rateInput()); + expect(stored(localStorageKeys.consumerSessionRateLimit)).toBe('1000000000'); + }); + + it('clearing the field commits 0 - the explicit OFF', () => { + window.localStorage.setItem(localStorageKeys.consumerSessionRateLimit, '250'); + renderToolbar(); + expect(rateInput().value).toBe('250'); + fireEvent.change(rateInput(), { target: { value: '' } }); + fireEvent.blur(rateInput()); + expect(stored(localStorageKeys.consumerSessionRateLimit)).toBe('0'); + expect(rateInput().placeholder).toBe('off'); + }); + + it('initializes from what an earlier session stored', () => { + window.localStorage.setItem(localStorageKeys.consumerSessionPauseAfterLoaded, '42'); + renderToolbar(); + expect(pauseInput().value).toBe('42'); + }); +}); diff --git a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.tsx b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.tsx index e8ca6fb1c..afb52316a 100644 --- a/ui/components/ui/ConsumerSession/Toolbar/Toolbar.tsx +++ b/ui/components/ui/ConsumerSession/Toolbar/Toolbar.tsx @@ -1,15 +1,80 @@ import React from 'react'; +import useLocalStorage from 'use-local-storage-state'; import s from './Toolbar.module.css' import pauseIcon from './icons/pause.svg'; import resumeIcon from './icons/resume.svg'; import resetIcon from './icons/reset.svg'; import consoleIcon from './icons/console.svg'; import * as I18n from '../../../app/contexts/I18n/I18n'; -import { SessionState, ConsumerSessionConfig, MessageDescriptor } from '../types'; +import { SessionState, ConsumerSessionConfig, MessageDescriptor, MessageDeliveryOrder } from '../types'; import SmallButton from '../../SmallButton/SmallButton'; import Input from '../../Input/Input'; import ExportMessagesButton from './ExportMessagesButton/ExportMessagesButton'; import { tooltipId } from '../../Tooltip/Tooltip'; +import { localStorageKeys } from '../../../local-storage-keys'; + +/** + * A small non-negative integer input committed on blur or Enter, with 0 meaning "off". + * + * DRAFT-COMMITTED, like the other numeric inputs in the session config: keystrokes edit a local + * draft, and only a valid commit reaches storage - so a half-typed value can never become the + * live setting, and an invalid one reverts to what was there before. + */ +export const DeliveryControlInput: React.FC<{ + testId: string; + caption: string; + title: string; + value: number; + onCommit: (n: number) => void; +}> = (props) => { + const [draft, setDraft] = React.useState<string>(props.value > 0 ? String(props.value) : ''); + + // An external change (another tab via the storage event, or a reset) replaces the draft - the + // input shows the live value whenever the user is not mid-edit. + React.useEffect(() => { + setDraft(props.value > 0 ? String(props.value) : ''); + }, [props.value]); + + // Above this, "messages per second" and "messages to load" stop meaning anything - and the + // wire carries an int64, so the ceiling also keeps the value inside every representation. + const maxCommittable = 1_000_000_000; + + const commit = () => { + // Digits only ever enter the draft (invalid text is REJECTED wholesale below, not stripped + // into a different number), so the only invalid draft is the empty string - the explicit off. + const parsed = draft === '' ? 0 : Number.parseInt(draft, 10); + const next = Number.isFinite(parsed) && parsed > 0 ? Math.min(parsed, maxCommittable) : 0; + setDraft(next > 0 ? String(next) : ''); + props.onCommit(next); + }; + + return ( + <div className={s.DeliveryControl} data-tooltip-id={tooltipId} data-tooltip-html={props.title}> + <input + className={s.DeliveryControlInput} + data-testid={props.testId} + value={draft} + placeholder="off" + inputMode="numeric" + onChange={(e) => { + // REJECT invalid text, never repair it: stripping turned a pasted "1.5" into 15 and + // "1e3" into 13 - a different number than the user gave, silently. + const next = e.target.value; + if (/^[0-9]{0,10}$/.test(next)) { + setDraft(next); + } + }} + onBlur={commit} + onKeyDown={(e) => { + if (e.key === 'Enter') { + (e.target as HTMLInputElement).blur(); + } + }} + /> + <span className={s.DeliveryControlCaption}>{props.caption}</span> + </div> + ); +}; export type ToolbarProps = { sessionState: SessionState; @@ -25,12 +90,60 @@ export type ToolbarProps = { searchInResults: string, onSearchInResultsChange: (v: string) => void, numFoundInResults: number, + /** How many messages the ordering layer emitted out of the selected-time order so far - + * best-effort mode's honest quality gauge (clock skew past the reorder window, retries). Under + * guaranteed the chip is NOT rendered: the server ticks this counter and the seam counter on + * the same commit-phase inversions, so showing both would count each event twice. */ + orderingLateDeliveries: number, + /** Whether the server actually RUNS an ordering layer for this session. A single-stream + * session builds none and pays no reorder latency, whatever the config asked - the chip must + * not claim otherwise. */ + orderingActive: boolean, + /** Topic/partition streams that have held delivery past the server's warning interval. Under + * the replay design this is the documented time-seek corner (a stream the seek positioned past + * its end); the disclosure and its escape stay for it. */ + orderingWaitingStreams: number, + /** Messages that used publish time because the selected time was absent. */ + orderKeyFallbacks: number, + /** Ordering violations the guaranteed replay delivered flagged: a producer clock wrote an + * earlier timestamp into a pause window, or the source log itself stores an inversion (the + * proto names both causes). Monotonic per session. */ + replaySeamViolations: number, + /** Switches THIS session's delivery order, applied by whoever owns the session configuration. + * + * A Guaranteed stream that a time seek positioned past its end can hold the merge (the + * documented stall corner), so disclosing the wait is not enough - the same chip that reports + * the stall has to be able to end it. When no caller can apply the change the control is not + * rendered at all: a button that does nothing is worse than no button. */ + onDeliveryOrderChange?: (order: MessageDeliveryOrder) => void, }; const Toolbar: React.FC<ToolbarProps> = (props) => { const i18n = I18n.useContext(); + // Browser-wide delivery controls, NOT session config: both live in localStorage and ride the + // session's requests (the rate on each Resume, the auto-pause purely client-side), so neither + // can travel with a saved session into a library item. + const [rateLimit, setRateLimit] = useLocalStorage<number>(localStorageKeys.consumerSessionRateLimit, { defaultValue: 0 }); + const [pauseAfter, setPauseAfter] = useLocalStorage<number>(localStorageKeys.consumerSessionPauseAfterLoaded, { defaultValue: 0 }); + const playButtonState = (props.sessionState === 'new' || props.sessionState === 'paused') ? 'play' : 'pause'; + // Be defensive at this presentation boundary too: older/incomplete callers that omit the + // field have the same meaning as protobuf UNSPECIFIED and old saved JSON - Guaranteed, the + // product default (owner decision 2026-08-11; third move of this default). + const messageDeliveryOrder = props.config?.messageDeliveryOrder ?? 'guaranteed'; + // What the warning mark beside the loaded count reports. Guaranteed: the seam counter (the late + // counter double-ticks on the same commit-phase inversions). Best effort: the late counter. + const outOfOrderCount = messageDeliveryOrder === 'guaranteed' + ? props.replaySeamViolations + : props.orderingLateDeliveries; + + // No runtime config means the stored configuration could not be converted into one - there is + // nothing to send. Play used to stay enabled and start a session that could never leave + // "initializing", which reads as a hang rather than as the configuration error it is. This only + // matters for the transition that BUILDS a session from the config - Play from `new`; pause and + // resume are name-only RPCs that never touch it (see the disabled scoping below). + const isConfigUnusable = props.config === undefined; let playButtonOnClick: () => void; switch (props.sessionState) { @@ -55,10 +168,25 @@ const Toolbar: React.FC<ToolbarProps> = (props) => { svgIcon={playButtonState === 'play' ? resumeIcon : pauseIcon} onClick={playButtonOnClick} type={'primary'} - disabled={props.sessionState !== 'new' && props.sessionState !== 'paused' && props.sessionState !== 'running'} + disabled={(isConfigUnusable && props.sessionState === 'new') || (props.sessionState !== 'new' && props.sessionState !== 'paused' && props.sessionState !== 'running')} /> </div> + <DeliveryControlInput + testId="cs-rate-limit" + caption="msg/s limit" + value={rateLimit} + onCommit={setRateLimit} + title={'Show at most this many messages per second. Leave empty for full speed.<br/>Takes effect when you press Play. Your browser remembers it - it is not saved with the session.'} + /> + <DeliveryControlInput + testId="cs-pause-after" + caption="pause after" + value={pauseAfter} + onCommit={setPauseAfter} + title={'Load exactly this many more messages, then pause - Play loads the next batch. Leave empty to turn this off.<br/>Your browser remembers it - it is not saved with the session.'} + /> + <div className={s.Control}> <SmallButton testId="cs-stop" @@ -73,14 +201,17 @@ const Toolbar: React.FC<ToolbarProps> = (props) => { <div className={s.Control}> <SmallButton testId="cs-tools" - title={"Toggle additional tools"} + title={"Toggle More tools"} svgIcon={consoleIcon} onClick={props.onToggleConsoleClick} - text={"Tools"} + text={"More tools"} type="regular" /> </div> + </div> + + <div className={s.ToolbarRight}> <div className={s.Control} style={{ position: 'relative', width: '320rem' }} @@ -101,9 +232,6 @@ const Toolbar: React.FC<ToolbarProps> = (props) => { </div> )} </div> - </div> - - <div className={s.ToolbarRight}> <div className={s.MessagesLoadedStats}> <div className={s.MessagesLoadedStat}> <strong className={s.MessagesLoadedStatValue} data-testid="cs-processed">{i18n.formatLongNumber(props.messagesProcessed)}</strong> @@ -126,6 +254,95 @@ const Toolbar: React.FC<ToolbarProps> = (props) => { </div> </div> + {/* Out-of-order disclosure, to the RIGHT of the counters it qualifies. The mode decides + which counter is the honest one: under Guaranteed the seam counter owns commit-phase + inversions (the late counter double-ticks on the same events); under Best effort the + late counter IS the mode's quality gauge. The tooltip carries the count in bold and + the plain-words cause; each guaranteed row also carries its own "!" marker. */} + {outOfOrderCount > 0 && ( + <span + className={s.OrderWarning} + data-testid="cs-order-warning" + role="img" + aria-label="Some messages were shown out of order" + data-tooltip-id={tooltipId} + data-tooltip-html={ + messageDeliveryOrder === 'guaranteed' + ? `<strong>${i18n.formatLongNumber(outOfOrderCount)}</strong> message${outOfOrderCount === 1 ? ' is' : 's are'} out of order on screen. ` + + 'Guaranteed did not reorder them: they were written to the topic out of order - ' + + 'typical when several producers share a topic, because each stamps its own clock, ' + + 'and send retries or batch flushes reorder them further - or a pause fell between ' + + 'replays. Each one is marked with "!".' + : `<strong>${i18n.formatLongNumber(outOfOrderCount)}</strong> message${outOfOrderCount === 1 ? ' was' : 's were'} shown out of order: ` + + 'they arrived late, after newer messages were already on screen. Nothing is lost. ' + + 'Best effort waits up to ~0.75 s for late messages, then moves on. ' + + 'Each one is marked with "!".' + } + > + ! + </span> + )} + + {/* The ordering STATUS area. The always-on mode label it used to carry ("Best effort · + Publish time") was removed 2026-08-11 (owner instruction) - the mode is configuration, + one click away, and the bar tells only what needs telling: a stall being disclosed + (with its one-click escape) and the order-key fallbacks. Renders nothing otherwise. */} + {props.config !== undefined && props.orderingActive + && (props.orderingWaitingStreams > 0 || props.orderKeyFallbacks > 0) && ( + <div + className={s.MessagesLoadedStats} + data-testid="cs-order-chip" + role="status" + > + <div className={s.MessagesLoadedStat}> + {props.orderingWaitingStreams > 0 && ( + <strong className={s.MessagesLoadedStatValue} data-testid="cs-order-waiting"> + waiting for {i18n.formatLongNumber(props.orderingWaitingStreams)} topics/partitions + </strong> + )} + {/* The escape from that wait, RIGHT NEXT TO IT. The replay waits only on streams + still inside their recorded range - but a range that can no longer be delivered + (trimmed by retention, or the start position seeked past the end) holds that + wait forever and looks exactly like an empty topic; the disclosure above says + so, and this ends it in one click, without going back through the configuration + screen. Offered only for a Guaranteed session that is actually stalled, and + only when a caller can apply the change. */} + {messageDeliveryOrder === 'guaranteed' + && props.orderingWaitingStreams > 0 + && props.onDeliveryOrderChange !== undefined && ( + <button + type="button" + data-testid="cs-order-switch-best-effort" + onClick={() => props.onDeliveryOrderChange?.('best-effort')} + title={ + 'Stop waiting for the silent topics/partitions. Switches this session to Best effort, ' + + 'which merges within a bounded reorder window and delivers late messages out of order ' + + 'rather than holding everything for them. Nothing already loaded is lost.' + } + style={{ + marginLeft: '8rem', + padding: '0 6rem', + border: '1rem solid currentColor', + borderRadius: '3rem', + background: 'transparent', + color: 'inherit', + font: 'inherit', + cursor: 'pointer', + whiteSpace: 'nowrap' + }} + > + Continue with Best effort + </button> + )} + {props.orderKeyFallbacks > 0 && ( + <strong className={s.MessagesLoadedStatValue} data-testid="cs-order-key-fallbacks"> +  · {i18n.formatLongNumber(props.orderKeyFallbacks)} used publish time + </strong> + )} + </div> + </div> + )} + <div> <ExportMessagesButton messages={props.messages} diff --git a/ui/components/ui/ConsumerSession/conversions/conversions.spec.ts b/ui/components/ui/ConsumerSession/conversions/conversions.spec.ts index 8ec326443..0003cc050 100644 --- a/ui/components/ui/ConsumerSession/conversions/conversions.spec.ts +++ b/ui/components/ui/ConsumerSession/conversions/conversions.spec.ts @@ -1,5 +1,12 @@ -import { partialMessageDescriptorToSerializable } from "./conversions"; -import { PartialMessageDescriptor } from "../types"; +import { partialMessageDescriptorToSerializable, startFromFromPb, startFromToPb } from "./conversions"; +import { ConsumerSessionStartFrom, PartialMessageDescriptor } from "../types"; +import * as managedPb from "../../../../grpc-web/tools/teal/pulsar/ui/library/v1/managed_items_pb"; +import { + managedConsumerSessionStartFromFromPb, + managedConsumerSessionStartFromToPb, +} from "../../LibraryBrowser/model/user-managed-items-conversions-pb"; +import { consumerSessionStartFromFromValOrRef } from "../../LibraryBrowser/model/resolved-items-conversions"; +import { relativeDateTimeValueMax } from "../../RelativeDateTimePicker/relative-date-time"; describe("partialMessageDescriptorToSerializable", () => { const testData: { @@ -108,3 +115,155 @@ describe("partialMessageDescriptorToSerializable", () => { } ); }); + +describe("startFrom wire mapping", () => { + // startFromToPb wrote nthMessageAfterEarliest/nthMessageBeforeLatest while startFromFromPb had no + // case for them, so the mapping was one-way: saving a "skip first n" session worked and loading it + // back threw "Unknown StartFrom value case". Per-mode tests are what let that survive - a mode with + // no test simply has no failing test. So this is keyed by the union's own `type`, which makes + // TypeScript refuse to compile the file if a new mode is added without a fixture here. + const fixtures: Record<ConsumerSessionStartFrom["type"], ConsumerSessionStartFrom> = { + earliestMessage: { type: "earliestMessage" }, + latestMessage: { type: "latestMessage" }, + nthMessageAfterEarliest: { type: "nthMessageAfterEarliest", n: 42 }, + nthMessageBeforeLatest: { type: "nthMessageBeforeLatest", n: 7 }, + // The two approximate modes carry the SAME payload, so the round trip CANNOT tell a SYMMETRIC + // oneof swap - both directions consistently reaching for the wrong case - from a correct + // mapping: fromPb(toPb(x)) still equals x while a stored item would flip modes on load. The + // asymmetric pin below ("write their own oneof field") is what actually catches that, by + // checking the raw pb one direction only. + approximateEntryPosition: { type: "approximateEntryPosition", fraction: 0.6 }, + approximatePublishTimePosition: { type: "approximatePublishTimePosition", fraction: 0.6 }, + messageId: { type: "messageId", hexString: "a1 b2 c3" }, + // Whole seconds only: startFromToPb floors to epoch seconds, so sub-second input cannot survive. + dateTime: { type: "dateTime", dateTime: new Date(1_700_000_000_000) }, + relativeDateTime: { + type: "relativeDateTime", + relativeDateTime: { unit: "hour", value: 3, isRoundedToUnitStart: true }, + }, + }; + + it.each(Object.entries(fixtures))("round-trips %s through protobuf", (_type, startFrom) => { + expect(startFromFromPb(startFromToPb(startFrom))).toEqual(startFrom); + }); + + it("preserves n rather than defaulting it to zero", () => { + // n rides a wrapper message, so a branch that returned the right `type` with a dropped payload + // would still satisfy a test that only checked the discriminant. + const skip = startFromFromPb(startFromToPb({ type: "nthMessageAfterEarliest", n: 1234 })); + const latest = startFromFromPb(startFromToPb({ type: "nthMessageBeforeLatest", n: 5678 })); + expect(skip).toEqual({ type: "nthMessageAfterEarliest", n: 1234 }); + expect(latest).toEqual({ type: "nthMessageBeforeLatest", n: 5678 }); + }); + + it("round-trips n = 0, which is also the protobuf int default", () => { + expect(startFromFromPb(startFromToPb({ type: "nthMessageAfterEarliest", n: 0 }))) + .toEqual({ type: "nthMessageAfterEarliest", n: 0 }); + expect(startFromFromPb(startFromToPb({ type: "nthMessageBeforeLatest", n: 0 }))) + .toEqual({ type: "nthMessageBeforeLatest", n: 0 }); + }); + + // The round trip above is blind to a symmetric oneof swap between the two approximate modes: they + // carry the same one-double payload, so a startFromToPb that wrote the wrong case AND a + // startFromFromPb that read the same wrong case would round-trip cleanly while silently flipping + // the mode of every stored item on load. Pin the WRITE side against the raw protobuf, one + // direction, so such a swap in startFromToPb cannot hide behind a matching read. + describe("the approximate modes write their own oneof field", () => { + it("sets the entry-position case for approximateEntryPosition, and only it", () => { + const pbValue = startFromToPb({ type: "approximateEntryPosition", fraction: 0.6 }); + expect(pbValue.hasStartFromApproximateEntryPosition()).toBe(true); + expect(pbValue.hasStartFromApproximatePublishTimePosition()).toBe(false); + expect(pbValue.getStartFromApproximateEntryPosition()!.getFraction()).toBe(0.6); + }); + + it("sets the publish-time-position case for approximatePublishTimePosition, and only it", () => { + const pbValue = startFromToPb({ type: "approximatePublishTimePosition", fraction: 0.6 }); + expect(pbValue.hasStartFromApproximatePublishTimePosition()).toBe(true); + expect(pbValue.hasStartFromApproximateEntryPosition()).toBe(false); + expect(pbValue.getStartFromApproximatePublishTimePosition()!.getFraction()).toBe(0.6); + }); + }); +}); + +/** + * A relative start position that was SAVED, not typed. + * + * The picker refuses anything the model cannot carry, so no in-memory fixture built through the UI + * can hold a bad value - but the library can. Its `ManagedRelativeDateTimeSpec.value` is an int64 + * while the request's `RelativeDateTime.value` is an int32, and older builds wrote whatever they + * were given. So these cases are built as bytes and loaded back the way a stored item actually + * arrives, which is the only shape that can represent the defect at all. + */ +describe("a relative start position loaded from the library", () => { + const savedRelativeStartFrom = (value: number): ConsumerSessionStartFrom => { + const savedPb = managedConsumerSessionStartFromToPb({ + metadata: { type: "consumer-session-start-from", id: "sf-1", name: "Saved", descriptionMarkdown: "" }, + spec: { + startFrom: { + type: "relativeDateTime", + relativeDateTime: { + type: "value", + val: { + metadata: { type: "relative-date-time", id: "rel-1", name: "Saved", descriptionMarkdown: "" }, + spec: { unit: "hour", value: 1, isRoundedToUnitStart: false }, + }, + }, + }, + }, + }); + + // The number as stored. The field is an int64, so every one of these genuinely survives a save + // and comes back on load. + savedPb.getSpec()!.getStartFromRelativeDateTime()!.getVal()!.getSpec()!.setValue(value); + + // Bytes in, model out - exactly how a library item reaches the session. + const restored = managedPb.ManagedConsumerSessionStartFrom.deserializeBinary(savedPb.serializeBinary()); + return consumerSessionStartFromFromValOrRef({ + type: "value", + val: managedConsumerSessionStartFromFromPb(restored), + }); + }; + + it.each([ + // Subtracting a negative is an instant in the FUTURE, under a label that reads "ago". It + // serializes cleanly, so nothing downstream ever questions it. + ["a negative value", -1], + // Past int32 the generated serializer fails an internal assertion while the request is being + // written - Play dies with a message about protobuf internals, on a session the user cannot + // see anything wrong with. + ["a value the request field cannot carry", relativeDateTimeValueMax + 1], + ])("is refused with an actionable message when it is %s", (_case, value) => { + const startFrom = savedRelativeStartFrom(value); + + expect(() => startFromToPb(startFrom)).toThrow(/relative start position/i); + expect(() => startFromToPb(startFrom)).toThrow(String(value)); + }); + + it.each([ + ["zero - a position the user can ask for on purpose", 0], + ["the largest value the model carries", relativeDateTimeValueMax], + ])("still starts a session from %s", (_case, value) => { + const startFrom = savedRelativeStartFrom(value); + + const request = startFromToPb(startFrom); + expect(request.getStartFromRelativeDateTime()!.getValue()).toBe(value); + // ...and the request survives the write. A value the field cannot hold fails HERE, deep inside + // generated code, which is why it has to be refused before it gets this far. + expect(() => request.serializeBinary()).not.toThrow(); + }); + + // Fractions and NaN cannot ride an int64 field, so they cannot come back from the library - but + // the same boundary is what stands between any other producer of this value and a request that + // dies in the serializer, so it refuses them too. + it.each([ + ["a fraction", 1.5], + ["not a number", Number.NaN], + ])("refuses %s at the same boundary", (_case, value) => { + const startFrom: ConsumerSessionStartFrom = { + type: "relativeDateTime", + relativeDateTime: { unit: "hour", value, isRoundedToUnitStart: false }, + }; + + expect(() => startFromToPb(startFrom)).toThrow(/relative start position/i); + }); +}); diff --git a/ui/components/ui/ConsumerSession/conversions/conversions.ts b/ui/components/ui/ConsumerSession/conversions/conversions.ts index 5c45a287c..99c2511b0 100644 --- a/ui/components/ui/ConsumerSession/conversions/conversions.ts +++ b/ui/components/ui/ConsumerSession/conversions/conversions.ts @@ -1,6 +1,8 @@ import { Timestamp } from "google-protobuf/google/protobuf/timestamp_pb"; import * as pb from "../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb"; import { hexStringFromByteArray, hexStringToByteArray } from "../../../conversions/conversions"; +import { messageIdError } from "../SessionConfiguration/StartFromInput/message-id"; +import { relativeDateTimeValueMax } from "../../RelativeDateTimePicker/relative-date-time"; import { MessageDescriptor, PartialMessageDescriptor, @@ -29,7 +31,8 @@ import { TestResult, ChainTestResult, JsMessageFilter, - ValueProjectionResult + ValueProjectionResult, + MessageDeliveryOrder } from "../types"; import { @@ -72,6 +75,7 @@ export function messageDescriptorFromPb(message: pb.Message): MessageDescriptor topic: message.getTopic()?.getValue() ?? null, sessionContextStateJson: message.getSessionContextStateJson()?.getValue() ?? null, debugStdout: message.getDebugStdout()?.getValue() ?? null, + deliveredOutOfOrder: message.getDeliveredOutOfOrder(), sessionTargetIndex: message.getSessionTargetIndex()?.getValue() ?? null, @@ -288,6 +292,48 @@ export function startFromFromPb(startFrom: pb.ConsumerSessionStartFrom): Consume case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_LATEST_MESSAGE: return { type: 'latestMessage' }; + // These two were missing while startFromToPb wrote them, so the mapping was one-way: a saved + // "skip first n" / "latest n" config serialized fine and then threw "Unknown StartFrom value + // case" on the way back in. + case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_NTH_MESSAGE_AFTER_EARLIEST: { + const nthMessageAfterEarliestPb = startFrom.getStartFromNthMessageAfterEarliest(); + if (nthMessageAfterEarliestPb === undefined) { + throw new Error('NthMessageAfterEarliest should be defined.'); + } + + return { type: 'nthMessageAfterEarliest', n: nthMessageAfterEarliestPb.getN() }; + } + + case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_NTH_MESSAGE_BEFORE_LATEST: { + const nthMessageBeforeLatestPb = startFrom.getStartFromNthMessageBeforeLatest(); + if (nthMessageBeforeLatestPb === undefined) { + throw new Error('NthMessageBeforeLatest should be defined.'); + } + + return { type: 'nthMessageBeforeLatest', n: nthMessageBeforeLatestPb.getN() }; + } + + // The two approximate modes carry the SAME payload - one double - so a branch that reached for + // the other one's oneof case would still produce a valid-looking fraction; the only symptom + // would be a session positioned by the wrong rule. + case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_APPROXIMATE_ENTRY_POSITION: { + const approximateEntryPositionPb = startFrom.getStartFromApproximateEntryPosition(); + if (approximateEntryPositionPb === undefined) { + throw new Error('ApproximateEntryPosition should be defined.'); + } + + return { type: 'approximateEntryPosition', fraction: approximateEntryPositionPb.getFraction() }; + } + + case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_APPROXIMATE_PUBLISH_TIME_POSITION: { + const approximatePublishTimePositionPb = startFrom.getStartFromApproximatePublishTimePosition(); + if (approximatePublishTimePositionPb === undefined) { + throw new Error('ApproximatePublishTimePosition should be defined.'); + } + + return { type: 'approximatePublishTimePosition', fraction: approximatePublishTimePositionPb.getFraction() }; + } + case pb.ConsumerSessionStartFrom.StartFromCase.START_FROM_MESSAGE_ID: { const byteArray = startFrom.getStartFromMessageId()?.getMessageId_asU8(); if (byteArray === undefined) { @@ -761,7 +807,7 @@ export function consumerSessionTargetToPb(v: ConsumerSessionTarget): pb.Consumer return targetPb; } -function startFromToPb(startFrom: ConsumerSessionStartFrom): pb.ConsumerSessionStartFrom { +export function startFromToPb(startFrom: ConsumerSessionStartFrom): pb.ConsumerSessionStartFrom { const startFromPb = new pb.ConsumerSessionStartFrom(); switch (startFrom.type) { @@ -777,23 +823,58 @@ function startFromToPb(startFrom: ConsumerSessionStartFrom): pb.ConsumerSessionS case 'nthMessageBeforeLatest': startFromPb.setStartFromNthMessageBeforeLatest(new pb.NthMessageBeforeLatest().setN(startFrom.n)); break; - case 'messageId': + case 'approximateEntryPosition': + startFromPb.setStartFromApproximateEntryPosition(new pb.ApproximateEntryPosition().setFraction(startFrom.fraction)); + break; + case 'approximatePublishTimePosition': + startFromPb.setStartFromApproximatePublishTimePosition(new pb.ApproximatePublishTimePosition().setFraction(startFrom.fraction)); + break; + case 'messageId': { + // The last point where a start position of zero bytes can still be stopped. The shared hex + // parser accepts blank text - an empty byte payload is a real thing - but an empty start + // position is not, and the server refuses it after a full create round trip. + const idError = messageIdError(startFrom.hexString); + if (idError !== undefined) { + throw new Error(idError); + } + startFromPb.setStartFromMessageId(new pb.MessageId().setMessageId(hexStringToByteArray(startFrom.hexString))); break; + } case 'dateTime': const epochSeconds = Math.floor(startFrom.dateTime.getTime() / 1000); const timestampPb = new Timestamp(); timestampPb.setSeconds(epochSeconds); startFromPb.setStartFromDateTime(new pb.DateTime().setDateTime(timestampPb)); break; - case 'relativeDateTime': + case 'relativeDateTime': { + // The last point where a number this start position cannot mean can still be stopped. The + // picker refuses these on the way in, but a value can also arrive from the LIBRARY, whose + // stored field is an int64 while this request field is an int32 - and older builds saved + // whatever they were handed. Left alone: + // + // - a negative is subtracted as a negative, so "n hours ago" seeks into the FUTURE, and it + // serializes cleanly, so nothing downstream ever questions it; + // - a fraction, NaN or anything past int32 fails an assertion inside the generated + // serializer, which kills Play with a message about protobuf internals. + // + // Refused here instead, where the create path turns it into a notification the user can act + // on, and where the configuration is still exactly as they saved it. + const { unit, value, isRoundedToUnitStart } = startFrom.relativeDateTime; + if (!Number.isInteger(value) || value < 0 || value > relativeDateTimeValueMax) { + throw new Error( + `Relative start position must be a whole number of ${unit}s from 0 to ${relativeDateTimeValueMax}, but it is ${value}.` + ); + } + const relativeDateTimePb = new pb.RelativeDateTime(); - relativeDateTimePb.setUnit(dateTimeUnitToPb(startFrom.relativeDateTime.unit)) - relativeDateTimePb.setValue(startFrom.relativeDateTime.value) - relativeDateTimePb.setIsRoundedToUnitStart(startFrom.relativeDateTime.isRoundedToUnitStart) + relativeDateTimePb.setUnit(dateTimeUnitToPb(unit)) + relativeDateTimePb.setValue(value) + relativeDateTimePb.setIsRoundedToUnitStart(isRoundedToUnitStart) startFromPb.setStartFromRelativeDateTime(relativeDateTimePb); break; + } default: throw new Error(`Unknown StartFrom type. ${startFrom}`); } @@ -801,6 +882,22 @@ function startFromToPb(startFrom: ConsumerSessionStartFrom): pb.ConsumerSessionS return startFromPb; } +/** + * The wire symbol for a delivery order. ONE mapping, deliberately: the session config sent by Play + * and the live SetDeliveryOrder switch must ask for the same thing, and two copies of this ternary + * could drift into a running session that no longer matches its own configuration. + * + * An unrecognized/absent value maps to Guaranteed, the product default (owner decision + * 2026-08-11) - which is what proto3 absence means on the server too. + */ +export function messageDeliveryOrderToPb(order: MessageDeliveryOrder | undefined): pb.MessageDeliveryOrder { + return order === 'as-received' + ? pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED + : order === 'best-effort' + ? pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + : pb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED; +} + export function consumerSessionConfigToPb(config: ConsumerSessionConfig): pb.ConsumerSessionConfig { const startFromPb = startFromToPb(config.startFrom); const targetsPb = config.targets.map(consumerSessionTargetToPb); @@ -816,6 +913,14 @@ export function consumerSessionConfigToPb(config: ConsumerSessionConfig): pb.Con configPb.setPauseTriggerChain(pauseTriggerChainPb); configPb.setColoringRuleChain(coloringRuleChainPb); configPb.setValueProjectionList(valueProjectionListPb); + configPb.setMessageDeliveryOrder(messageDeliveryOrderToPb(config.messageDeliveryOrder)); + configPb.setDeliveryOrderKey( + config.deliveryOrderKey === 'broker-publish-time' + ? pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME + : config.deliveryOrderKey === 'event-time' + ? pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME + : pb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME + ); return configPb; } @@ -869,4 +974,3 @@ export function valueProjectionResultToPb(v: ValueProjectionResult): pb.ValuePro } return resultPb; } - diff --git a/ui/components/ui/ConsumerSession/keyboard.spec.ts b/ui/components/ui/ConsumerSession/keyboard.spec.ts new file mode 100644 index 000000000..0aab72468 --- /dev/null +++ b/ui/components/ui/ConsumerSession/keyboard.spec.ts @@ -0,0 +1,148 @@ +import { KeyboardEvent } from "react"; +import { VirtuosoHandle } from "react-virtuoso"; +import { handleKeyDown } from "./keyboard"; +import { MessageDescriptor } from "./types"; +import { genEmptyMessageDescriptor } from "./testing"; + +/** + * Regression: the message table is focusable (tabIndex={0}) so keyboard users can reach the + * ArrowUp/ArrowDown/j/k navigation - but handleKeyDown called event.preventDefault() for EVERY key + * before looking at which key it was, so Tab / Shift-Tab could never move focus back out of the + * table. Only the keys the handler actually acts on may be swallowed. + * + * CS-26 (e2e) covers the navigation keys; nothing covered the keys that must pass through. + */ +const messages: MessageDescriptor[] = [0, 1, 2].map((i) => + genEmptyMessageDescriptor({ numMessageProcessed: i, displayIndex: i }), +); + +type Pressed = { + preventDefaultCalls: number; + selectionUpdates: number[][]; + scrolledTo: { index: number }[]; +}; + +function press( + key: string, + opts?: { shiftKey?: boolean; selected?: number[]; messages?: MessageDescriptor[] }, +): Pressed { + // handleKeyDown debounces itself against a module-level timestamp (64ms) - step the fake clock + // well past it so every press is handled on its own merits. + jest.advanceTimersByTime(1000); + + const result: Pressed = { preventDefaultCalls: 0, selectionUpdates: [], scrolledTo: [] }; + const event = { + key, + shiftKey: opts?.shiftKey ?? false, + preventDefault: () => { + result.preventDefaultCalls += 1; + }, + } as unknown as KeyboardEvent<HTMLDivElement>; + + handleKeyDown({ + event, + messages: opts?.messages ?? messages, + selectedMessages: opts?.selected ?? [0], + setSelectedMessages: (selected) => result.selectionUpdates.push(selected), + virtuoso: { + scrollIntoView: (location: { index: number }) => result.scrolledTo.push(location), + } as unknown as VirtuosoHandle, + }); + + return result; +} + +// File-level, NOT per describe: `handleKeyDown`'s debounce timestamp is module state shared by every +// test here, and it only ever moves forwards. Re-installing fake timers between describes resets the +// clock to the real "now", which is BEHIND the timestamp the previous describe left behind - every +// press after that looks like it arrived within 64ms of the last one and is silently dropped. +beforeAll(() => jest.useFakeTimers()); +afterAll(() => jest.useRealTimers()); + +describe("handleKeyDown only swallows the keys it handles", () => { + it("lets Tab through so focus can leave the message table", () => { + const got = press("Tab"); + expect(got.preventDefaultCalls).toBe(0); + expect(got.selectionUpdates).toEqual([]); + }); + + it("lets Shift-Tab through so focus can leave the message table backwards", () => { + const got = press("Tab", { shiftKey: true }); + expect(got.preventDefaultCalls).toBe(0); + expect(got.selectionUpdates).toEqual([]); + }); + + it.each(["a", "Escape", "Enter", "PageDown", "/"])("lets the unhandled key %s through", (key) => { + expect(press(key).preventDefaultCalls).toBe(0); + }); + + it.each(["ArrowUp", "k", "ArrowDown", "j"])("still swallows the navigation key %s", (key) => { + expect(press(key).preventDefaultCalls).toBe(1); + }); + + it("still moves the selection with the navigation keys", () => { + expect(press("ArrowDown", { selected: [0] }).selectionUpdates).toEqual([[1]]); + expect(press("j", { selected: [1] }).selectionUpdates).toEqual([[2]]); + expect(press("ArrowDown", { selected: [2] }).selectionUpdates).toEqual([[0]]); // wraps + expect(press("ArrowUp", { selected: [0] }).selectionUpdates).toEqual([[2]]); // wraps + expect(press("k", { selected: [2] }).selectionUpdates).toEqual([[1]]); + }); +}); + +/** + * ...and how navigation STARTS. + * + * The table is focusable precisely so a keyboard user can reach the navigation above - but every + * key it acted on required a message to be selected ALREADY, and the only thing that selected one + * was a mouse click. So a keyboard-only user could focus the table, press every key on it and never + * select a row: the message details panel was unreachable without a pointer. Neither the tests above + * nor e2e CS-26 noticed, because both seed the selection with a click first. + */ +describe("handleKeyDown starts navigating from no selection", () => { + it.each(["ArrowDown", "j"])("selects the first message on %s", (key) => { + expect(press(key, { selected: [] }).selectionUpdates).toEqual([[0]]); + }); + + it.each(["ArrowUp", "k"])("selects the last message on %s", (key) => { + // Up from nothing means the end of the list, which is where a session that has been running + // leaves the messages worth looking at. + expect(press(key, { selected: [] }).selectionUpdates).toEqual([[2]]); + }); + + it("selects the first message on Enter", () => { + expect(press("Enter", { selected: [] }).selectionUpdates).toEqual([[0]]); + }); + + it("swallows the Enter it acts on, and only that one", () => { + // Swallow exactly what was acted on: an Enter pressed with a selection already in place is not + // this handler's, and taking it would deny it to anything else on the page. + expect(press("Enter", { selected: [] }).preventDefaultCalls).toBe(1); + expect(press("Enter", { selected: [0] }).preventDefaultCalls).toBe(0); + }); + + it("scrolls the row it just selected into view", () => { + // Selecting a row nobody can see is not selecting it: the table is virtualized, and the last + // row of a long session is far off screen. + expect(press("ArrowUp", { selected: [] }).scrolledTo).toEqual([{ index: 2, align: "end" }]); + expect(press("ArrowDown", { selected: [] }).scrolledTo).toEqual([{ index: 0, align: "start" }]); + }); + + it("starts from a selection of several messages too", () => { + // The same dead end: `!== 1` covered "none" and "many" alike. + expect(press("ArrowDown", { selected: [1, 2] }).selectionUpdates).toEqual([[0]]); + }); + + it("does nothing at all when the table has no messages", () => { + const got = press("ArrowDown", { selected: [], messages: [] }); + + expect(got.selectionUpdates).toEqual([]); + expect(got.scrolledTo).toEqual([]); + }); + + it("still lets Tab out of a table with nothing selected", () => { + const got = press("Tab", { selected: [] }); + + expect(got.preventDefaultCalls).toBe(0); + expect(got.selectionUpdates).toEqual([]); + }); +}); diff --git a/ui/components/ui/ConsumerSession/keyboard.ts b/ui/components/ui/ConsumerSession/keyboard.ts index 2a5274ac7..846bd5e5b 100644 --- a/ui/components/ui/ConsumerSession/keyboard.ts +++ b/ui/components/ui/ConsumerSession/keyboard.ts @@ -12,12 +12,21 @@ export type HandleKeyDownProps = { const arrowUpKeys = ['ArrowUp', 'k']; const arrowDownKeys = ['ArrowDown', 'j']; +/** Keys that start navigating from nothing, without moving an existing selection. */ +const enterKeys = ['Enter']; let lastKeyDownTime = new Date().getTime(); export function handleKeyDown(props: HandleKeyDownProps) { const { event, messages, selectedMessages, setSelectedMessages, virtuoso } = props; - event.preventDefault(); + + // Only swallow the keys this handler acts on. The table is focusable, so preventing the default + // for every key trapped focus inside it - Tab / Shift-Tab could no longer move focus out. + // Enter is deliberately NOT here: it is acted on only when there is no selection to move, and it + // is swallowed there rather than for every press. + if (arrowUpKeys.includes(event.key) || arrowDownKeys.includes(event.key)) { + event.preventDefault(); + } // Debounce frequent events for better performance const keyDownTime = new Date().getTime(); @@ -26,7 +35,32 @@ export function handleKeyDown(props: HandleKeyDownProps) { } lastKeyDownTime = keyDownTime; + const select = (index: number) => { + setSelectedMessages([messages[index].numMessageProcessed!]); + virtuoso.scrollIntoView({ index, align: index === 0 ? 'start' : 'end' }); + }; + + // Nothing (or a whole group) is selected: this press has to CREATE the first selection, or the + // table is unreachable without a pointer - the table is focusable precisely so that it is not. + // Down/Enter start at the top, Up starts at the bottom, which is where a session that has been + // running leaves the messages worth looking at. if (selectedMessages.length !== 1) { + if (messages.length === 0) { + return; + } + + if (arrowDownKeys.includes(event.key) || enterKeys.includes(event.key)) { + // Acted on, so swallowed - unlike an Enter pressed with a selection already in place, which + // this handler leaves entirely alone. + event.preventDefault(); + select(0); + return; + } + + if (arrowUpKeys.includes(event.key)) { + select(messages.length - 1); + } + return; } diff --git a/ui/components/ui/ConsumerSession/message-columns.ts b/ui/components/ui/ConsumerSession/message-columns.ts index 90bdb5833..96a5346a1 100644 --- a/ui/components/ui/ConsumerSession/message-columns.ts +++ b/ui/components/ui/ConsumerSession/message-columns.ts @@ -1,8 +1,9 @@ // Canonical keys + default widths (px) for the resizable columns of the consumer-session -// message table. The `index` column is intentionally excluded: it stays a fixed-width sticky -// column whose width feeds `publishTime`'s sticky offset. Value-projection columns are dynamic -// and keep their own configured widths. +// message table - INCLUDING `index` (resizable since 2026-08-11; it stays sticky, and its live +// width feeds `publishTime`'s sticky offset through the --cs-index-cell-total variable). +// Value-projection columns are dynamic and keep their own configured widths. export type MessageColumnKey = + | 'index' | 'publishTime' | 'key' | 'value' @@ -21,6 +22,7 @@ export type MessageColumnKey = | 'sessionContextState'; export const messageColumnDefaultWidths: Record<MessageColumnKey, number> = { + index: 36, publishTime: 180, key: 160, value: 240, @@ -38,3 +40,51 @@ export const messageColumnDefaultWidths: Record<MessageColumnKey, number> = { redeliveryCount: 130, sessionContextState: 380, }; + +/** Everything except the sticky index / publish-time pair - what the drag-reorder system (and + * therefore `columnOrder` everywhere) actually carries. */ +export type ReorderableMessageColumnKey = Exclude<MessageColumnKey, 'index' | 'publishTime'>; + +/** The message columns a user may DRAG into any order - this is also the default order. */ +export const reorderableMessageColumns: ReorderableMessageColumnKey[] = [ + 'key', + 'value', + 'sessionTargetIndex', + 'topic', + 'producerName', + 'schemaVersion', + 'size', + 'properties', + 'eventTime', + 'brokerPublishTime', + 'messageId', + 'sequenceId', + 'orderingKey', + 'redeliveryCount', + 'sessionContextState', +]; + +/** Everything the header needs to render one reorderable column, keyed like the row cells. */ +import type { SortKey } from './sort'; + +export const messageThMeta: Record<MessageColumnKey, { testId: string; title: string; sortKey: SortKey; helpKey: string }> = { + // `index` and `publishTime` render their headers bespoke (sticky pair); their entries exist so + // this record stays total over the key type. + index: { testId: 'cs-th-index', title: '#', sortKey: 'index', helpKey: 'index' }, + publishTime: { testId: 'cs-th-publishTime', title: 'Publish time', sortKey: 'publishTime', helpKey: 'publishTime' }, + key: { testId: 'cs-th-key', title: 'Key', sortKey: 'key', helpKey: 'key' }, + value: { testId: 'cs-th-value', title: 'Value', sortKey: 'value', helpKey: 'value' }, + sessionTargetIndex: { testId: 'cs-th-target', title: 'Target', sortKey: 'sessionTargetIndex', helpKey: 'sessionTargetIndex' }, + topic: { testId: 'cs-th-topic', title: 'Topic', sortKey: 'topic', helpKey: 'topic' }, + producerName: { testId: 'cs-th-producer', title: 'Producer', sortKey: 'producerName', helpKey: 'producerName' }, + schemaVersion: { testId: 'cs-th-schemaVersion', title: 'Schema version', sortKey: 'schemaVersion', helpKey: 'schemaVersion' }, + size: { testId: 'cs-th-size', title: 'Size', sortKey: 'size', helpKey: 'size' }, + properties: { testId: 'cs-th-properties', title: 'Properties', sortKey: 'properties', helpKey: 'propertiesMap' }, + eventTime: { testId: 'cs-th-eventTime', title: 'Event time', sortKey: 'eventTime', helpKey: 'eventTime' }, + brokerPublishTime: { testId: 'cs-th-brokerPublishTime', title: 'Broker pub. time', sortKey: 'brokerPublishTime', helpKey: 'brokerPublishTime' }, + messageId: { testId: 'cs-th-messageId', title: 'Message Id', sortKey: 'messageId', helpKey: 'messageId' }, + sequenceId: { testId: 'cs-th-sequenceId', title: 'Sequence Id', sortKey: 'sequenceId', helpKey: 'sequenceId' }, + orderingKey: { testId: 'cs-th-orderingKey', title: 'Ordering key', sortKey: 'orderingKey', helpKey: 'orderingKey' }, + redeliveryCount: { testId: 'cs-th-redeliveryCount', title: 'Redelivery count', sortKey: 'redeliveryCount', helpKey: 'redeliveryCount' }, + sessionContextState: { testId: 'cs-th-sessionContextState', title: 'Session Context State', sortKey: 'sessionContextStateJson', helpKey: 'sessionContextStateJson' }, +}; diff --git a/ui/components/ui/ConsumerSession/sort.test.ts b/ui/components/ui/ConsumerSession/sort.test.ts new file mode 100644 index 000000000..b98fa8b84 --- /dev/null +++ b/ui/components/ui/ConsumerSession/sort.test.ts @@ -0,0 +1,60 @@ +/** + * BUG-3 regression: sortMessages must not reorder the array it is given. + * + * ConsumerSession passes the `messages` React state array straight into sortMessages whenever the + * session is paused and the search box is empty, so an in-place `sort()`/`reverse()` rewrites state + * into VISUAL order. The retention logic (`.slice(-numDisplayItems)`) then drops messages by the + * last visual sort instead of by arrival order. + */ +import { genEmptyMessageDescriptor } from './testing'; +import { sortMessages } from './sort'; + +// Arrival order 3, 1, 2 - deliberately unsorted for every key under test. +const arrivalOrder = [3, 1, 2]; +const makeMessages = () => + arrivalOrder.map((n) => + genEmptyMessageDescriptor({ + displayIndex: n, + key: `k-${n}`, + value: `v-${n}`, + publishTime: n, + numMessageProcessed: n, + }) + ); + +describe('BUG-3: sortMessages is immutable', () => { + it('sorts by index without touching the input array', () => { + const input = makeMessages(); + const sorted = sortMessages(input, { key: 'index', direction: 'desc' }); + + expect(sorted.map((m) => m.displayIndex)).toEqual([3, 2, 1]); + expect(input.map((m) => m.displayIndex)).toEqual(arrivalOrder); + expect(sorted).not.toBe(input); + }); + + it('sorts by key without touching the input array', () => { + const input = makeMessages(); + const sorted = sortMessages(input, { key: 'key', direction: 'asc' }); + + expect(sorted.map((m) => m.key)).toEqual(['k-1', 'k-2', 'k-3']); + expect(input.map((m) => m.displayIndex)).toEqual(arrivalOrder); + }); + + it('sorts by value without touching the input array', () => { + const input = makeMessages(); + const sorted = sortMessages(input, { key: 'value', direction: 'desc' }); + + expect(sorted.map((m) => m.value)).toEqual(['v-3', 'v-2', 'v-1']); + expect(input.map((m) => m.displayIndex)).toEqual(arrivalOrder); + }); + + it('keeps arrival order intact across repeated sorts of the same array', () => { + // The paused session re-sorts the SAME state array on every sort click / re-render. + const input = makeMessages(); + sortMessages(input, { key: 'index', direction: 'desc' }); + sortMessages(input, { key: 'key', direction: 'asc' }); + sortMessages(input, { key: 'publishTime', direction: 'desc' }); + + expect(input.map((m) => m.displayIndex)).toEqual(arrivalOrder); + }); +}); diff --git a/ui/components/ui/ConsumerSession/sort.ts b/ui/components/ui/ConsumerSession/sort.ts index 62d4be2bb..5bb7ae37d 100644 --- a/ui/components/ui/ConsumerSession/sort.ts +++ b/ui/components/ui/ConsumerSession/sort.ts @@ -55,7 +55,9 @@ export const sortMessages = ( undefs: MessageDescriptor[], sortFn: SortFn ): MessageDescriptor[] { - let result = defs.sort(sortFn); + // Copy first: `defs` is often the caller's array (the ConsumerSession `messages` state when the + // session is paused), and an in-place sort would rewrite it into visual order. + let result = defs.slice().sort(sortFn); result = sort.direction === "asc" ? result : result.reverse(); return result.concat(undefs); } diff --git a/ui/components/ui/ConsumerSession/testing.ts b/ui/components/ui/ConsumerSession/testing.ts index 0c0970026..0b86c54f1 100644 --- a/ui/components/ui/ConsumerSession/testing.ts +++ b/ui/components/ui/ConsumerSession/testing.ts @@ -36,6 +36,7 @@ export function genMessageDescriptor( sessionValueProjectionListResult: [], numMessageProcessed: 0, numMessageSent: 0, + deliveredOutOfOrder: false, ...override, }; } @@ -73,6 +74,7 @@ export function genEmptyMessageDescriptor( sessionValueProjectionListResult: [], numMessageProcessed: 0, numMessageSent: 0, + deliveredOutOfOrder: false, ...override, }; } diff --git a/ui/components/ui/ConsumerSession/types.ts b/ui/components/ui/ConsumerSession/types.ts index e3049284d..e93f31dca 100644 --- a/ui/components/ui/ConsumerSession/types.ts +++ b/ui/components/ui/ConsumerSession/types.ts @@ -77,6 +77,12 @@ export type ConsumerSessionStartFrom = { type: "latestMessage" } | { type: "nthMessageAfterEarliest", n: number } | { type: "nthMessageBeforeLatest", n: number } | + // Approximate data position: proportional to retained entries, per physical topic. One entry can + // hold a batch, so this is deliberately not presented as an exact message percentile. + { type: "approximateEntryPosition", fraction: number } | + // Approximate publish-time position: interpolated between observed first- and final-entry publish + // times, per logical topic. All partitions of the topic use the same cutoff. + { type: "approximatePublishTimePosition", fraction: number } | { type: "messageId"; hexString: string } | { type: "dateTime"; dateTime: Date } | { @@ -84,6 +90,41 @@ export type ConsumerSessionStartFrom = relativeDateTime: RelativeDateTime }; +/** How a session reading more than one topic or partition interleaves messages. Guaranteed waits + * for every source without a timeout; Best effort uses a ~0.75 s reorder window; Fastest does not + * reorder across sources. A single source needs no ordering layer. A missing value means + * Guaranteed. + * + * `'best-effort'` is deliberately NOT named after a timestamp: which timestamp the merge compares + * is a separate choice, `DeliveryOrderKey`. The wire constant it maps to still reads + * `MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME` because it predates that choice, and its + * name and value 2 are frozen for compatibility - the mapping lives in + * `conversions/conversions.ts` and `LibraryBrowser/model/user-managed-items-conversions-pb.ts`. + * These three strings are also the `<option value>`s of the Delivery order select, so the e2e and + * jest assertions on `cs-delivery-order` move with them. */ +export type MessageDeliveryOrder = 'as-received' | 'best-effort' | 'guaranteed'; + +/** Which Pulsar timestamp the delivery order compares (not the message `orderingKey`). Broker + * publish time requires broker entry metadata. Event time is optional and application-set. + * Missing broker/event timestamps use publish time and are counted. */ +export type DeliveryOrderKey = 'publish-time' | 'broker-publish-time' | 'event-time'; + +/** + * The guaranteed replay's caught-up announcement, as one record: the boundary the replay delivered + * up to, and the topics excluded from this replay chunk. + * + * The wire also carries `replay_newer_entries_approx`, deliberately NOT kept here: it counts broker + * ENTRIES rather than messages (under batching one entry holds many), so it could only ever be + * shown as a lower bound in a unit no reader thinks in - dropped 2026-08-11. + * `excludedTopics` is capped at 5 names by the server; + * `excludedTopicCount` carries the true count. + */ +export type ReplayCaughtUpStats = { + boundaryAtMs: number; + excludedTopics: string[]; + excludedTopicCount: number; +}; + export type ConsumerSessionConfig = { startFrom: ConsumerSessionStartFrom; targets: ConsumerSessionTarget[]; @@ -91,7 +132,10 @@ export type ConsumerSessionConfig = { pauseTriggerChain: ConsumerSessionPauseTriggerChain; coloringRuleChain: ColoringRuleChain; valueProjectionList: ValueProjectionList; - numDisplayItems: number + numDisplayItems: number; + // Runtime configs are normalized at the saved-item boundary, so this is always explicit. + messageDeliveryOrder: MessageDeliveryOrder; + deliveryOrderKey?: DeliveryOrderKey; }; export type ValueProjectionResult = { @@ -124,6 +168,13 @@ export type MessageDescriptor = { numMessageProcessed: Nullable<number>; numMessageSent: Nullable<number>; debugStdout: Nullable<string>; + /** The guaranteed replay delivered this message out of the selected-time order across a pause + * seam (a producer clock wrote an earlier timestamp into the pause window). Detected at + * emission by the server, flagged loudly, never dropped. Absent on the wire means false. */ + /** The ordering layer emitted this row below an already-emitted key (renamed from + * replaySeamViolation on 2026-08-11, when Best effort's late emissions joined Guaranteed's + * seam violations in carrying it). The row shows the out-of-order marker for it. */ + deliveredOutOfOrder: boolean; sessionTargetIndex: Nullable<number>, diff --git a/ui/components/ui/Input/Input.module.css b/ui/components/ui/Input/Input.module.css index c92382fea..36a9a9f65 100644 --- a/ui/components/ui/Input/Input.module.css +++ b/ui/components/ui/Input/Input.module.css @@ -145,6 +145,12 @@ opacity: 0.5; } +/* Shown, because the setting is part of what is being read - but not offered as a control. */ +.AddonReadOnly, +.AddonReadOnly:hover { + cursor: default; +} + .AddonLabel { width: 16rem; height: 20rem; diff --git a/ui/components/ui/Input/Input.test.tsx b/ui/components/ui/Input/Input.test.tsx new file mode 100644 index 000000000..069a7bb6f --- /dev/null +++ b/ui/components/ui/Input/Input.test.tsx @@ -0,0 +1,163 @@ +/** + * @jest-environment jsdom + * + * The shared text/number field, and specifically WHO WINS when two different things have an opinion + * about whether it is editable. + * + * `isReadOnly` is how the whole app renders a referenced (library-owned) configuration: the value on + * screen belongs to a stored item and must not be edited in place. Callers independently pass + * `inputProps` to forward native attributes such as `min`/`max`/`step`, and a caller that also + * mentions `disabled` there - even as `undefined`, which is what `inputProps={{ disabled: + * props.disabled }}` produces on a component with no `disabled` prop - must not be able to hand the + * field back to the user. + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import Input from './Input'; + +const renderInput = (props: Record<string, unknown>) => { + const onChange = jest.fn(); + render(<Input testId="the-input" value="5" onChange={onChange} {...(props as Record<string, never>)} />); + return { onChange, input: () => screen.getByTestId('the-input') as HTMLInputElement }; +}; + +describe('a read-only Input', () => { + it('stays disabled when a caller passes inputProps without a disabled of its own', () => { + // `{ disabled: props.disabled, min: 0 }` with no `disabled` prop set - the exact shape the + // start-from number fields pass. + const { input } = renderInput({ isReadOnly: true, inputProps: { disabled: undefined, min: 0 } }); + + expect(input().disabled).toBe(true); + }); + + it('stays disabled when a caller explicitly passes disabled: false', () => { + const { input } = renderInput({ isReadOnly: true, inputProps: { disabled: false } }); + + expect(input().disabled).toBe(true); + }); + + it('refuses real typing, not merely styling', async () => { + // userEvent, not fireEvent: fireEvent dispatches the change event straight at the element and + // would "type" into a disabled field that no browser would accept. + const { onChange, input } = renderInput({ isReadOnly: true, inputProps: { disabled: undefined } }); + + await userEvent.type(input(), '9'); + + expect(onChange).not.toHaveBeenCalled(); + }); + + it('still honours a caller that disables the field on its own', () => { + const { input } = renderInput({ inputProps: { disabled: true } }); + + expect(input().disabled).toBe(true); + }); + + it('leaves an ordinary field editable', () => { + const { onChange, input } = renderInput({ inputProps: { disabled: undefined, min: 0 } }); + + expect(input().disabled).toBe(false); + fireEvent.change(input(), { target: { value: '9' } }); + expect(onChange).toHaveBeenCalledWith('9'); + }); + + it('still forwards the other native attributes it was given', () => { + // The spread has to keep working - `min`/`max`/`step` are what the number fields rely on. + const { input } = renderInput({ inputProps: { min: 0, max: 100, step: 'any' } }); + + expect(input().getAttribute('min')).toBe('0'); + expect(input().getAttribute('max')).toBe('100'); + expect(input().getAttribute('step')).toBe('any'); + }); +}); + +/** + * `disabled` on the native `<input>` stops TYPING and nothing else. This component ships two other + * controls of its own, and both mutate: + * + * - the addons, which are how the regex `m`/`i` flags and the match-case switch are toggled - they + * change the value the filter is evaluated with, not merely how it is displayed; + * - the clear button, which sets the value to `''`. + * + * Neither is a `<button>`, so neither inherits anything from the field being disabled: they are + * plain divs with an `onClick`, live in every read-only render, and a click on one edits a + * configuration the user is only supposed to be looking at. + */ +describe('a read-only Input still has controls that mutate', () => { + const addon = (onClick: () => void) => ({ + id: 'the-addon', + isEnabled: false, + onClick, + label: 'm', + }); + + const renderWithAddon = (props: Record<string, unknown>) => { + const onAddonClick = jest.fn(); + const onChange = jest.fn(); + render( + <Input + testId="the-input" + value="5" + onChange={onChange} + addons={[addon(onAddonClick)]} + {...(props as Record<string, never>)} + /> + ); + // The addon is a div, not a role - the label is all it has. + return { onAddonClick, onChange, addonEl: () => screen.getByText('m') }; + }; + + it('does not toggle an addon when it is read-only', () => { + const { onAddonClick, addonEl } = renderWithAddon({ isReadOnly: true }); + + fireEvent.click(addonEl()); + + expect(onAddonClick).not.toHaveBeenCalled(); + }); + + it('still toggles an addon when it is editable', () => { + // The counterpart: "never toggles" is otherwise satisfiable by never toggling at all. + const { onAddonClick, addonEl } = renderWithAddon({}); + + fireEvent.click(addonEl()); + + expect(onAddonClick).toHaveBeenCalledTimes(1); + }); + + it('keeps showing what the addon state IS, since that is part of the value', () => { + // Suppressing the mutation must not hide the setting: a reader of a stored filter needs to see + // that it is case-insensitive. + render( + <Input + testId="the-input" + value="5" + onChange={() => undefined} + addons={[{ id: 'a', isEnabled: true, onClick: () => undefined, label: 'i' }]} + isReadOnly + /> + ); + + expect(screen.getByText('i')).toBeTruthy(); + }); + + it('does not offer to clear the value when it is read-only', () => { + const onChange = jest.fn(); + const { container } = render( + <Input testId="the-input" value="5" onChange={onChange} clearable isReadOnly /> + ); + + // Whatever remains on screen, nothing there may empty the field. + Array.from(container.querySelectorAll('div')).forEach((el) => fireEvent.click(el)); + + expect(onChange).not.toHaveBeenCalled(); + }); + + it('still clears the value when it is editable', () => { + const onChange = jest.fn(); + const { container } = render(<Input testId="the-input" value="5" onChange={onChange} clearable />); + + Array.from(container.querySelectorAll('div')).forEach((el) => fireEvent.click(el)); + + expect(onChange).toHaveBeenCalledWith(''); + }); +}); diff --git a/ui/components/ui/Input/Input.tsx b/ui/components/ui/Input/Input.tsx index 3543ddb56..d591b4992 100644 --- a/ui/components/ui/Input/Input.tsx +++ b/ui/components/ui/Input/Input.tsx @@ -42,9 +42,18 @@ const Input: React.FC<InputProps> = ({ value, placeholder, isError, iconSvg, cle } }, [inputRef.current]); + // `disabled` stops typing into the native field and NOTHING else. The addons and the clear button + // below are plain divs with an `onClick`, so a read-only field kept two working ways to edit the + // value it was only supposed to be showing: an addon changes what the value MEANS (the regex + // `m`/`i` flags, match-case), and clear empties it outright. + // + // The addons stay on screen, because which flags are set is part of what is being read; the clear + // button does not, because it is nothing but a mutation. + const isClearable = clearable && !isReadOnly; + let paddingRightRem = 12; addons?.forEach(() => paddingRightRem += 24); - if (clearable) { + if (isClearable) { paddingRightRem += 24 } @@ -66,7 +75,7 @@ const Input: React.FC<InputProps> = ({ value, placeholder, isError, iconSvg, cle ${s.InputInput} ${isError ? s.InputInputWithError : ''} ${iconSvg ? s.InputInputWithIcon : ''} - ${clearable ? s.InputInputClearable : ''} + ${isClearable ? s.InputInputClearable : ''} `} type={type || 'text'} value={value} @@ -78,8 +87,12 @@ const Input: React.FC<InputProps> = ({ value, placeholder, isError, iconSvg, cle inputRef?.current?.blur(); } }} - disabled={inputProps?.disabled || isReadOnly} {...inputProps} + // AFTER the spread, deliberately: callers forward native attributes through `inputProps`, + // and one that mentions `disabled` at all - including the `disabled: props.disabled` that + // evaluates to `undefined` - used to overwrite the computed value and hand a read-only + // field back to the user. + disabled={inputProps?.disabled || isReadOnly} data-testid={testId} /> {iconSvg && (<div className={s.InputIcon}> @@ -92,8 +105,8 @@ const Input: React.FC<InputProps> = ({ value, placeholder, isError, iconSvg, cle return ( <div key={addon.id} - onClick={addon.onClick} - className={`${s.Addon} ${addon.isEnabled ? '' : s.AddonDisabled}`} + onClick={isReadOnly ? undefined : addon.onClick} + className={`${s.Addon} ${addon.isEnabled ? '' : s.AddonDisabled} ${isReadOnly ? s.AddonReadOnly : ''}`} data-tooltip-id={addon.help ? tooltipId : undefined} data-tooltip-html={addon.help ? renderToStaticMarkup(<>{addon.help}</>) : undefined} > @@ -104,7 +117,7 @@ const Input: React.FC<InputProps> = ({ value, placeholder, isError, iconSvg, cle })} </div> )} - {clearable && ( + {isClearable && ( <div className={s.Clear} onClick={() => onChange('')}> <SvgIcon svg={clearIcon} /> </div> diff --git a/ui/components/ui/Input/StringFilterInput/StringFilterInput.test.tsx b/ui/components/ui/Input/StringFilterInput/StringFilterInput.test.tsx new file mode 100644 index 000000000..c1cb0aef8 --- /dev/null +++ b/ui/components/ui/Input/StringFilterInput/StringFilterInput.test.tsx @@ -0,0 +1,57 @@ +/** + * @jest-environment jsdom + * + * Match-case is an addon on the shared Input, and an addon is a plain div with an `onClick` - so + * `disabled` on the field does nothing for it. In a read-only (library-owned) configuration, a + * click on it used to flip the stored item's matching behaviour: same text, different matches. + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import StringFilterInput from './StringFilterInput'; + +const renderFilter = (isReadOnly: boolean) => { + const onIsMatchCaseChange = jest.fn(); + const onChange = jest.fn(); + const { container } = render( + <StringFilterInput + testId="the-filter" + value="abc" + onChange={onChange} + isMatchCase={false} + onIsMatchCaseChange={onIsMatchCaseChange} + isReadOnly={isReadOnly} + /> + ); + + // The match-case addon carries an icon rather than a label, and both the CSS module class names + // and the icon itself are stubbed under jest - so it is identified by being a clickable thing in + // this field rather than by a selector that jest cannot see. Clicking everything also answers the + // stronger question: is there ANY affordance here that mutates? + const clickEverything = () => Array.from(container.querySelectorAll('div')).forEach((el) => fireEvent.click(el)); + + return { onIsMatchCaseChange, onChange, clickEverything }; +}; + +describe('match case in a read-only string filter', () => { + it('cannot be toggled', () => { + const { onIsMatchCaseChange, clickEverything } = renderFilter(true); + + clickEverything(); + + expect(onIsMatchCaseChange).not.toHaveBeenCalled(); + }); + + it('can be toggled when the filter is editable', () => { + const { onIsMatchCaseChange, clickEverything } = renderFilter(false); + + clickEverything(); + + expect(onIsMatchCaseChange).toHaveBeenCalledWith(true); + }); + + it('leaves the text field unusable too', () => { + renderFilter(true); + + expect((screen.getByTestId('the-filter') as HTMLInputElement).disabled).toBe(true); + }); +}); diff --git a/ui/components/ui/LibraryBrowser/default-library-items.ts b/ui/components/ui/LibraryBrowser/default-library-items.ts index 0fd459d19..d07ef43af 100644 --- a/ui/components/ui/LibraryBrowser/default-library-items.ts +++ b/ui/components/ui/LibraryBrowser/default-library-items.ts @@ -54,7 +54,8 @@ export function getDefaultManagedItem(itemType: ManagedItemType, libraryContext: type: "value", val: getDefaultManagedItem("value-projection-list", libraryContext) as ManagedValueProjectionList }, - numDisplayItems: undefined + numDisplayItems: undefined, + messageDeliveryOrder: 'guaranteed' } } @@ -150,11 +151,16 @@ export function getDefaultManagedItem(itemType: ManagedItemType, libraryContext: return v; } case "consumer-session-start-from": { + // Earliest by default: a fresh session shows the topic's existing data on Play, instead of + // sitting silently until new traffic happens to arrive. A non-persistent topic retains + // nothing and the server refuses every history-based position there, so it keeps Latest. + const isLiveOnlyTopic = libraryContext.pulsarResource.type === 'topic' + && libraryContext.pulsarResource.topicPersistency === 'non-persistent'; const v: ManagedConsumerSessionStartFrom = { metadata, spec: { startFrom: { - type: "latestMessage" + type: isLiveOnlyTopic ? "latestMessage" : "earliestMessage" } } }; diff --git a/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.test.tsx b/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.test.tsx new file mode 100644 index 000000000..799680e3b --- /dev/null +++ b/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.test.tsx @@ -0,0 +1,156 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The overwrite flow shares SaveItemDialog's failure mode: `libraryItemToPb` runs synchronously in + * `saveItem`, outside any catch, so an item carrying an unparseable message-id hex string used to + * escape the async click handler as an unhandled rejection - Overwrite clicked, nothing sent, + * nothing said. The guard must turn that into a notification and send nothing. + * + * The transport and the notifier are replaced; the dialog, its SearchResults (which lists nothing + * here - no search contexts) and the selected-item fetch are real. The item editor is not the + * wiring under test. + */ +const mockClients = { current: undefined as unknown }; +jest.mock('../../../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +const mockNotifications = { current: undefined as unknown }; +jest.mock('../../../../app/contexts/Notifications', () => ({ + useContext: () => mockNotifications.current, +})); + +jest.mock('../../LibraryItemEditor/LibraryItemEditor', () => ({ + __esModule: true, + default: () => <div data-testid="mock-library-item-editor" />, +})); + +import React from 'react'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import * as Modals from '../../../../app/contexts/Modals/Modals'; +import OverwriteExistingItemDialog from './OverwriteExistingItemDialog'; +import { libraryItemToPb } from '../../model/library-conversions'; +import * as pb from '../../../../../grpc-web/tools/teal/pulsar/ui/library/v1/library_pb'; +import { Status } from '../../../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../../../grpc-web/google/rpc/code_pb'; + +const okStatus = () => { + const s = new Status(); + s.setCode(Code.OK); + s.setMessage(''); + return s; +}; + +/** A start-from item whose message id carries `hexString`. */ +const startFromLibraryItem = (id: string, name: string, hexString: string) => ({ + metadata: { updatedAt: '', availableForContexts: [] }, + spec: { + metadata: { id, name, descriptionMarkdown: '', type: 'consumer-session-start-from' }, + spec: { + startFrom: { + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: `${id}-mid`, name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString }, + }, + }, + }, + }, + }, +}); + +const makeHarness = () => { + const saveRequests: any[] = []; + const notifyError = jest.fn(); + const notifySuccess = jest.fn(); + + // The item already in the library - what the dialog fetches and offers to overwrite. It has to + // be valid: it round-trips through fromPb on arrival. + const existingItem = startFromLibraryItem('existing-1', 'existing', '08 c3'); + + mockClients.current = { + libraryServiceClient: { + getLibraryItem: () => { + const res = new pb.GetLibraryItemResponse(); + res.setStatus(okStatus()); + res.setItem(libraryItemToPb(existingItem as never)); + return Promise.resolve(res); + }, + saveLibraryItem: (req: unknown) => { + saveRequests.push(req); + const res = new pb.SaveLibraryItemResponse(); + res.setStatus(okStatus()); + return Promise.resolve(res); + }, + listLibraryItems: () => Promise.reject(new Error('no search contexts in these tests')), + }, + }; + + mockNotifications.current = { + notifySuccess, + notifyInfo: jest.fn(), + notifyWarn: jest.fn(), + notifyError, + }; + + return { saveRequests, notifyError, notifySuccess }; +}; + +const overwriteButton = () => screen.getByTestId('lib-overwrite-confirm') as HTMLButtonElement; + +const renderDialog = async (libraryItem: unknown, onSaved = jest.fn()) => { + await act(async () => { + render( + <MemoryRouter> + <Modals.DefaultProvider> + <OverwriteExistingItemDialog + libraryItem={libraryItem as never} + libraryContext={{} as never} + onCanceled={jest.fn()} + onSaved={onSaved} + itemIdToOverwrite="existing-1" + /> + </Modals.DefaultProvider> + </MemoryRouter> + ); + }); + + // Overwrite stays disabled until the selected item's fetch lands. + await waitFor(() => expect(overwriteButton().disabled).toBe(false)); + return { onSaved }; +}; + +const clickOverwrite = async () => { + await act(async () => { + fireEvent.click(overwriteButton()); + }); +}; + +describe('overwriting with an item the pb conversion refuses', () => { + it('surfaces the parse error instead of wedging silently', async () => { + const harness = makeHarness(); + const { onSaved } = await renderDialog(startFromLibraryItem('draft-1', 'a draft', 'zz')); + + await clickOverwrite(); + + expect(harness.notifyError).toHaveBeenCalledTimes(1); + expect(String(harness.notifyError.mock.calls[0][0])).toContain('Invalid hex string'); + expect(harness.saveRequests).toHaveLength(0); + expect(onSaved).not.toHaveBeenCalled(); + }); + + it('still overwrites with an item whose hex actually parses', async () => { + const harness = makeHarness(); + const { onSaved } = await renderDialog(startFromLibraryItem('draft-1', 'a draft', '08 c3 03')); + + await clickOverwrite(); + + expect(harness.saveRequests).toHaveLength(1); + expect(harness.notifyError).not.toHaveBeenCalled(); + expect(onSaved).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.tsx b/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.tsx index 408315761..a28ee712e 100644 --- a/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.tsx +++ b/ui/components/ui/LibraryBrowser/dialogs/OverwriteExistingItemDialog/OverwriteExistingItemDialog.tsx @@ -126,8 +126,18 @@ const OverwriteExistingItemDialog: React.FC<OverwriteExistingItemDialogProps> = const itemToSave = cloneDeep(libraryItem); itemToSave.spec.metadata.id = selectedItem.spec.metadata.id; + // Same guard as SaveItemDialog: the pb conversion parses free-text fields (a message id's hex + // string) and throws synchronously on invalid input. Uncaught, that is only an unhandled + // rejection out of this async click handler and the dialog wedges with no feedback. + let itemPb: pb.LibraryItem; + try { + itemPb = libraryItemToPb(itemToSave); + } catch (err) { + notifyError(`Unable to save library item. ${err}`); + return; + } + const req = new pb.SaveLibraryItemRequest(); - const itemPb = libraryItemToPb(itemToSave); req.setItem(itemPb); const res = await libraryServiceClient.saveLibraryItem(req, null).catch(err => { diff --git a/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.test.tsx b/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.test.tsx new file mode 100644 index 000000000..40105b80f --- /dev/null +++ b/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.test.tsx @@ -0,0 +1,192 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * Saving an item the pb conversion cannot serialise - e.g. a message-id hex string the start-from + * editor flags inline but still commits - used to throw SYNCHRONOUSLY out of `saveItem`, outside + * any catch. Out of an async click handler that is only an unhandled rejection: no notification, + * no request, and a Save button that appears to do nothing at all. The dialog must surface the + * parse error instead, and stay usable for the next attempt. + * + * The gRPC transport and the notifier are replaced; the dialog itself is real. The item editor and + * the matchers input are not the wiring under test - the save path reads the item from state, never + * back through either input. + */ +const mockClients = { current: undefined as unknown }; +jest.mock('../../../../app/contexts/GrpcClient/GrpcClient', () => ({ + useContext: () => mockClients.current, +})); + +const mockNotifications = { current: undefined as unknown }; +jest.mock('../../../../app/contexts/Notifications', () => ({ + useContext: () => mockNotifications.current, +})); + +jest.mock('../../LibraryItemEditor/LibraryItemEditor', () => ({ + __esModule: true, + default: () => <div data-testid="mock-library-item-editor" />, +})); +jest.mock('../../SearchEditor/ResourceMatchersInput/ResourceMatchersInput', () => ({ + __esModule: true, + default: () => <div data-testid="mock-resource-matchers-input" />, +})); + +import React from 'react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import * as Modals from '../../../../app/contexts/Modals/Modals'; +import SaveItemDialog from './SaveItemDialog'; +import * as pb from '../../../../../grpc-web/tools/teal/pulsar/ui/library/v1/library_pb'; +import { Status } from '../../../../../grpc-web/google/rpc/status_pb'; +import { Code } from '../../../../../grpc-web/google/rpc/code_pb'; + +const okStatus = () => { + const s = new Status(); + s.setCode(Code.OK); + s.setMessage(''); + return s; +}; + +const makeHarness = () => { + const saveRequests: any[] = []; + const notifyError = jest.fn(); + const notifySuccess = jest.fn(); + + mockClients.current = { + libraryServiceClient: { + saveLibraryItem: (req: unknown) => { + saveRequests.push(req); + const res = new pb.SaveLibraryItemResponse(); + res.setStatus(okStatus()); + return Promise.resolve(res); + }, + }, + }; + + mockNotifications.current = { + notifySuccess, + notifyInfo: jest.fn(), + notifyWarn: jest.fn(), + notifyError, + }; + + return { saveRequests, notifyError, notifySuccess }; +}; + +/** A start-from item whose message id carries `hexString` - the editor commits it as typed. */ +const startFromLibraryItem = (hexString: string) => ({ + metadata: { updatedAt: '', availableForContexts: [] }, + spec: { + metadata: { + id: 'start-from-1', + name: 'a start from', + descriptionMarkdown: '', + type: 'consumer-session-start-from', + }, + spec: { + startFrom: { + type: 'messageId', + messageId: { + type: 'value', + val: { + metadata: { id: 'mid-1', name: '', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString }, + }, + }, + }, + }, + }, +}); + +/** The same hex string saved as a standalone message-id item - the other conversion entry point. */ +const messageIdLibraryItem = (hexString: string) => ({ + metadata: { updatedAt: '', availableForContexts: [] }, + spec: { + metadata: { id: 'mid-item-1', name: 'a message id', descriptionMarkdown: '', type: 'message-id' }, + spec: { hexString }, + }, +}); + +const renderDialog = (libraryItem: unknown, onSaved = jest.fn()) => { + render( + <MemoryRouter> + <Modals.DefaultProvider> + <SaveItemDialog + libraryItem={libraryItem as never} + isExistingItem={false} + libraryContext={{} as never} + onCanceled={jest.fn()} + onSaved={onSaved} + /> + </Modals.DefaultProvider> + </MemoryRouter> + ); + return { onSaved }; +}; + +const clickSave = async () => { + await act(async () => { + fireEvent.click(screen.getByTestId('lib-save-dialog-save')); + }); +}; + +describe('saving an item the pb conversion refuses', () => { + it.each([ + ['a start-from wrapping a message id', startFromLibraryItem('zz')], + ['a standalone message-id item', messageIdLibraryItem('zz')], + ])('surfaces the parse error for %s instead of wedging silently', async (_name, libraryItem) => { + const harness = makeHarness(); + const { onSaved } = renderDialog(libraryItem); + + await clickSave(); + + expect(harness.notifyError).toHaveBeenCalledTimes(1); + expect(String(harness.notifyError.mock.calls[0][0])).toContain('Invalid hex string'); + // Nothing left the dialog: no request was built, so none was sent, and nothing was "saved". + expect(harness.saveRequests).toHaveLength(0); + expect(onSaved).not.toHaveBeenCalled(); + }); + + it.each([ + ['a BLANK start-from message id', startFromLibraryItem('')], + ['a BLANK standalone message-id item', messageIdLibraryItem(' ')], + ])('refuses to persist %s - a value that cannot start a session must not be storable', async (_name, libraryItem) => { + // The shared parser maps blank text to a ZERO-LENGTH byte array, so a freshly selected + // Message-ID mode used to save fine and then fail every later Play of the item. + const harness = makeHarness(); + const { onSaved } = renderDialog(libraryItem); + + await clickSave(); + + expect(harness.notifyError).toHaveBeenCalledTimes(1); + expect(String(harness.notifyError.mock.calls[0][0])).toContain('Enter the message id'); + expect(harness.saveRequests).toHaveLength(0); + expect(onSaved).not.toHaveBeenCalled(); + }); + + it('leaves the dialog usable after the refusal - the next click still answers', async () => { + // The wedge was the symptom: pre-guard, the first click died as an unhandled rejection and so + // did every following one. Two clicks, two notifications, is the observable difference. + const harness = makeHarness(); + renderDialog(startFromLibraryItem('zz')); + + await clickSave(); + await clickSave(); + + expect(harness.notifyError).toHaveBeenCalledTimes(2); + expect(harness.saveRequests).toHaveLength(0); + }); + + it('still saves an item whose hex actually parses', async () => { + // The counterpart: "never throw out of Save" is trivially satisfiable by never saving. + const harness = makeHarness(); + const { onSaved } = renderDialog(startFromLibraryItem('08 c3 03 10 cd 04 20 00 30 01')); + + await clickSave(); + + expect(harness.saveRequests).toHaveLength(1); + expect(harness.notifyError).not.toHaveBeenCalled(); + expect(harness.notifySuccess).toHaveBeenCalledTimes(1); + expect(onSaved).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.tsx b/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.tsx index de83d4b17..b52c28693 100644 --- a/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.tsx +++ b/ui/components/ui/LibraryBrowser/dialogs/SaveItemDialog/SaveItemDialog.tsx @@ -44,8 +44,19 @@ const SaveItemDialog: React.FC<SaveItemDialogProps> = (props) => { return; } + // The pb conversion parses free-text fields (a message id's hex string) and throws on input the + // inline editors flag but do not block. It runs synchronously inside this async click handler, + // so without the guard the throw is only an unhandled rejection - the dialog would just sit + // there, saving nothing and saying nothing. + let itemPb: pb.LibraryItem; + try { + itemPb = libraryItemToPb(libraryItem); + } catch (err) { + notifyError(`Unable to save library item. ${err}`); + return; + } + const req = new pb.SaveLibraryItemRequest(); - const itemPb = libraryItemToPb(libraryItem); req.setItem(itemPb); const res = await libraryServiceClient.saveLibraryItem(req, null).catch(err => { diff --git a/ui/components/ui/LibraryBrowser/model/resolved-items-conversions.ts b/ui/components/ui/LibraryBrowser/model/resolved-items-conversions.ts index 6cc3b9fea..8d19952ad 100644 --- a/ui/components/ui/LibraryBrowser/model/resolved-items-conversions.ts +++ b/ui/components/ui/LibraryBrowser/model/resolved-items-conversions.ts @@ -4,7 +4,7 @@ import { TopicSelector } from "../../ConsumerSession/topic-selector/topic-select import { BasicMessageFilterTarget } from "../../ConsumerSession/basic-message-filter-types"; import { ValueProjection, ValueProjectionList } from "../../ConsumerSession/value-projections/value-projections"; import { Deserializer } from "../../ConsumerSession/deserializer/deserializer"; -import { defaultNumDisplayItems } from "../../ConsumerSession/SessionConfiguration/SessionConfiguration"; +import { displayItemLimit } from "../../ConsumerSession/SessionConfiguration/display-items"; export function messageFilterFromValOrRef(v: ManagedMessageFilterValOrRef): MessageFilter { if (v.val === undefined) { @@ -103,6 +103,8 @@ export function consumerSessionStartFromFromValOrRef(v: ManagedConsumerSessionSt case 'latestMessage': return { type: 'latestMessage' }; case 'nthMessageAfterEarliest': return { type: 'nthMessageAfterEarliest', n: spec.startFrom.n }; case 'nthMessageBeforeLatest': return { type: 'nthMessageBeforeLatest', n: spec.startFrom.n }; + case 'approximateEntryPosition': return { type: 'approximateEntryPosition', fraction: spec.startFrom.fraction }; + case 'approximatePublishTimePosition': return { type: 'approximatePublishTimePosition', fraction: spec.startFrom.fraction }; case 'dateTime': return { type: 'dateTime', dateTime: dateTimeFromValOrRef(spec.startFrom.dateTime) @@ -223,7 +225,16 @@ export function consumerSessionConfigFromValOrRef(v: ManagedConsumerSessionConfi coloringRuleChain: coloringRuleChainFromValOrRef(spec.coloringRuleChain), pauseTriggerChain: consumerSessionPauseTriggerChainFromValOrRef(spec.pauseTriggerChain), valueProjectionList: valueProjectionListFromValOrRef(spec.valueProjectionList), - numDisplayItems: spec.numDisplayItems === undefined ? defaultNumDisplayItems : spec.numDisplayItems + // A persisted spec is JSON on disk: written by an older build, hand-edited, or committed by a + // field that did not validate. A limit of zero (or a negative, or a fraction) turns the session's + // `slice(-limit)` retention into "keep everything", so it is not passed through as given. + numDisplayItems: displayItemLimit(spec.numDisplayItems), + // A spec saved before the field existed names no order. It runs under the product default - + // Guaranteed (owner decision 2026-08-11) - which is exactly what the server does with proto3 + // absence, so the session the browser describes and the session the server builds cannot + // disagree. + messageDeliveryOrder: spec.messageDeliveryOrder ?? 'guaranteed', + deliveryOrderKey: spec.deliveryOrderKey } } diff --git a/ui/components/ui/LibraryBrowser/model/start-from-approximate-positions.spec.ts b/ui/components/ui/LibraryBrowser/model/start-from-approximate-positions.spec.ts new file mode 100644 index 000000000..956a7543b --- /dev/null +++ b/ui/components/ui/LibraryBrowser/model/start-from-approximate-positions.spec.ts @@ -0,0 +1,137 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * The entry-position and publish-time-position modes each have to survive four separate hand-written + * mappings: the library spec <-> protobuf + * pair (persisted saved sessions), the managed-item -> runtime conversion, and the runtime -> + * protobuf conversion that actually reaches the server. Each one is a `switch` over the mode union; + * a missing branch there is either a thrown "Unknown ..." at runtime or a setting that is silently + * dropped on the way to Pulsar, so all four are pinned here for both modes. + * + * THE TRAP THIS FILE EXISTS FOR, now that there are two of them: their payloads are identical - one + * double - so a branch that reaches for the other mode's oneof case still produces a valid-looking + * fraction. Every assertion below therefore names the WIRE FIELD as well as the value; a test that + * only checked the fraction would pass while the server positioned the session by the wrong rule. + * + * jsdom + the mocks below: these conversions sit in a module graph that reaches components, and from + * there the ESM-only mermaid/nanoid, which jest does not transform. Neither is exercised. + */ +jest.mock('mermaid', () => ({ __esModule: true, default: { initialize: () => undefined } })); +jest.mock('nanoid', () => ({ nanoid: () => 'test-id' })); + +import * as pb from '../../../../grpc-web/tools/teal/pulsar/ui/library/v1/managed_items_pb'; +import * as consumerPb from '../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'; +import { + managedConsumerSessionStartFromSpecFromPb, + managedConsumerSessionStartFromSpecToPb, +} from './user-managed-items-conversions-pb'; +import { consumerSessionStartFromFromValOrRef } from './resolved-items-conversions'; +import { startFromFromPb, startFromToPb } from '../../ConsumerSession/conversions/conversions'; + +type Mode = 'approximateEntryPosition' | 'approximatePublishTimePosition'; + +const managedSpec = (type: Mode, fraction: number) => ({ startFrom: { type, fraction } as const }); + +// NOTE: nothing in this file may put an imported binding in a TYPE position - esbuild-jest runs it +// through babel, which refuses that outright ("Cannot transform the imported binding ... since it's +// also used in a type annotation"). Hence the plain value tables below and the inline getters, in +// place of the helper functions that would otherwise need the pb message types by name. + +/** The library oneof case each mode must occupy. */ +const managedExpectedCase = { + approximateEntryPosition: pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_APPROXIMATE_ENTRY_POSITION, + approximatePublishTimePosition: pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_APPROXIMATE_PUBLISH_TIME_POSITION, +} satisfies Record<Mode, unknown>; + +/** The same, for the runtime start-from that reaches the server. */ +const runtimeExpectedCase = { + approximateEntryPosition: consumerPb.ConsumerSessionStartFrom.StartFromCase.START_FROM_APPROXIMATE_ENTRY_POSITION, + approximatePublishTimePosition: consumerPb.ConsumerSessionStartFrom.StartFromCase.START_FROM_APPROXIMATE_PUBLISH_TIME_POSITION, +} satisfies Record<Mode, unknown>; + +const modes: Mode[] = ['approximateEntryPosition', 'approximatePublishTimePosition']; + +describe.each(modes)('%s: library spec <-> protobuf', (mode) => { + it('round-trips through the persisted library representation, in its OWN oneof case', () => { + const specPb = managedConsumerSessionStartFromSpecToPb(managedSpec(mode, 0.6)); + + // The oneof must actually be set to this mode's case - an unset oneof would serialize to nothing + // and read back as "Unknown ManagedConsumerSessionStartFromSpec", and the OTHER mode's case + // would read back as the wrong mode with the right number. + expect(specPb.getStartFromCase()).toBe(managedExpectedCase[mode]); + const fraction = mode === 'approximateEntryPosition' + ? specPb.getStartFromApproximateEntryPosition()?.getFraction() + : specPb.getStartFromApproximatePublishTimePosition()?.getFraction(); + expect(fraction).toBe(0.6); + + expect(managedConsumerSessionStartFromSpecFromPb(specPb)).toEqual(managedSpec(mode, 0.6)); + }); + + it('round-trips through a real serialize/deserialize cycle, like a saved session does', () => { + const bytes = managedConsumerSessionStartFromSpecToPb(managedSpec(mode, 0.125)).serializeBinary(); + const decoded = pb.ManagedConsumerSessionStartFromSpec.deserializeBinary(bytes); + + expect(managedConsumerSessionStartFromSpecFromPb(decoded)).toEqual(managedSpec(mode, 0.125)); + }); + + it('keeps the boundary values distinguishable from an unset field', () => { + // 0.0 is the earliest retained message and is a legal, meaningful value - but it is also the + // protobuf default for a double, so it only survives because the WRAPPER message is set. + const zeroPb = managedConsumerSessionStartFromSpecToPb(managedSpec(mode, 0)); + const zeroDecoded = pb.ManagedConsumerSessionStartFromSpec.deserializeBinary(zeroPb.serializeBinary()); + expect(managedConsumerSessionStartFromSpecFromPb(zeroDecoded)).toEqual(managedSpec(mode, 0)); + + const onePb = managedConsumerSessionStartFromSpecToPb(managedSpec(mode, 1)); + expect(managedConsumerSessionStartFromSpecFromPb(onePb)).toEqual(managedSpec(mode, 1)); + }); +}); + +describe.each(modes)('%s: managed item -> runtime start-from', (mode) => { + it('resolves to the runtime mode instead of falling off the switch', () => { + const valOrRef = { + type: 'value' as const, + val: { + metadata: { id: 'x', name: '', descriptionMarkdown: '', type: 'consumer-session-start-from' as const }, + spec: managedSpec(mode, 0.42), + }, + }; + + expect(consumerSessionStartFromFromValOrRef(valOrRef)).toEqual({ type: mode, fraction: 0.42 }); + }); +}); + +describe.each(modes)('%s: runtime start-from <-> protobuf', (mode) => { + it('reaches the server in its own oneof case', () => { + const startFromPb = startFromToPb({ type: mode, fraction: 0.6 }); + + expect(startFromPb.getStartFromCase()).toBe(runtimeExpectedCase[mode]); + const fraction = mode === 'approximateEntryPosition' + ? startFromPb.getStartFromApproximateEntryPosition()?.getFraction() + : startFromPb.getStartFromApproximatePublishTimePosition()?.getFraction(); + expect(fraction).toBe(0.6); + }); + + it('reads back from the wire', () => { + const bytes = startFromToPb({ type: mode, fraction: 0.75 }).serializeBinary(); + const decoded = consumerPb.ConsumerSessionStartFrom.deserializeBinary(bytes); + + expect(startFromFromPb(decoded)).toEqual({ type: mode, fraction: 0.75 }); + }); +}); + +describe('the two modes are not interchangeable', () => { + it('sets a different wire field for each, at both layers', () => { + // Stated as an inequality as well, so that a future edit which points both mappings at one field + // fails here even if each mode's own round trip is self-consistent. + expect(runtimeExpectedCase.approximateEntryPosition).not.toBe(runtimeExpectedCase.approximatePublishTimePosition); + expect(managedExpectedCase.approximateEntryPosition).not.toBe(managedExpectedCase.approximatePublishTimePosition); + + const entryPb = startFromToPb({ type: 'approximateEntryPosition', fraction: 0.6 }); + const publishTimePb = startFromToPb({ type: 'approximatePublishTimePosition', fraction: 0.6 }); + expect(entryPb.getStartFromCase()).not.toBe(publishTimePb.getStartFromCase()); + // ...and neither one leaves a value in the other's field. + expect(entryPb.getStartFromApproximatePublishTimePosition()).toBeUndefined(); + expect(publishTimePb.getStartFromApproximateEntryPosition()).toBeUndefined(); + }); +}); diff --git a/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.spec.ts b/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.spec.ts new file mode 100644 index 000000000..fa4049d43 --- /dev/null +++ b/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.spec.ts @@ -0,0 +1,156 @@ +import { managedDateTimeSpecFromPb, managedDateTimeSpecToPb } from "./user-managed-items-conversions-pb"; + +/** + * A saved DateTime and the session request built from it must agree. The picker is second-granular, + * but a Date still carries millis, and `startFromToPb`'s dateTime case floors to whole epoch seconds + * (`Math.floor(getTime() / 1000)`). If the at-rest value keeps those millis - as `Timestamp.fromDate` + * would, stashing them in `nanos` - then loading a saved item and starting a session lands earlier + * than the value on disk, and save-then-load is not a fixed point. + */ +describe("a saved managed DateTime is stored at whole-second granularity", () => { + it("drops the sub-second part the session request would drop anyway", () => { + const withMillis = new Date(1_700_000_000_123); // 123 ms past the whole second + const loaded = managedDateTimeSpecFromPb(managedDateTimeSpecToPb({ dateTime: withMillis })); + + // Equal to what startFromToPb would send for the same Date - the whole second, no millis. + expect(loaded.dateTime.getTime()).toBe(1_700_000_000_000); + expect(loaded.dateTime.getMilliseconds()).toBe(0); + }); + + it("writes no nanos, so the timestamp is second-exact on the wire", () => { + const savedPb = managedDateTimeSpecToPb({ dateTime: new Date(1_700_000_000_999) }); + + expect(savedPb.getDateTime()!.getSeconds()).toBe(1_700_000_000); + // `Timestamp.fromDate` would stash the 999 ms here as 999_000_000 nanos - the richer at-rest + // value the session then throws away. + expect(savedPb.getDateTime()!.getNanos()).toBe(0); + }); +}); + +/** + * The delivery-order choice must survive save -> load through the MANAGED-LIBRARY protobuf, not + * just the runtime session request. It used to exist only in the TypeScript spec type, so Play + * honoured it while a saved and reopened session silently fell back to independent - the setting + * looked sticky right up until the page reloaded. + */ +describe("a saved consumer-session config keeps its delivery order", () => { + const { managedConsumerSessionConfigSpecFromPb, managedConsumerSessionConfigSpecToPb } = + require("./user-managed-items-conversions-pb"); + const { getDefaultManagedItem } = require("../default-library-items"); + + const libraryContext = { + pulsarResource: { + type: 'topic', + tenant: 'public', + namespace: 'default', + topicPersistency: 'persistent', + topic: 'a-topic', + }, + }; + + const defaultSpec = () => getDefaultManagedItem('consumer-session-config', libraryContext).spec; + + // Owner decision (2026-08-11, direct instruction): the default is Guaranteed - the third move + // of this default; the plan file's decision log is the record. + it("writes the new managed-item default as Guaranteed order", () => { + const saved = managedConsumerSessionConfigSpecToPb(defaultSpec()); + const loaded = managedConsumerSessionConfigSpecFromPb(saved); + expect(saved.getMessageDeliveryOrder()).toBe( + require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb') + .MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ); + expect(loaded.messageDeliveryOrder).toBe('guaranteed'); + }); + + it("normalizes an older UNSPECIFIED field to the Guaranteed default", () => { + const consumerPb = require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'); + const saved = managedConsumerSessionConfigSpecToPb(defaultSpec()); + // Set the protobuf field itself to zero after building an otherwise-valid object. Going + // through our writer here would canonicalize a missing JSON value to explicit BEST_EFFORT and + // would not exercise the legacy wire value at all. + saved.setMessageDeliveryOrder( + consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED + ); + const loaded = managedConsumerSessionConfigSpecFromPb(saved); + expect(saved.getMessageDeliveryOrder()).toBe( + consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED + ); + expect(loaded.messageDeliveryOrder).toBe('guaranteed'); + }); + + it("loads an item SERIALIZED before the field existed - field absent in the bytes - as Guaranteed, the default", () => { + // The compatibility case that actually ships: BYTES written by a build with no field 8 at + // all, not an in-memory object whose enum happens to read zero. Proto3 omits a zero enum, so + // round-tripping through serializeBinary produces exactly those pre-field bytes. + const jspb = require('google-protobuf'); + const managedPb = require('../../../../grpc-web/tools/teal/pulsar/ui/library/v1/managed_items_pb'); + const consumerPb = require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb'); + + const written = managedConsumerSessionConfigSpecToPb(defaultSpec()); + written.setMessageDeliveryOrder(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED); + const preFieldBytes = written.serializeBinary(); + + // Read the tags back off the bytes: "field 8 is absent" has to be a statement about the + // ENCODING, not about a getter that happily synthesizes the proto3 zero. + const reader = new jspb.BinaryReader(preFieldBytes); + const encodedFields: number[] = []; + while (reader.nextField() && !reader.isEndGroup()) { + encodedFields.push(reader.getFieldNumber()); + reader.skipField(); + } + expect(encodedFields.length).toBeGreaterThan(0); + expect(encodedFields).not.toContain(8); + + const parsed = managedPb.ManagedConsumerSessionConfigSpec.deserializeBinary(preFieldBytes); + expect(parsed.getMessageDeliveryOrder()) + .toBe(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_UNSPECIFIED); + expect(managedConsumerSessionConfigSpecFromPb(parsed).messageDeliveryOrder).toBe('guaranteed'); + }); + + it("writes a missing older JSON value as the explicit Guaranteed default", () => { + const legacy = { ...defaultSpec(), messageDeliveryOrder: undefined }; + const saved = managedConsumerSessionConfigSpecToPb(legacy); + expect(saved.getMessageDeliveryOrder()).toBe( + require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb') + .MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ); + }); + + it("an explicit Best effort choice round-trips - it now coincides with the default, and stays explicit", () => { + const spec = { ...defaultSpec(), messageDeliveryOrder: 'best-effort' }; + const saved = managedConsumerSessionConfigSpecToPb(spec); + expect(saved.getMessageDeliveryOrder()).toBe( + require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb') + .MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + ); + expect(managedConsumerSessionConfigSpecFromPb(saved).messageDeliveryOrder).toBe('best-effort'); + }); + + it("an explicit Fastest choice round-trips and stays distinct from the default", () => { + const spec = { ...defaultSpec(), messageDeliveryOrder: 'as-received' }; + const loaded = managedConsumerSessionConfigSpecFromPb(managedConsumerSessionConfigSpecToPb(spec)); + expect(loaded.messageDeliveryOrder).toBe('as-received'); + }); + + it("an explicit Guaranteed choice round-trips and is never replaced by the best-effort default", () => { + const spec = { ...defaultSpec(), messageDeliveryOrder: 'guaranteed' }; + const saved = managedConsumerSessionConfigSpecToPb(spec); + expect(saved.getMessageDeliveryOrder()).toBe( + require('../../../../grpc-web/tools/teal/pulsar/ui/api/v1/consumer_pb') + .MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED + ); + expect(managedConsumerSessionConfigSpecFromPb(saved).messageDeliveryOrder).toBe('guaranteed'); + }); + + it("the selected order time round-trips, and its absence stays absent", () => { + const bare = managedConsumerSessionConfigSpecFromPb(managedConsumerSessionConfigSpecToPb(defaultSpec())); + expect(bare.deliveryOrderKey).toBeUndefined(); + + const withKey = { ...defaultSpec(), messageDeliveryOrder: 'guaranteed', deliveryOrderKey: 'broker-publish-time' }; + const loaded = managedConsumerSessionConfigSpecFromPb(managedConsumerSessionConfigSpecToPb(withKey)); + expect(loaded.deliveryOrderKey).toBe('broker-publish-time'); + + const eventKey = { ...defaultSpec(), messageDeliveryOrder: 'best-effort', deliveryOrderKey: 'event-time' }; + expect(managedConsumerSessionConfigSpecFromPb(managedConsumerSessionConfigSpecToPb(eventKey)).deliveryOrderKey).toBe('event-time'); + }); +}); diff --git a/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.ts b/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.ts index 52c869ff6..4d29c8658 100644 --- a/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.ts +++ b/ui/components/ui/LibraryBrowser/model/user-managed-items-conversions-pb.ts @@ -33,6 +33,7 @@ import { } from "../../ConsumerSession/conversions/conversions"; import { Timestamp } from "google-protobuf/google/protobuf/timestamp_pb"; import { hexStringFromByteArray, hexStringToByteArray } from "../../../conversions/conversions"; +import { messageIdError } from "../../ConsumerSession/SessionConfiguration/StartFromInput/message-id"; import { basicMessageFilterFromPb, basicMessageFilterTargetFromPb, basicMessageFilterTargetToPb, basicMessageFilterToPb } from "../../ConsumerSession/conversions/basic-message-filter-conversions"; import { Int32Value, Int64Value } from "google-protobuf/google/protobuf/wrappers_pb"; import { deserializerFromPb, deserializerToPb } from "../../ConsumerSession/deserializer/deserializer"; @@ -109,6 +110,15 @@ export function managedMessageIdSpecFromPb(v: pb.ManagedMessageIdSpec): t.Manage } export function managedMessageIdSpecToPb(v: t.ManagedMessageIdSpec): pb.ManagedMessageIdSpec { + // The SAME refusal the Play path makes, at the persistence sink: the shared parser maps blank + // text to a zero-length byte array, so a freshly selected Message-ID mode could be SAVED blank + // and every later Play of that item failed. A value that cannot start a session must not be + // storable either. + const invalidReason = messageIdError(v.hexString); + if (invalidReason !== undefined) { + throw new Error(invalidReason); + } + const specPb = new pb.ManagedMessageIdSpec(); const messageIdPb = new consumerPb.MessageId(); messageIdPb.setMessageId(hexStringToByteArray(v.hexString)); @@ -169,7 +179,13 @@ export function managedDateTimeSpecFromPb(v: pb.ManagedDateTimeSpec): t.ManagedD export function managedDateTimeSpecToPb(v: t.ManagedDateTimeSpec): pb.ManagedDateTimeSpec { const specPb = new pb.ManagedDateTimeSpec(); - specPb.setDateTime(Timestamp.fromDate(v.dateTime)); + // Store whole epoch seconds, dropping any sub-second part. The session request floors this exact + // value to seconds (startFromToPb's dateTime case), so `Timestamp.fromDate` - which would keep the + // millis as nanos - leaves the at-rest value richer than the session can use, and save-then-load + // is not stable. The second-granular picker never produces sub-second input anyway. + const timestampPb = new Timestamp(); + timestampPb.setSeconds(Math.floor(v.dateTime.getTime() / 1000)); + specPb.setDateTime(timestampPb); return specPb; } @@ -291,6 +307,10 @@ export function managedConsumerSessionStartFromSpecFromPb(v: pb.ManagedConsumerS return { startFrom: { type: 'nthMessageAfterEarliest', n: v.getStartFromNthMessageAfterEarliest()!.getN() } }; case pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_NTH_MESSAGE_BEFORE_LATEST: return { startFrom: { type: 'nthMessageBeforeLatest', n: v.getStartFromNthMessageBeforeLatest()!.getN() } }; + case pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_APPROXIMATE_ENTRY_POSITION: + return { startFrom: { type: 'approximateEntryPosition', fraction: v.getStartFromApproximateEntryPosition()!.getFraction() } }; + case pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_APPROXIMATE_PUBLISH_TIME_POSITION: + return { startFrom: { type: 'approximatePublishTimePosition', fraction: v.getStartFromApproximatePublishTimePosition()!.getFraction() } }; case pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_MESSAGE_ID: return { startFrom: { type: 'messageId', messageId: managedMessageIdValOrRefFromPb(v.getStartFromMessageId()!) } }; case pb.ManagedConsumerSessionStartFromSpec.StartFromCase.START_FROM_DATE_TIME: @@ -317,6 +337,12 @@ export function managedConsumerSessionStartFromSpecToPb(v: t.ManagedConsumerSess case 'nthMessageBeforeLatest': specPb.setStartFromNthMessageBeforeLatest(new consumerPb.NthMessageBeforeLatest().setN(v.startFrom.n)); break; + case 'approximateEntryPosition': + specPb.setStartFromApproximateEntryPosition(new consumerPb.ApproximateEntryPosition().setFraction(v.startFrom.fraction)); + break; + case 'approximatePublishTimePosition': + specPb.setStartFromApproximatePublishTimePosition(new consumerPb.ApproximatePublishTimePosition().setFraction(v.startFrom.fraction)); + break; case 'messageId': specPb.setStartFromMessageId(managedMessageIdValOrRefToPb(v.startFrom.messageId)); break; @@ -1171,7 +1197,22 @@ export function managedConsumerSessionConfigSpecFromPb(v: pb.ManagedConsumerSess pauseTriggerChain: managedConsumerSessionPauseTriggerChainValOrRefFromPb(v.getPauseTriggerChain()!), coloringRuleChain: managedColoringRuleChainValOrRefFromPb(v.getColoringRuleChain()!), valueProjectionList: managedValueProjectionListValOrRefFromPb(v.getValueProjectionList()!), - numDisplayItems: v.getNumDisplayItems()?.getValue() + numDisplayItems: v.getNumDisplayItems()?.getValue(), + // Absent/unspecified on the wire means the DEFAULT, which is Guaranteed (owner decision + // 2026-08-11) - the server maps proto absence the same way, for items saved before the field + // existed as much as for new ones. Best effort and Fastest are only ever explicit choices. + messageDeliveryOrder: v.getMessageDeliveryOrder() === consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED + ? 'as-received' + : v.getMessageDeliveryOrder() === consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME + ? 'best-effort' + : 'guaranteed', + deliveryOrderKey: v.getDeliveryOrderKey() === consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME + ? 'broker-publish-time' + : v.getDeliveryOrderKey() === consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME + ? 'event-time' + : v.getDeliveryOrderKey() === consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME + ? 'publish-time' + : undefined }; } @@ -1188,6 +1229,24 @@ export function managedConsumerSessionConfigSpecToPb(v: t.ManagedConsumerSession specPb.setNumDisplayItems(new Int64Value().setValue(v.numDisplayItems)); } + if (v.messageDeliveryOrder === 'as-received') { + specPb.setMessageDeliveryOrder(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_AS_RECEIVED); + } else if (v.messageDeliveryOrder === 'best-effort') { + specPb.setMessageDeliveryOrder(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_BEST_EFFORT_BY_PUBLISH_TIME); + } else { + // 'guaranteed' or a missing value from an older JSON item: both write the default (owner + // decision 2026-08-11), so a re-saved legacy item carries the order it will actually run under. + specPb.setMessageDeliveryOrder(consumerPb.MessageDeliveryOrder.MESSAGE_DELIVERY_ORDER_GUARANTEED); + } + + if (v.deliveryOrderKey === 'broker-publish-time') { + specPb.setDeliveryOrderKey(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_BROKER_PUBLISH_TIME); + } else if (v.deliveryOrderKey === 'event-time') { + specPb.setDeliveryOrderKey(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_EVENT_TIME); + } else if (v.deliveryOrderKey === 'publish-time') { + specPb.setDeliveryOrderKey(consumerPb.DeliveryOrderKey.DELIVERY_ORDER_KEY_PUBLISH_TIME); + } + return specPb; } diff --git a/ui/components/ui/LibraryBrowser/model/user-managed-items.ts b/ui/components/ui/LibraryBrowser/model/user-managed-items.ts index 9c0331603..8243e410d 100644 --- a/ui/components/ui/LibraryBrowser/model/user-managed-items.ts +++ b/ui/components/ui/LibraryBrowser/model/user-managed-items.ts @@ -1,4 +1,4 @@ -import { ConsumerSessionEventBytesDelivered, ConsumerSessionEventBytesProcessed, ConsumerSessionEventMessageDecodeFailed, ConsumerSessionEventMessagesDelivered, ConsumerSessionEventMessagesProcessed, ConsumerSessionEventTimeElapsed, ConsumerSessionEventTopicEndReached, ConsumerSessionEventUnexpectedErrorOccurred, ConsumerSessionPauseTriggerChainMode, DateTimeUnit, JsMessageFilter, MessageFilter, MessageFilterChainMode } from "../../ConsumerSession/types"; +import { ConsumerSessionEventBytesDelivered, ConsumerSessionEventBytesProcessed, ConsumerSessionEventMessageDecodeFailed, ConsumerSessionEventMessagesDelivered, ConsumerSessionEventMessagesProcessed, ConsumerSessionEventTimeElapsed, ConsumerSessionEventTopicEndReached, ConsumerSessionEventUnexpectedErrorOccurred, ConsumerSessionPauseTriggerChainMode, DateTimeUnit, JsMessageFilter, MessageFilter, MessageFilterChainMode, DeliveryOrderKey, MessageDeliveryOrder } from "../../ConsumerSession/types"; import { TopicSelector, MultiTopicSelector, NamespacedRegexTopicSelector } from "../../ConsumerSession/topic-selector/topic-selector"; import { BasicMessageFilter, BasicMessageFilterTarget } from "../../ConsumerSession/basic-message-filter-types"; import { Deserializer } from "../../ConsumerSession/deserializer/deserializer"; @@ -114,12 +114,16 @@ export type StartFromEarliestMessage = { type: 'earliestMessage' }; export type StartFromLatestMessage = { type: 'latestMessage' }; export type StartFromNthMessageAfterEarliest = { type: 'nthMessageAfterEarliest', n: number }; export type StartFromNthMessageBeforeLatest = { type: 'nthMessageBeforeLatest', n: number }; +/** Approximate position over retained entries, in [0, 1], resolved per physical topic. */ +export type StartFromApproximateEntryPosition = { type: 'approximateEntryPosition', fraction: number }; +/** Approximate position between retained boundary-entry publish times, in [0, 1], per logical topic. */ +export type StartFromApproximatePublishTimePosition = { type: 'approximatePublishTimePosition', fraction: number }; export type StartFromMessageId = { type: 'messageId', messageId: ManagedMessageIdValOrRef }; export type StartFromDateTime = { type: 'dateTime', dateTime: ManagedDateTimeValOrRef }; export type StartFromRelativeDateTime = { type: 'relativeDateTime', relativeDateTime: ManagedRelativeDateTimeValOrRef }; export type ManagedConsumerSessionStartFromSpec = { - startFrom: StartFromEarliestMessage | StartFromLatestMessage | StartFromNthMessageAfterEarliest | StartFromNthMessageBeforeLatest | StartFromMessageId | StartFromDateTime | StartFromRelativeDateTime, + startFrom: StartFromEarliestMessage | StartFromLatestMessage | StartFromNthMessageAfterEarliest | StartFromNthMessageBeforeLatest | StartFromApproximateEntryPosition | StartFromApproximatePublishTimePosition | StartFromMessageId | StartFromDateTime | StartFromRelativeDateTime, }; export type ManagedConsumerSessionStartFrom = { metadata: ManagedItemMetadata, @@ -258,7 +262,10 @@ export type ManagedConsumerSessionConfigSpec = { pauseTriggerChain: ManagedConsumerSessionPauseTriggerChainValOrRef, coloringRuleChain: ManagedColoringRuleChainValOrRef, valueProjectionList: ManagedValueProjectionListValOrRef, - numDisplayItems: number | undefined + numDisplayItems: number | undefined, + // Optional on disk for older JSON; a missing value resolves to Guaranteed. + messageDeliveryOrder?: MessageDeliveryOrder, + deliveryOrderKey?: DeliveryOrderKey }; export type ManagedConsumerSessionConfig = { diff --git a/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.module.css b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.module.css index 4b9c3f70d..fd736ba1d 100644 --- a/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.module.css +++ b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.module.css @@ -15,6 +15,12 @@ flex: 1 1 auto; } +.Error { + color: var(--accent-color-red); + font-size: x-small; + margin-bottom: 12rem; +} + .Checkbox { display: flex; align-items: center; diff --git a/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.test.tsx b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.test.tsx new file mode 100644 index 000000000..b6ba7a0a9 --- /dev/null +++ b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.test.tsx @@ -0,0 +1,163 @@ +/** + * @jest-environment jsdom + * + * "n <unit> ago", and what the number is allowed to be. + * + * The field used to commit `Number(v)` after every keystroke, and `Number` answers something for + * nearly anything. Each of those answers is a different INSTANT, chosen silently: + * + * - `Number('')` is 0, so clearing the field to retype it means "now" - not "unset"; + * - a negative is a FUTURE instant, under a label that says "ago", and the server subtracts it + * without complaint; + * - a fraction and anything past 2,147,483,647 are coerced on the way through protobuf's int32, + * so the value that travels is not the value on screen (2147483648 arrives as -2147483648, i.e. + * an instant roughly 68 years in the FUTURE for `year`). + * + * None of those are refusals, so none of them are visible - the session simply starts somewhere + * else. The picker is not reachable from Playwright with a bad value either (a `type=number` field + * silently drops what it cannot parse), which is why this lives here. + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import RelativeDateTimePicker from './RelativeDateTimePicker'; +import { relativeDateTimeValueMax } from './relative-date-time'; +import { RelativeDateTime } from '../ConsumerSession/types'; + +const relativeDateTime = (value: number): RelativeDateTime => ({ value, unit: 'hour', isRoundedToUnitStart: false }); + +/** The picker with a parent that applies what it is handed, as the real editor does. */ +const renderPicker = (initial: RelativeDateTime = relativeDateTime(3), isReadOnly = false) => { + const onChange = jest.fn(); + const Controlled = () => { + const [value, setValue] = React.useState(initial); + return ( + <RelativeDateTimePicker + value={value} + onChange={(v) => { + setValue(v); + onChange(v); + }} + isReadOnly={isReadOnly} + /> + ); + }; + + render(<Controlled />); + return onChange; +}; + +const valueInput = () => screen.getByTestId('relative-date-time-value') as HTMLInputElement; +const error = () => screen.queryByTestId('relative-date-time-value-error'); +const lastValue = (onChange: jest.Mock) => onChange.mock.calls[onChange.mock.calls.length - 1][0]; + +describe('the "how long ago" number', () => { + it('commits a whole number of units', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '12' } }); + + expect(error()).toBeNull(); + expect(lastValue(onChange)).toEqual({ value: 12, unit: 'hour', isRoundedToUnitStart: false }); + }); + + it('accepts zero - "0 hours ago" is now, which is a position like any other', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '0' } }); + + expect(error()).toBeNull(); + expect(lastValue(onChange).value).toBe(0); + }); + + it('refuses an emptied field rather than quietly meaning "now"', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses a negative, which would be a FUTURE instant under a label that says "ago"', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '-5' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses a fraction instead of letting the wire round it', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '1.5' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('refuses an exponent instead of reading part of it', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '1e3' } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('accepts the largest value the int32 model can carry', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: String(relativeDateTimeValueMax) } }); + + expect(error()).toBeNull(); + expect(lastValue(onChange).value).toBe(relativeDateTimeValueMax); + expect(relativeDateTimeValueMax).toBe(2147483647); + }); + + it('refuses one past it, which the int32 wire turns into a future instant', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: String(relativeDateTimeValueMax + 1) } }); + + expect(error()).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it('keeps the refused text on screen so it can be corrected, and recovers', () => { + const onChange = renderPicker(); + + fireEvent.change(valueInput(), { target: { value: '-5' } }); + expect(valueInput().value).toBe('-5'); + + fireEvent.change(valueInput(), { target: { value: '7' } }); + + expect(error()).toBeNull(); + expect(lastValue(onChange).value).toBe(7); + }); + + it('says which value is still in effect while the entry is refused', () => { + // The refused entry does not undo the last valid one, so Play starts from THAT. + renderPicker(relativeDateTime(3)); + + fireEvent.change(valueInput(), { target: { value: '-5' } }); + + expect(error()?.textContent).toMatch(/\b3\b/); + }); + + it('leaves the other fields of the value alone when the number is edited', () => { + const onChange = renderPicker({ value: 3, unit: 'week', isRoundedToUnitStart: true }); + + fireEvent.change(valueInput(), { target: { value: '9' } }); + + expect(lastValue(onChange)).toEqual({ value: 9, unit: 'week', isRoundedToUnitStart: true }); + }); +}); + +describe('a read-only relative time', () => { + it('does not let the number be edited', () => { + renderPicker(relativeDateTime(3), true); + + expect(valueInput().disabled).toBe(true); + }); +}); diff --git a/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.tsx b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.tsx index 0f4c28e93..d57609f73 100644 --- a/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.tsx +++ b/ui/components/ui/RelativeDateTimePicker/RelativeDateTimePicker.tsx @@ -1,10 +1,11 @@ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import s from './RelativeDateTimePicker.module.css' import { DateTimeUnit, RelativeDateTime } from '../ConsumerSession/types'; import Select from '../Select/Select'; import Checkbox from '../Checkbox/Checkbox'; import Input from '../Input/Input'; import Toggle from '../Toggle/Toggle'; +import { relativeDateTimeValueFromText, relativeDateTimeValueMax } from './relative-date-time'; export type RelativeDateTimePickerProps = { value: RelativeDateTime; @@ -17,14 +18,42 @@ const pluralize = (n: number, singular: string) => { }; const RelativeDateTimePicker: React.FC<RelativeDateTimePickerProps> = (props) => { + // The typed text, kept here rather than derived from the committed value on every render, so an + // in-progress or refused entry stays on screen (and stays correctable) without ever becoming the + // instant the session starts from. Clearing the field to retype it is the ordinary case, and it + // used to commit 0 - "now" - on the way through. + const [draft, setDraft] = useState<string>(() => String(props.value.value)); + + // Adopt a value that changed elsewhere (a library item that resolved, a picked item), but leave a + // draft that already means the same number alone. + useEffect(() => { + if (relativeDateTimeValueFromText(draft) !== props.value.value) { + setDraft(String(props.value.value)); + } + }, [props.value.value]); + + const onDraftChange = (v: string) => { + setDraft(v); + + const value = relativeDateTimeValueFromText(v); + if (value !== undefined) { + props.onChange({ ...props.value, value }); + } + }; + + const isInvalid = relativeDateTimeValueFromText(draft) === undefined; + return ( <div className={s.RelativeDateTimePicker}> <div className={s.ValueAndUnit}> <div className={s.Value}> <Input + testId="relative-date-time-value" type="number" - value={String(props.value.value)} - onChange={(v) => props.onChange({ ...props.value, value: Number(v) })} + value={draft} + onChange={onDraftChange} + isError={isInvalid} + inputProps={{ min: 0, max: relativeDateTimeValueMax, step: 1 }} isReadOnly={props.isReadOnly} /> </div> @@ -49,6 +78,14 @@ const RelativeDateTimePicker: React.FC<RelativeDateTimePickerProps> = (props) => <strong>ago</strong> </div> + {isInvalid && ( + <div className={s.Error} data-testid="relative-date-time-value-error"> + {/* A refused entry does not undo the last valid one, so a session started now would use + THAT - saying which number that is turns a silent difference into a visible one. */} + Enter a whole number of {props.value.unit}s from 0 to {relativeDateTimeValueMax}. The value in effect is still {props.value.value}. + </div> + )} + <div className={s.Checkbox}> <Toggle value={props.value.isRoundedToUnitStart} diff --git a/ui/components/ui/RelativeDateTimePicker/relative-date-time.ts b/ui/components/ui/RelativeDateTimePicker/relative-date-time.ts new file mode 100644 index 000000000..400fe2c80 --- /dev/null +++ b/ui/components/ui/RelativeDateTimePicker/relative-date-time.ts @@ -0,0 +1,35 @@ +/** + * How far back "n <unit> ago" is allowed to be. + * + * The model is a protobuf `int32` (`RelativeDateTime.value`, Scala `Int`), and the server subtracts + * it from "now" without a trust-boundary check of its own - so whatever this field commits IS the + * instant the session starts from. `Number()` accepts far more than that model can carry, and every + * excess value is silently coerced rather than refused: + * + * - blank becomes 0, which reads as "now" rather than as "nothing entered"; + * - a negative is subtracted as a negative, i.e. an instant in the FUTURE, under a label that says + * "ago"; + * - anything past 2,147,483,647 wraps on the wire into a negative, with the same effect. + */ +export const relativeDateTimeValueMax = 2147483647; + +/** + * The number of units a text field means, or `undefined` when it does not mean one. `undefined` is + * "refuse this", never "use zero" - zero is a position the user can ask for on purpose. + */ +export function relativeDateTimeValueFromText(raw: string): number | undefined { + const trimmed = raw.trim(); + + // No sign, no decimal point, no exponent: each of those is a value the int32 model cannot carry + // as typed, and coercing it silently is what this exists to prevent. + if (!/^\d+$/.test(trimmed)) { + return undefined; + } + + const value = Number(trimmed); + if (value > relativeDateTimeValueMax) { + return undefined; + } + + return value; +} diff --git a/ui/components/ui/Select/Select.tsx b/ui/components/ui/Select/Select.tsx index 9e468e0e1..e68f885a2 100644 --- a/ui/components/ui/Select/Select.tsx +++ b/ui/components/ui/Select/Select.tsx @@ -6,7 +6,9 @@ import arrowDownIcon from './arrow-down.svg'; export type ListItem<V> = { type: 'item', value: V, - title: string + title: string, + /** Kept visible but not selectable - an option that is inapplicable here, rather than absent. */ + disabled?: boolean } | { type: 'group', title: string, items: ListItem<V>[] } | { type: 'empty', title: string }; export type List<V> = ListItem<V>[] @@ -21,6 +23,10 @@ export type SelectProps<V> = { size?: 'regular' | 'small'; isReadOnly?: boolean; testId?: string; + /** Forwarded to the native select so an external <label htmlFor> can address it. */ + id?: string; + /** Forwarded as aria-describedby so visible help text is announced with the control. */ + ariaDescribedBy?: string; } function Select<V extends string>(props: SelectProps<V>): React.ReactElement { @@ -30,7 +36,7 @@ function Select<V extends string>(props: SelectProps<V>): React.ReactElement { } const valueKey = item.value.toString(); - return <option key={valueKey} value={valueKey}>{item.title}</option> + return <option key={valueKey} value={valueKey} disabled={item.disabled}>{item.title}</option> } return ( @@ -42,6 +48,8 @@ function Select<V extends string>(props: SelectProps<V>): React.ReactElement { `}> {props.value === undefined && <div className={s.Placeholder}>{props.placeholder}</div>} <select + id={props.id} + aria-describedby={props.ariaDescribedBy} className={`${s.Select} ${props.disabled ? s.DisabledSelect : ''}`} onChange={(v) => props.onChange(v.target.value as V)} value={props.value} diff --git a/ui/components/ui/Table/Table.columnOrder.test.tsx b/ui/components/ui/Table/Table.columnOrder.test.tsx new file mode 100644 index 000000000..cfa071943 --- /dev/null +++ b/ui/components/ui/Table/Table.columnOrder.test.tsx @@ -0,0 +1,134 @@ +/** + * @jest-environment jsdom + * @jest-environment-options {"customExportConditions": ["node"]} + * + * Column reorder DRIVEN THROUGH THE HEADERS of the shared table: which order a drop produces, and + * that the result is what gets persisted. The pure rules live in `useColumnOrder.spec.ts`; this is + * the half that decides what the DOM asks for - and a drop target the DOM can never emit makes a + * position unreachable no matter how correct the pure helper is. + */ +const mockNotifications = { + notifySuccess: jest.fn(), + notifyInfo: jest.fn(), + notifyWarn: jest.fn(), + notifyError: jest.fn(), +}; +jest.mock('../../app/contexts/Notifications', () => ({ + useContext: () => mockNotifications, +})); + +// The production table virtualizes rows and measures its viewport; jsdom lays nothing out, so the +// same header/rows are rendered into a plain table here (the pattern TopicPositions.test.tsx uses). +jest.mock('react-virtuoso', () => { + const ReactRuntime = require('react'); + return { + TableVirtuoso: (props: any) => + ReactRuntime.createElement( + 'table', + null, + ReactRuntime.createElement('thead', null, props.fixedHeaderContent()), + ReactRuntime.createElement( + 'tbody', + null, + props.data.map((entry: any, index: number) => + ReactRuntime.createElement('tr', { key: index }, props.itemContent(index, entry))) + ) + ), + }; +}); + +import React from 'react'; +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { SWRConfig } from 'swr'; +import Table from './Table'; + +type ColumnKey = 'a' | 'b' | 'c'; +type Entry = { id: string }; + +const columns = { + columns: { + a: { title: 'A', render: () => 'a' }, + b: { title: 'B', render: () => 'b' }, + c: { title: 'C', render: () => 'c' }, + }, + defaultConfig: [ + { columnKey: 'a' as const, visibility: 'visible' as const, width: 100 }, + { columnKey: 'b' as const, visibility: 'visible' as const, width: 100 }, + { columnKey: 'c' as const, visibility: 'visible' as const, width: 100 }, + ], +}; + +const renderTable = async (tableId: string) => { + await act(async () => { + render( + <SWRConfig value={{ shouldRetryOnError: false, refreshInterval: 0, provider: () => new Map() }}> + <Table<ColumnKey, Entry, never> + tableId={tableId} + dataLoader={{ cacheKey: [tableId], loader: async () => [{ id: '1' }] }} + columns={columns} + getId={(entry) => entry.id} + autoRefresh={{ intervalMs: 0 }} + toolbar={{ visibility: 'hidden' }} + /> + </SWRConfig> + ); + }); +}; + +const headerOrder = () => + screen.getAllByTestId('table-th').map((th) => th.getAttribute('data-column-key')); + +const th = (columnKey: ColumnKey) => + screen.getAllByTestId('table-th').find((el) => el.getAttribute('data-column-key') === columnKey)!; + +/** One native drag gesture: pick a header up, hover another, drop it there. */ +const dragOnto = (dragged: ColumnKey, target: ColumnKey) => { + const dataTransfer = { setData: jest.fn(), getData: () => dragged, dropEffect: '', effectAllowed: '' }; + fireEvent.dragStart(th(dragged), { dataTransfer }); + fireEvent.dragOver(th(target), { dataTransfer }); + fireEvent.drop(th(target), { dataTransfer }); + fireEvent.dragEnd(th(dragged), { dataTransfer }); +}; + +describe('dragging a column header of the shared table', () => { + beforeEach(() => { + window.localStorage.clear(); + }); + + afterEach(() => { + cleanup(); + window.localStorage.clear(); + }); + + it('moves the dragged column onto a LATER position, including the last one', async () => { + await renderTable('drag-to-end'); + expect(headerOrder()).toEqual(['a', 'b', 'c']); + + // The whole header is one drop target and it highlights as one, so dropping A on C means "put + // A where C is" - A ends up last. Reading every drop as "insert BEFORE the target" instead + // makes the final position unreachable by any gesture: A on C would yield b,a,c, and no + // header exists to the right of C to drop on. + dragOnto('a', 'c'); + + expect(headerOrder()).toEqual(['b', 'c', 'a']); + // ...and the order that is on screen is the order that survives a reload. + expect(JSON.parse(window.localStorage.getItem('table:drag-to-end:column-order')!)).toEqual(['b', 'c', 'a']); + }); + + it('moves it onto an EARLIER position, including the first one', async () => { + await renderTable('drag-to-start'); + + dragOnto('c', 'a'); + + expect(headerOrder()).toEqual(['c', 'a', 'b']); + expect(JSON.parse(window.localStorage.getItem('table:drag-to-start:column-order')!)).toEqual(['c', 'a', 'b']); + }); + + it('dropping a column on itself changes nothing', async () => { + await renderTable('drag-self'); + + dragOnto('b', 'b'); + + expect(headerOrder()).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/ui/components/ui/Table/Table.module.css b/ui/components/ui/Table/Table.module.css index 403417662..956b5e673 100644 --- a/ui/components/ui/Table/Table.module.css +++ b/ui/components/ui/Table/Table.module.css @@ -230,4 +230,24 @@ height: 12rem; border-radius: 4rem; background-color: var(--surface-color); -} \ No newline at end of file +} + +/* size='small': dense variant for embedded tables. Presentation only. */ +.SizeSmall .Th, +.SizeSmall .Td { + font-size: 12rem; +} + +.SizeSmall .Th .ThContent { + padding: 2rem 6rem; +} + +.SizeSmall .Td .TdContent { + padding: 2rem 6rem; +} + +/* The drop-position indicator while a column drag hovers this header: the dragged column will + land immediately BEFORE it. */ +.ThDragOver { + box-shadow: inset 3rem 0 0 0 var(--accent-color, #4a72ff); +} diff --git a/ui/components/ui/Table/Table.tsx b/ui/components/ui/Table/Table.tsx index 88b0beb5d..b5107a2e2 100644 --- a/ui/components/ui/Table/Table.tsx +++ b/ui/components/ui/Table/Table.tsx @@ -8,6 +8,7 @@ import arrowUpIcon from './arrow-up.svg'; import { useDebounce } from 'use-debounce'; import { useColumnWidths, ColumnConstraint } from '../resizable/useColumnWidths'; import ColumnResizeHandle from '../resizable/ColumnResizeHandle'; +import { useColumnOrder } from '../resizable/useColumnOrder'; import * as Notifications from '../../app/contexts/Notifications'; import useSWR, { SWRConfiguration, mutate } from 'swr'; import { renderToStaticMarkup } from 'react-dom/server'; @@ -21,6 +22,7 @@ import SmallButton from '../SmallButton/SmallButton'; import refreshIcon from './refresh.svg'; import NoData from '../NoData/NoData'; import { tooltipId } from '../Tooltip/Tooltip'; +import { sortTableData } from './sorting'; import { TableFilterDescriptor, TableFilterValue } from './filters/types'; import filterIcon from './filter.svg'; @@ -58,6 +60,9 @@ export type Column<DE, LD> = { title: ReactNode, render: (data: DE, lazyData?: LD) => ReactNode, sortFn?: (a: { data: DE, lazyData: LD | undefined }, b: { data: DE, lazyData: LD | undefined }) => number, + /** Marks rows whose rendered sort value is absent. Missing rows stay after populated rows in + * both directions; unlike encoding absence in sortFn, this is not inverted by descending sort. */ + isSortValueMissing?: (entry: { data: DE, lazyData: LD | undefined }) => boolean, filter?: { descriptor: TableFilterDescriptor, testFn: (data: DE, lazyData: LD | undefined, filterValue: TableFilterValue) => boolean, @@ -96,17 +101,42 @@ export type TableProps<CK extends ColumnKey, DE, LD> = { }, itemNamePlural?: string, toolbar?: { visibility: 'visible' | 'hidden' }, + /** 'small' tightens paddings and type for dense, embedded tables (e.g. the Tools panel). + * Purely presentational - every behavior is identical. */ + size?: 'default' | 'small', + /** Rows this returns true for stay FIRST under every sort, in both directions - for summary + * rows that describe the table rather than compete with it. Column sortFns stay direction-blind + * plain comparators; a comparator cannot hold a row on top by itself, because 'desc' is the + * reverse of the sorted array and would flip it to the bottom. */ + pinFirst?: (dataEntry: DE) => boolean, }; function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): ReactElement | null { - const scrollContainerRef = React.useRef<HTMLDivElement>(null); + // State, not a ref: TableVirtuoso must first mount with the REAL scroll parent. Mounting with + // undefined and switching to customScrollParent on a later render leaves a 0-height viewport + // measurement that only a window resize refreshes - no rows render until then. + const [scrollContainer, setScrollContainer] = useState<HTMLDivElement | null>(null); const [lazyData, setLazyData] = useState<Record<DataEntryKey, LD>>({}); const [lazyDataLoading, setLazyDataLoading] = useState<Record<string, boolean>>({}); const [itemsRendered, setItemsRendered] = useState<ListItem<DE>[]>([]); const [itemsRenderedDebounced] = useDebounce(itemsRendered, 250); const [sort, setSort] = useState<Sort<CK>>(props.defaultSort ?? { type: 'none' }); const { notifyError } = Notifications.useContext(); - const columnsConfig = props.columns.defaultConfig.filter(column => column.visibility === 'visible'); + const visibleConfig = props.columns.defaultConfig.filter(column => column.visibility === 'visible'); + // Sticky-left columns are pinned to the FRONT and are not reorderable: a sticky column in the + // middle of the scroll area would float over its neighbours. Everything else can be dragged + // into any order, and the order is remembered per table like the widths are. + const stickyConfig = visibleConfig.filter(column => column.stickyTo === 'left'); + const reorderableConfig = visibleConfig.filter(column => column.stickyTo !== 'left'); + const { order: columnOrder, moveColumn } = useColumnOrder<CK>( + props.tableId, + reorderableConfig.map(column => column.columnKey) + ); + const columnsConfig = [ + ...stickyConfig, + ...columnOrder.flatMap(columnKey => reorderableConfig.filter(column => column.columnKey === columnKey)) + ]; + const draggingColumnRef = React.useRef<CK | undefined>(undefined); // ONE GLOBAL auto-refresh preference shared by every table - an owner design decision // ("we either want to refresh any table, or not"), deliberately NOT per-table. // use-local-storage-state syncs all hook instances on the same key, so flipping the toggle on @@ -192,7 +222,8 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea columnKey: CK, isSortable: boolean, filter: Column<DE, LD>['filter'], - style?: React.CSSProperties + style?: React.CSSProperties, + isDraggable?: boolean }; const Th = useMemo(() => (thProps: ThProps) => { const handleColumnHeaderClick = () => { @@ -225,6 +256,47 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea const isColumnFiltered = Boolean(filtersInUse[thProps.columnKey]); + // Native HTML5 drag: the resize handle's mousedown sets suppressSortClickRef before any drag + // gesture can begin, so a resize never turns into a column drag. The drag-over indicator is + // toggled IMPERATIVELY (classList), never via state: this Th component's TYPE is recreated + // when its memo deps change, and a state update mid-drag would remount the th under the + // cursor - the browser then abandons the drop and the drag silently does nothing. + const dragProps = !thProps.isDraggable ? {} : { + draggable: true, + onDragStart: (e: React.DragEvent) => { + if (suppressSortClickRef.current) { + e.preventDefault(); + return; + } + draggingColumnRef.current = thProps.columnKey; + e.dataTransfer.setData('text/plain', thProps.columnKey); + e.dataTransfer.effectAllowed = 'move'; + }, + onDragEnd: () => { + draggingColumnRef.current = undefined; + }, + onDragOver: (e: React.DragEvent) => { + if (draggingColumnRef.current === undefined) { + return; + } + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + (e.currentTarget as HTMLElement).classList.add(s.ThDragOver); + }, + onDragLeave: (e: React.DragEvent) => { + (e.currentTarget as HTMLElement).classList.remove(s.ThDragOver); + }, + onDrop: (e: React.DragEvent) => { + e.preventDefault(); + (e.currentTarget as HTMLElement).classList.remove(s.ThDragOver); + const dragged = draggingColumnRef.current; + draggingColumnRef.current = undefined; + if (dragged !== undefined && dragged !== thProps.columnKey) { + moveColumn(dragged, thProps.columnKey); + } + } + }; + return ( <th className={`${s.Th} ${thProps.isSortable ? s.SortableTh : ''}`} @@ -233,6 +305,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea data-testid="table-th" data-column-key={thProps.columnKey} data-sort-direction={(sort.type === 'by-single-column' && sort.column === thProps.columnKey) ? sort.direction : undefined} + {...dragProps} > <div data-tooltip-id={tooltipId} @@ -284,7 +357,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea <ColumnResizeHandle onResizeStart={(clientX) => startColumnResize(thProps.columnKey, clientX)} /> </th> ); - }, [sort, props.columns, filtersInUse, startColumnResize]); + }, [sort, props.columns, filtersInUse, startColumnResize, moveColumn]); const sortedData = useMemo(() => { const activeFilters = Object.entries<FilterInUse>(filtersInUseDebounced as Record<string, FilterInUse>).filter(([_, filter]) => { @@ -303,20 +376,35 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea }); }); + const pin = (rows: DE[]): DE[] => { + const pinFirst = props.pinFirst; + if (pinFirst === undefined) { + return rows; + } + return [...rows.filter((de) => pinFirst(de)), ...rows.filter((de) => !pinFirst(de))]; + }; + if (sort.type === 'by-single-column') { - const sortFn = props.columns.columns[sort.column]!.sortFn; - const sorted = sortFn ? - dataToSort.sort((a, b) => sortFn( - { data: a, lazyData: lazyData[props.getId(a)] }, - { data: b, lazyData: lazyData[props.getId(b)] }) - ) : - dataToSort; - - return sort.direction === 'asc' ? sorted : [...sorted].reverse(); + const column = props.columns.columns[sort.column]!; + const sortFn = column.sortFn; + const entryOf = (dataEntry: DE) => ({ data: dataEntry, lazyData: lazyData[props.getId(dataEntry)] }); + if (sortFn === undefined) { + return pin(dataToSort); + } + + return sortTableData({ + data: dataToSort, + direction: sort.direction, + sortFn, + entryOf, + isSortValueMissing: column.isSortValueMissing, + // The synthetic summary is pinned only after column ordering. + pinFirst: props.pinFirst + }); } - return data; - }, [loadedData, sort, lazyData, filtersInUseDebounced]); + return pin(data); + }, [loadedData, sort, lazyData, filtersInUseDebounced, props.pinFirst]); if (data.length === 0) { return ( @@ -327,7 +415,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea } return ( - <div className={s.Table}> + <div className={`${s.Table} ${props.size === 'small' ? s.SizeSmall : ''}`}> <div className={s.Toolbars}> {Boolean(Object.keys(filtersInUse).length) && <div className={s.FiltersToolbar}> <FiltersToolbar<CK> @@ -370,12 +458,12 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea )} </div> - <div className={s.ScrollContainer} ref={scrollContainerRef}> - <TableVirtuoso + <div className={s.ScrollContainer} ref={setScrollContainer}> + {scrollContainer !== null && <TableVirtuoso data={sortedData} overscan={{ - main: (scrollContainerRef?.current?.clientHeight || 0) / 3, - reverse: (scrollContainerRef?.current?.clientHeight || 0) / 3 + main: (scrollContainer.clientHeight || 0) / 3, + reverse: (scrollContainer.clientHeight || 0) / 3 }} fixedHeaderContent={() => ( <tr> @@ -390,6 +478,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea isSortable={Boolean(props.columns.columns[columnConfig.columnKey]!.sortFn)} filter={props.columns.columns[columnConfig.columnKey]!.filter} style={{ width: getColumnWidth(columnConfig.columnKey), ...style }} + isDraggable={columnConfig.stickyTo !== 'left'} /> ); })} @@ -424,7 +513,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea </> ); }} - customScrollParent={scrollContainerRef.current || undefined} + customScrollParent={scrollContainer} totalCount={data?.length} itemsRendered={(items) => { const isShouldUpdate = !isEqual(itemsRendered, items) @@ -432,7 +521,7 @@ function Table<CK extends ColumnKey, DE, LD>(props: TableProps<CK, DE, LD>): Rea setItemsRendered(() => items); } }} - /> + />} </div> </div> ); diff --git a/ui/components/ui/Table/sorting.spec.ts b/ui/components/ui/Table/sorting.spec.ts new file mode 100644 index 000000000..728f6c105 --- /dev/null +++ b/ui/components/ui/Table/sorting.spec.ts @@ -0,0 +1,52 @@ +import { sortTableData } from './sorting'; + +type Row = { id: string; value?: number; summary?: boolean }; +type Lazy = { value?: number }; + +const rows: Row[] = [ + { id: 'summary', value: 999, summary: true }, + { id: 'equal-a', value: 2 }, + { id: 'missing' }, + { id: 'low', value: 1 }, + { id: 'equal-b', value: 2 } +]; + +const sort = (direction: 'asc' | 'desc', isMissing = false): string[] => + sortTableData<Row, Lazy>({ + data: rows, + direction, + entryOf: (data) => ({ data, lazyData: undefined }), + sortFn: (a, b) => (a.data.value ?? 0) - (b.data.value ?? 0), + isSortValueMissing: isMissing ? (entry) => entry.data.value === undefined : undefined, + pinFirst: (row) => row.summary === true + }).map((row) => row.id); + +describe('sortTableData', () => { + it('sorts both directions, keeps equal rows stable, and pins summaries after sorting', () => { + expect(sort('asc')).toEqual(['summary', 'missing', 'low', 'equal-a', 'equal-b']); + expect(sort('desc')).toEqual(['summary', 'equal-a', 'equal-b', 'low', 'missing']); + }); + + it('keeps missing values last in both directions', () => { + expect(sort('asc', true)).toEqual(['summary', 'low', 'equal-a', 'equal-b', 'missing']); + expect(sort('desc', true)).toEqual(['summary', 'equal-a', 'equal-b', 'low', 'missing']); + }); + + it('passes lazy values to the comparator and missing-value predicate', () => { + const lazyById: Record<string, Lazy | undefined> = { + a: { value: 10 }, + b: undefined, + c: { value: 5 } + }; + const lazyRows: Row[] = [{ id: 'a' }, { id: 'b' }, { id: 'c' }]; + const sorted = sortTableData<Row, Lazy>({ + data: lazyRows, + direction: 'desc', + entryOf: (data) => ({ data, lazyData: lazyById[data.id] }), + sortFn: (a, b) => (a.lazyData?.value ?? 0) - (b.lazyData?.value ?? 0), + isSortValueMissing: (entry) => entry.lazyData?.value === undefined + }); + + expect(sorted.map((row) => row.id)).toEqual(['a', 'c', 'b']); + }); +}); diff --git a/ui/components/ui/Table/sorting.ts b/ui/components/ui/Table/sorting.ts new file mode 100644 index 000000000..c888a8ad8 --- /dev/null +++ b/ui/components/ui/Table/sorting.ts @@ -0,0 +1,47 @@ +export type TableSortEntry<DE, LD> = { + data: DE; + lazyData: LD | undefined; +}; + +type SortTableDataOptions<DE, LD> = { + data: DE[]; + direction: 'asc' | 'desc'; + sortFn: (a: TableSortEntry<DE, LD>, b: TableSortEntry<DE, LD>) => number; + entryOf: (data: DE) => TableSortEntry<DE, LD>; + isSortValueMissing?: (entry: TableSortEntry<DE, LD>) => boolean; + pinFirst?: (data: DE) => boolean; +}; + +/** Sort one table column without coupling direction to absence or pinning. */ +export function sortTableData<DE, LD>(options: SortTableDataOptions<DE, LD>): DE[] { + const direction = options.direction === 'asc' ? 1 : -1; + const sorted = [...options.data].sort((a, b) => { + const aEntry = options.entryOf(a); + const bEntry = options.entryOf(b); + const aMissing = options.isSortValueMissing?.(aEntry) === true; + const bMissing = options.isSortValueMissing?.(bEntry) === true; + + // Absence is a separate ordering dimension: it stays last rather than becoming "largest" or + // "smallest" when direction flips. Returning zero for two missing values preserves row order. + if (aMissing !== bMissing) { + return aMissing ? 1 : -1; + } + if (aMissing) { + return 0; + } + + // Multiplying the comparator, instead of reversing an ascending result, also preserves the + // original order of equal populated rows in descending mode. + return direction * options.sortFn(aEntry, bEntry); + }); + + if (options.pinFirst === undefined) { + return sorted; + } + + // Summary rows describe the table rather than compete with it, so pin only after sorting. + return [ + ...sorted.filter((row) => options.pinFirst?.(row) === true), + ...sorted.filter((row) => options.pinFirst?.(row) !== true) + ]; +} diff --git a/ui/components/ui/resizable/PaneResizeHandle.module.css b/ui/components/ui/resizable/PaneResizeHandle.module.css index 0ede6483c..10bc10615 100644 --- a/ui/components/ui/resizable/PaneResizeHandle.module.css +++ b/ui/components/ui/resizable/PaneResizeHandle.module.css @@ -1,23 +1,55 @@ .PaneResizeHandle { flex: 0 0 auto; - align-self: stretch; position: relative; - width: 7rem; - cursor: col-resize; user-select: none; touch-action: none; z-index: 50; } -/* Overlay pinned to the right edge of a positioned container (the fixed sidebar). */ +.Horizontal { + align-self: stretch; + width: 7rem; + cursor: col-resize; +} + +.Vertical { + width: 100%; + height: 7rem; + cursor: row-resize; +} + +/* Overlay pinned to an edge of a positioned container. */ .PaneResizeHandle.Edge { position: absolute; +} + +.Horizontal.Edge { top: 0; - right: -3rem; height: 100%; } -.PaneResizeHandle::after { +.Horizontal.EdgeStart { + left: -3rem; +} + +.Horizontal.EdgeEnd { + right: -3rem; +} + +.Vertical.Edge { + left: 0; + width: 100%; +} + +.Vertical.EdgeStart { + top: -3rem; +} + +.Vertical.EdgeEnd { + bottom: -3rem; +} + +.Horizontal::after { content: ''; position: absolute; top: 0; @@ -29,6 +61,18 @@ transition: background-color 0.1s ease-in-out; } +.Vertical::after { + content: ''; + position: absolute; + top: 50%; + right: 0; + left: 0; + height: 2rem; + transform: translateY(-50%); + background-color: transparent; + transition: background-color 0.1s ease-in-out; +} + .PaneResizeHandle:hover::after, .PaneResizeHandle:active::after { background-color: var(--accent-color-blue); diff --git a/ui/components/ui/resizable/PaneResizeHandle.spec.tsx b/ui/components/ui/resizable/PaneResizeHandle.spec.tsx new file mode 100644 index 000000000..d643501de --- /dev/null +++ b/ui/components/ui/resizable/PaneResizeHandle.spec.tsx @@ -0,0 +1,127 @@ +/** @jest-environment jsdom */ + +/** + * The shared pane resize handle: a `role="separator"` that can be dragged with any pointer and - + * when handed its pane - operated from the keyboard, with the pane's size surfaced through + * aria-valuenow/min/max (WAI-ARIA window splitter pattern). The same diff that introduced the + * panes gave the message table's copy cell keyboard support; this control meets the same bar. + */ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import PaneResizeHandle from './PaneResizeHandle'; + +// jsdom has no PointerEvent constructor; React's root listener only cares about the event TYPE +// and the handler only reads MouseEvent fields, so a bubbling MouseEvent stands in for a pointer. +const pointerDown = (el: Element, init: MouseEventInit) => + fireEvent(el, new MouseEvent('pointerdown', { bubbles: true, ...init })); + +describe('PaneResizeHandle', () => { + const renderHandle = (over: Partial<React.ComponentProps<typeof PaneResizeHandle>> = {}) => { + const onResizeStart = jest.fn(); + const keyboardResize = jest.fn(); + const pane = { size: 300, minSize: 100, maxSize: 800, keyboardResize }; + render( + <div> + <PaneResizeHandle testId="handle" onResizeStart={onResizeStart} pane={pane} {...over} /> + </div> + ); + return { onResizeStart, keyboardResize, handle: screen.getByTestId('handle') }; + }; + + it('is a focusable separator that surfaces the pane size to assistive tech', () => { + const { handle } = renderHandle({ title: 'Drag to resize message inspector' }); + + // A handle dragged horizontally is a VERTICAL bar between side-by-side panes. + expect(handle.getAttribute('role')).toBe('separator'); + expect(handle.getAttribute('aria-orientation')).toBe('vertical'); + expect(handle.getAttribute('tabindex')).toBe('0'); + expect(handle.getAttribute('aria-label')).toBe('Drag to resize message inspector'); + expect(handle.getAttribute('aria-valuenow')).toBe('300'); + expect(handle.getAttribute('aria-valuemin')).toBe('100'); + expect(handle.getAttribute('aria-valuemax')).toBe('800'); + }); + + it('a vertical-axis handle is a horizontal separator', () => { + const { handle } = renderHandle({ resizeAxis: 'vertical' }); + + expect(handle.getAttribute('aria-orientation')).toBe('horizontal'); + }); + + // Each press carries the pane as MEASURED, so the step starts from what is on screen rather than + // from a stored preference a responsive cap has overtaken (see useResizablePane.spec.tsx, which + // pins that end to end). jsdom lays nothing out, so the measurement here is `undefined` - the + // hook then falls back to the stored size, exactly as it did before. + it('horizontal axis: left/right arrows operate the splitter, the perpendicular pair is left alone', () => { + const { handle, keyboardResize } = renderHandle(); + + fireEvent.keyDown(handle, { key: 'ArrowLeft' }); + expect(keyboardResize).toHaveBeenLastCalledWith('toward-start', undefined); + fireEvent.keyDown(handle, { key: 'ArrowRight' }); + expect(keyboardResize).toHaveBeenLastCalledWith('toward-end', undefined); + + fireEvent.keyDown(handle, { key: 'ArrowUp' }); + fireEvent.keyDown(handle, { key: 'ArrowDown' }); + expect(keyboardResize).toHaveBeenCalledTimes(2); + }); + + it('vertical axis: up/down arrows operate the splitter', () => { + const { handle, keyboardResize } = renderHandle({ resizeAxis: 'vertical' }); + + fireEvent.keyDown(handle, { key: 'ArrowUp' }); + expect(keyboardResize).toHaveBeenLastCalledWith('toward-start', undefined); + fireEvent.keyDown(handle, { key: 'ArrowDown' }); + expect(keyboardResize).toHaveBeenLastCalledWith('toward-end', undefined); + + fireEvent.keyDown(handle, { key: 'ArrowLeft' }); + fireEvent.keyDown(handle, { key: 'ArrowRight' }); + expect(keyboardResize).toHaveBeenCalledTimes(2); + }); + + it('Home and End jump the splitter to the bounds', () => { + const { handle, keyboardResize } = renderHandle(); + + fireEvent.keyDown(handle, { key: 'Home' }); + expect(keyboardResize).toHaveBeenLastCalledWith('min', undefined); + fireEvent.keyDown(handle, { key: 'End' }); + expect(keyboardResize).toHaveBeenLastCalledWith('max', undefined); + }); + + it('without a pane it is a plain drag separator: no tab stop, no value, keys inert', () => { + // The ratio-based splits pass no pane. A focusable separator whose keys do nothing would be + // worse for a keyboard user than no tab stop at all. + const { handle } = renderHandle({ pane: undefined }); + + expect(handle.getAttribute('role')).toBe('separator'); + expect(handle.getAttribute('tabindex')).toBeNull(); + expect(handle.getAttribute('aria-valuenow')).toBeNull(); + expect(() => fireEvent.keyDown(handle, { key: 'ArrowLeft' })).not.toThrow(); + }); + + it('a primary-pointer press begins the drag from the pressed coordinate', () => { + const { handle, onResizeStart } = renderHandle(); + + pointerDown(handle, { button: 0, clientX: 42, clientY: 7 }); + + // Horizontal axis reads clientX; the second argument is the parent's measured size, which + // jsdom reports as 0 - the wiring under test is which coordinate is picked and that a + // POINTER press starts the drag at all (a touch never fires mousedown). + expect(onResizeStart).toHaveBeenCalledTimes(1); + expect(onResizeStart).toHaveBeenCalledWith(42, 0); + }); + + it('a vertical-axis press reads the vertical coordinate', () => { + const { handle, onResizeStart } = renderHandle({ resizeAxis: 'vertical' }); + + pointerDown(handle, { button: 0, clientX: 42, clientY: 7 }); + + expect(onResizeStart).toHaveBeenCalledWith(7, 0); + }); + + it('a secondary-button press starts nothing', () => { + const { handle, onResizeStart } = renderHandle(); + + pointerDown(handle, { button: 2, clientX: 42 }); + + expect(onResizeStart).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/components/ui/resizable/PaneResizeHandle.tsx b/ui/components/ui/resizable/PaneResizeHandle.tsx index affba11a7..b7d59fa77 100644 --- a/ui/components/ui/resizable/PaneResizeHandle.tsx +++ b/ui/components/ui/resizable/PaneResizeHandle.tsx @@ -1,21 +1,142 @@ -import React from 'react'; +import React, { useLayoutEffect, useRef, useState } from 'react'; import s from './PaneResizeHandle.module.css'; +import { PaneKeyboardResizeIntent } from './useResizablePane'; + +/** + * What keyboard operability needs to know about the pane being sized. `UseResizablePane` satisfies + * it as-is, so a call site wires the whole thing with `pane={thePane}`. Without it the handle is a + * drag-only, non-focusable separator (the ratio-based splits have no fixed value to report). + */ +export type PaneResizeHandleKeyboardTarget = { + size: number; + minSize: number; + maxSize: number; + keyboardResize: (intent: PaneKeyboardResizeIntent, renderedSize?: number) => void; +}; export const PaneResizeHandle: React.FC<{ - onResizeStart: (startClientX: number, containerWidth: number) => void, + onResizeStart: (startClientPosition: number, containerSize: number) => void, + resizeAxis?: 'horizontal' | 'vertical', mode?: 'inline' | 'edge', + edgeSide?: 'start' | 'end', title?: string, -}> = ({ onResizeStart, mode = 'inline', title = 'Drag to resize' }) => ( - <div - className={`${s.PaneResizeHandle} ${mode === 'edge' ? s.Edge : ''}`} - title={title} - onMouseDown={(e) => { - e.preventDefault(); - e.stopPropagation(); - const containerWidth = e.currentTarget.parentElement?.getBoundingClientRect().width ?? 0; - onResizeStart(e.clientX, containerWidth); - }} - /> -); + testId?: string, + className?: string, + pane?: PaneResizeHandleKeyboardTarget, +}> = ({ + onResizeStart, + resizeAxis = 'horizontal', + mode = 'inline', + edgeSide = 'end', + title = 'Drag to resize', + testId, + className, + pane, +}) => { + // ARIA orientation describes the SEPARATOR, not the drag axis: a handle dragged horizontally is + // a vertical bar between side-by-side panes, and vice versa. The arrow keys follow the same + // geometry - the pair that moves along the drag axis operates the splitter (WAI-ARIA window + // splitter pattern), the perpendicular pair is left alone. + const orientation = resizeAxis === 'vertical' ? 'horizontal' : 'vertical'; + + const handleRef = useRef<HTMLDivElement | null>(null); + + // The pane as LAID OUT, which a responsive cap can make smaller than the stored preference the + // pane hook reports. Both halves of the control need it: an arrow key must step from what is on + // screen (the pointer path already does), and aria-valuenow must describe the separator where it + // actually sits. Measured from the same element the pointer path measures - the handle's pane. + const measurePane = (): number | undefined => { + const rect = handleRef.current?.parentElement?.getBoundingClientRect(); + const measured = resizeAxis === 'vertical' ? rect?.height : rect?.width; + return measured !== undefined && measured > 0 ? measured : undefined; + }; + + const [renderedSize, setRenderedSize] = useState<number | undefined>(undefined); + // After every size change, and on arrival - a focused separator is when the value is announced. + // No observer: a viewport change with no size change is picked up by the focus that has to + // precede any keystroke anyway, which keeps this to two cheap measurements. + useLayoutEffect(() => { + if (pane !== undefined) { + setRenderedSize(measurePane()); + } + }, [pane?.size, resizeAxis]); + + // What the pane REPORTS is the fallback: before the first layout, and wherever nothing measures + // (the value would otherwise read 0 and describe a collapsed pane that is not there). + const reportedSize = renderedSize ?? pane?.size; + + const keyboardIntent = (key: string): PaneKeyboardResizeIntent | undefined => { + if (key === 'Home') { + return 'min'; + } + if (key === 'End') { + return 'max'; + } + if (resizeAxis === 'vertical') { + return key === 'ArrowUp' ? 'toward-start' : key === 'ArrowDown' ? 'toward-end' : undefined; + } + return key === 'ArrowLeft' ? 'toward-start' : key === 'ArrowRight' ? 'toward-end' : undefined; + }; + + return ( + <div + ref={handleRef} + className={[ + s.PaneResizeHandle, + resizeAxis === 'vertical' ? s.Vertical : s.Horizontal, + mode === 'edge' ? s.Edge : '', + mode === 'edge' && edgeSide === 'start' ? s.EdgeStart : '', + mode === 'edge' && edgeSide === 'end' ? s.EdgeEnd : '', + className ?? '', + ].filter(Boolean).join(' ')} + data-testid={testId} + data-resize-axis={resizeAxis} + title={title} + role="separator" + aria-orientation={orientation} + aria-label={title} + // Focusable only when the keys actually do something: a tab stop that swallows arrows and + // does nothing is worse for a keyboard user than no stop at all. + tabIndex={pane === undefined ? undefined : 0} + aria-valuenow={reportedSize === undefined ? undefined : Math.round(reportedSize)} + aria-valuemin={pane === undefined ? undefined : Math.round(pane.minSize)} + aria-valuemax={pane === undefined ? undefined : Math.round(pane.maxSize)} + onFocus={() => { + if (pane !== undefined) { + setRenderedSize(measurePane()); + } + }} + // pointerdown, not mousedown: a touch drag fires only pointer events, and the drag helper + // listens for pointermove/pointerup to match. The stylesheet sets `touch-action: none` so + // the browser does not claim the gesture for scrolling first. + onPointerDown={(e) => { + if (e.button !== 0) { + return; + } + e.preventDefault(); + e.stopPropagation(); + const containerRect = e.currentTarget.parentElement?.getBoundingClientRect(); + const startClientPosition = resizeAxis === 'vertical' ? e.clientY : e.clientX; + const containerSize = resizeAxis === 'vertical' ? containerRect?.height ?? 0 : containerRect?.width ?? 0; + onResizeStart(startClientPosition, containerSize); + }} + onKeyDown={(e) => { + if (pane === undefined) { + return; + } + const intent = keyboardIntent(e.key); + if (intent === undefined) { + return; + } + // The arrows resize the pane; without this they would also scroll whatever contains it. + e.preventDefault(); + e.stopPropagation(); + // Measured now rather than read from state: the layout can have changed since the last + // render, and a step from a stale number is the whole defect this guards. + pane.keyboardResize(intent, measurePane()); + }} + /> + ); +}; export default PaneResizeHandle; diff --git a/ui/components/ui/resizable/dragResize.spec.ts b/ui/components/ui/resizable/dragResize.spec.ts new file mode 100644 index 000000000..eb438cecd --- /dev/null +++ b/ui/components/ui/resizable/dragResize.spec.ts @@ -0,0 +1,176 @@ +/** @jest-environment jsdom */ + +import { beginHorizontalDragResize, beginVerticalDragResize } from './dragResize'; + +describe('dragResize', () => { + let nextFrameId: number; + let animationFrames: Map<number, FrameRequestCallback>; + let originalRequestAnimationFrame: typeof globalThis.requestAnimationFrame | undefined; + let originalCancelAnimationFrame: typeof globalThis.cancelAnimationFrame | undefined; + + const flushAnimationFrames = () => { + const pending = Array.from(animationFrames.values()); + animationFrames.clear(); + pending.forEach((callback) => callback(0)); + }; + + beforeEach(() => { + nextFrameId = 1; + animationFrames = new Map(); + originalRequestAnimationFrame = globalThis.requestAnimationFrame; + originalCancelAnimationFrame = globalThis.cancelAnimationFrame; + + globalThis.requestAnimationFrame = jest.fn((callback: FrameRequestCallback) => { + const frameId = nextFrameId++; + animationFrames.set(frameId, callback); + return frameId; + }); + globalThis.cancelAnimationFrame = jest.fn((frameId: number) => { + animationFrames.delete(frameId); + }); + }); + + afterEach(() => { + // Also releases listeners if a failed assertion interrupted a test before its own pointerup. + document.dispatchEvent(new MouseEvent('pointerup')); + document.body.style.removeProperty('cursor'); + document.body.style.removeProperty('user-select'); + + if (originalRequestAnimationFrame) { + globalThis.requestAnimationFrame = originalRequestAnimationFrame; + } else { + delete (globalThis as Partial<typeof globalThis>).requestAnimationFrame; + } + if (originalCancelAnimationFrame) { + globalThis.cancelAnimationFrame = originalCancelAnimationFrame; + } else { + delete (globalThis as Partial<typeof globalThis>).cancelAnimationFrame; + } + }); + + // The drag listens for POINTER events (touch fires no mouse events mid-drag). jsdom has no + // PointerEvent constructor, and the handlers only read MouseEvent fields - so a MouseEvent + // dispatched under the pointer event TYPE drives them exactly like a real pointer would. + it('uses vertical pointer movement and reverses it for a bottom pane', () => { + const onChange = jest.fn(); + const onEnd = jest.fn(); + + beginVerticalDragResize({ + startClientY: 200, + startValue: 300, + min: 100, + max: 700, + sign: -1, + onChange, + onEnd, + }); + + // X deliberately moves the opposite way: only the Y coordinate may affect this resize. + document.dispatchEvent(new MouseEvent('pointermove', { clientX: -1_000, clientY: 150 })); + flushAnimationFrames(); + + expect(onChange).toHaveBeenLastCalledWith(350); + expect(document.body.style.cursor).toBe('row-resize'); + + document.dispatchEvent(new MouseEvent('pointerup')); + expect(onEnd).toHaveBeenCalledWith(350); + }); + + it('clamps a vertical resize at both bounds', () => { + const onChange = jest.fn(); + + beginVerticalDragResize({ + startClientY: 100, + startValue: 200, + min: 120, + max: 280, + onChange, + }); + + document.dispatchEvent(new MouseEvent('pointermove', { clientY: -100 })); + flushAnimationFrames(); + document.dispatchEvent(new MouseEvent('pointermove', { clientY: 500 })); + flushAnimationFrames(); + + expect(onChange.mock.calls.map(([value]) => value)).toEqual([120, 280]); + }); + + // `pointercancel`, not `pointerup`: the browser fires it when it takes the gesture away - a + // touch interrupted by a system gesture, palm rejection, the pointer leaving the surface. No + // `pointerup` follows it, so a drag that only listens for `pointerup` never ends: the document + // keeps its move/up listeners and the page keeps `user-select: none` and the resize cursor, with + // no gesture left that could clear them. On touch - the only place cancellation is common - that + // is exactly when an unselectable page with a resize cursor is worst. + it('ends the drag and restores the page when the pointer gesture is CANCELLED', () => { + const onChange = jest.fn(); + const onEnd = jest.fn(); + document.body.style.cursor = 'crosshair'; + document.body.style.userSelect = 'text'; + + beginHorizontalDragResize({ + startClientX: 10, + startValue: 200, + min: 100, + max: 400, + onChange, + onEnd, + }); + + document.dispatchEvent(new MouseEvent('pointermove', { clientX: 85 })); + document.dispatchEvent(new MouseEvent('pointercancel')); + + // The size the user last saw is committed and persisted - the cancellation ends the drag, it + // does not undo it - and `onEnd` runs, which is what releases the column header's suppressed + // sort click (useColumnWidths). + expect(onChange).toHaveBeenLastCalledWith(275); + expect(onEnd).toHaveBeenCalledWith(275); + + // Nothing of the drag is left on the page... + expect(document.body.style.cursor).toBe('crosshair'); + expect(document.body.style.userSelect).toBe('text'); + + // ...and no listener survives it: a later pointer stroke must not still be resizing. + document.dispatchEvent(new MouseEvent('pointermove', { clientX: 300 })); + flushAnimationFrames(); + expect(onChange).toHaveBeenCalledTimes(1); + expect(onEnd).toHaveBeenCalledTimes(1); + }); + + it('flushes the latest pending value on pointerup and cleans up the drag', () => { + const onChange = jest.fn(); + const onEnd = jest.fn(); + document.body.style.cursor = 'crosshair'; + document.body.style.userSelect = 'text'; + + beginHorizontalDragResize({ + startClientX: 10, + startValue: 200, + min: 100, + max: 400, + onChange, + onEnd, + }); + + expect(document.body.style.cursor).toBe('col-resize'); + expect(document.body.style.userSelect).toBe('none'); + + document.dispatchEvent(new MouseEvent('pointermove', { clientX: 85 })); + expect(onChange).not.toHaveBeenCalled(); + + document.dispatchEvent(new MouseEvent('pointerup')); + + expect(globalThis.cancelAnimationFrame).toHaveBeenCalledTimes(1); + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange).toHaveBeenCalledWith(275); + expect(onEnd).toHaveBeenCalledWith(275); + expect(document.body.style.cursor).toBe('crosshair'); + expect(document.body.style.userSelect).toBe('text'); + + document.dispatchEvent(new MouseEvent('pointermove', { clientX: 200 })); + document.dispatchEvent(new MouseEvent('pointerup')); + flushAnimationFrames(); + + expect(onChange).toHaveBeenCalledTimes(1); + expect(onEnd).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/components/ui/resizable/dragResize.ts b/ui/components/ui/resizable/dragResize.ts index 4a740454c..b17a70af7 100644 --- a/ui/components/ui/resizable/dragResize.ts +++ b/ui/components/ui/resizable/dragResize.ts @@ -1,5 +1,4 @@ -export type DragResizeOptions = { - startClientX: number; +type CommonDragResizeOptions = { startValue: number; min: number; max: number; @@ -8,13 +7,34 @@ export type DragResizeOptions = { onEnd?: (value: number) => void; }; -export function beginHorizontalDragResize(opts: DragResizeOptions): void { +export type DragResizeOptions = CommonDragResizeOptions & { + startClientX: number; +}; + +export type VerticalDragResizeOptions = CommonDragResizeOptions & { + startClientY: number; +}; + +function beginDragResize( + opts: CommonDragResizeOptions, + startClientPosition: number, + currentClientPosition: (event: MouseEvent) => number, + cursor: 'col-resize' | 'row-resize', +): void { const sign = opts.sign ?? 1; let raf = 0; let latest = opts.startValue; + const previousCursor = document.body.style.cursor; + const previousUserSelect = document.body.style.userSelect; + // POINTER events, not mouse events: a mouse fires both, but a touch drag fires only pointer + // events - with mouse listeners a touch could start a resize (via a pointerdown handler) and + // then never move or end it. The handlers only read client coordinates, which every pointer + // event carries (PointerEvent extends MouseEvent). const onMove = (e: MouseEvent) => { - latest = Math.min(opts.max, Math.max(opts.min, Math.round(opts.startValue + sign * (e.clientX - opts.startClientX)))); + latest = Math.min(opts.max, Math.max(opts.min, Math.round( + opts.startValue + sign * (currentClientPosition(e) - startClientPosition) + ))); if (!raf) { raf = requestAnimationFrame(() => { raf = 0; @@ -23,18 +43,45 @@ export function beginHorizontalDragResize(opts: DragResizeOptions): void { } }; - const onUp = () => { - if (raf) { cancelAnimationFrame(raf); } + // THE one way a drag ends, for every terminal event: commit what the user last saw, then undo + // everything the drag put on the document. It has to be idempotent - `pointercancel` is often + // followed by nothing at all, but a real browser can also deliver blur alongside an end - and it + // must run for `pointercancel` too. That event is how the browser TAKES a gesture away (a touch + // interrupted by a system gesture, palm rejection); no `pointerup` follows it, so ending only on + // `pointerup` left the document listeners installed and the page stuck unselectable under a + // resize cursor, with no remaining gesture that could clear them. + let isEnded = false; + const onEnd = () => { + if (isEnded) { + return; + } + isEnded = true; + if (raf) { + cancelAnimationFrame(raf); + raf = 0; + } opts.onChange(latest); opts.onEnd?.(latest); - document.removeEventListener('mousemove', onMove); - document.removeEventListener('mouseup', onUp); - document.body.style.removeProperty('cursor'); - document.body.style.removeProperty('user-select'); + document.removeEventListener('pointermove', onMove); + document.removeEventListener('pointerup', onEnd); + document.removeEventListener('pointercancel', onEnd); + window.removeEventListener('blur', onEnd); + document.body.style.cursor = previousCursor; + document.body.style.userSelect = previousUserSelect; }; - document.addEventListener('mousemove', onMove); - document.addEventListener('mouseup', onUp); - document.body.style.cursor = 'col-resize'; + document.addEventListener('pointermove', onMove); + document.addEventListener('pointerup', onEnd); + document.addEventListener('pointercancel', onEnd); + window.addEventListener('blur', onEnd); + document.body.style.cursor = cursor; document.body.style.userSelect = 'none'; } + +export function beginHorizontalDragResize(opts: DragResizeOptions): void { + beginDragResize(opts, opts.startClientX, event => event.clientX, 'col-resize'); +} + +export function beginVerticalDragResize(opts: VerticalDragResizeOptions): void { + beginDragResize(opts, opts.startClientY, event => event.clientY, 'row-resize'); +} diff --git a/ui/components/ui/resizable/useColumnOrder.spec.ts b/ui/components/ui/resizable/useColumnOrder.spec.ts new file mode 100644 index 000000000..aa99c1cc4 --- /dev/null +++ b/ui/components/ui/resizable/useColumnOrder.spec.ts @@ -0,0 +1,70 @@ +/** + * The pure halves of draggable column reorder: what a saved order means once the column set has + * changed underneath it, and what one drag does to the list. The DOM half (drag events on the + * shared Table's headers) is pinned end to end; these are the rules it applies. + */ +import { moveColumnTo, reconcileColumnOrder } from './useColumnOrder'; + +describe('reconcileColumnOrder', () => { + const defaults = ['a', 'b', 'c', 'd']; + + it('an empty store means the default order', () => { + expect(reconcileColumnOrder(defaults, [])).toEqual(['a', 'b', 'c', 'd']); + }); + + it('a full saved order wins as saved', () => { + expect(reconcileColumnOrder(defaults, ['d', 'b', 'a', 'c'])).toEqual(['d', 'b', 'a', 'c']); + }); + + it('a key that no longer exists is dropped, not crashed on', () => { + expect(reconcileColumnOrder(defaults, ['gone', 'c', 'a', 'b', 'd'])).toEqual(['c', 'a', 'b', 'd']); + }); + + it('a NEW column appears next to its default neighbour, not exiled to the end', () => { + // The saved order predates column 'c'; 'c' follows 'b' by default, so it lands after 'b'. + expect(reconcileColumnOrder(defaults, ['d', 'b', 'a'])).toEqual(['d', 'b', 'c', 'a']); + }); + + it('a new FIRST column with no surviving predecessor lands first', () => { + expect(reconcileColumnOrder(defaults, ['c', 'b', 'd'])).toEqual(['a', 'c', 'b', 'd']); + }); + + it('a duplicated stored key (a corrupt store) collapses to one', () => { + expect(reconcileColumnOrder(defaults, ['b', 'b', 'a', 'c', 'd'])).toEqual(['b', 'a', 'c', 'd']); + }); +}); + +describe('moveColumnTo', () => { + const order = ['a', 'b', 'c', 'd']; + + // One drop = "put the dragged column where the target is". Dropping on an EARLIER column pushes + // the target back, dropping on a LATER one pulls it forward - so the dragged column always ends + // up at the index the target had, and every position is reachable by some drop. The whole header + // is the drop target and it highlights as one, which is what makes that the reading. + it('takes the position of a target to its LEFT', () => { + expect(moveColumnTo(order, 'd', 'b')).toEqual(['a', 'd', 'b', 'c']); + }); + + it('takes the position of a target to its RIGHT', () => { + expect(moveColumnTo(order, 'a', 'c')).toEqual(['b', 'c', 'a', 'd']); + }); + + // The position the old "always insert before the target" rule could not express: there is no + // header to the right of the last one, so nothing could ever be dropped past it. + it('dropping on the LAST column puts the dragged column last', () => { + expect(moveColumnTo(order, 'a', 'd')).toEqual(['b', 'c', 'd', 'a']); + }); + + it('dropping on the FIRST column puts the dragged column first', () => { + expect(moveColumnTo(order, 'c', 'a')).toEqual(['c', 'a', 'b', 'd']); + }); + + it('dropping a column on itself changes nothing', () => { + expect(moveColumnTo(order, 'c', 'c')).toEqual(order); + }); + + it('an unknown dragged or target key changes nothing', () => { + expect(moveColumnTo(order, 'x' as never, 'b')).toEqual(order); + expect(moveColumnTo(order, 'b', 'x' as never)).toEqual(order); + }); +}); diff --git a/ui/components/ui/resizable/useColumnOrder.ts b/ui/components/ui/resizable/useColumnOrder.ts new file mode 100644 index 000000000..7acf6194a --- /dev/null +++ b/ui/components/ui/resizable/useColumnOrder.ts @@ -0,0 +1,76 @@ +import { useCallback } from 'react'; +import useLocalStorage from 'use-local-storage-state'; + +/** + * Persisted per-table column order - the reorder sibling of `useColumnWidths`, stored under + * `table:{tableId}:column-order` as the full ordered list of column keys. + * + * The stored order is reconciled against the CURRENT default order on every read: keys that no + * longer exist are dropped, and keys the build added since the order was saved are inserted next + * to their default neighbours - so a saved order never hides a new column and never crashes on a + * removed one. + */ +export function useColumnOrder<CK extends string>( + tableId: string, + defaultOrder: CK[], +): { order: CK[]; moveColumn: (dragged: CK, target: CK) => void } { + const [stored, setStored] = useLocalStorage<string[]>(`table:${tableId}:column-order`, { defaultValue: [] }); + + const order = reconcileColumnOrder(defaultOrder, stored); + + const moveColumn = useCallback( + (dragged: CK, target: CK) => { + setStored((prev) => moveColumnTo(reconcileColumnOrder(defaultOrder, prev ?? []), dragged, target)); + }, + // The default order is a render-stable list of literals in every caller. + // eslint-disable-next-line react-hooks/exhaustive-deps + [setStored, defaultOrder.join('\u0000')], + ); + + return { order, moveColumn }; +} + +/** The stored order, made safe against a changed column set. Exported for tests. */ +export function reconcileColumnOrder<CK extends string>(defaultOrder: CK[], stored: string[]): CK[] { + const known = stored.filter((key): key is CK => (defaultOrder as string[]).includes(key)); + const result: CK[] = Array.from(new Set(known)); + defaultOrder.forEach((key) => { + if (result.includes(key)) { + return; + } + // A column the saved order has never seen: place it after its nearest PRECEDING default + // neighbour that survived, so a new column appears where the design put it instead of + // being exiled to the end. + const defaultIndex = defaultOrder.indexOf(key); + let insertAt = 0; + for (let i = defaultIndex - 1; i >= 0; i--) { + const at = result.indexOf(defaultOrder[i]); + if (at >= 0) { + insertAt = at + 1; + break; + } + } + result.splice(insertAt, 0, key); + }); + return result; +} + +/** + * `dragged` moved to `target`'s position, everything else stable. + * + * A drop lands on ONE header and that whole header highlights, so the gesture reads as "put the + * dragged column where this one is" - and that is what it does: dropping on a later column pushes + * the target and everything between it back, dropping on an earlier one pushes them forward. + * + * Reading every drop as "insert BEFORE the target" instead would leave the LAST position + * unreachable by any gesture, because reaching it would need a header to the right of the last one + * to drop on, and there is none. + */ +export function moveColumnTo<CK extends string>(order: CK[], dragged: CK, target: CK): CK[] { + const at = order.indexOf(target); + if (dragged === target || at < 0 || !order.includes(dragged)) { + return order; + } + const without = order.filter((key) => key !== dragged); + return [...without.slice(0, at), dragged, ...without.slice(at)]; +} diff --git a/ui/components/ui/resizable/useResizablePane.spec.tsx b/ui/components/ui/resizable/useResizablePane.spec.tsx new file mode 100644 index 000000000..1ab34db60 --- /dev/null +++ b/ui/components/ui/resizable/useResizablePane.spec.tsx @@ -0,0 +1,233 @@ +/** @jest-environment jsdom */ + +import React from 'react'; +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { PaneKeyboardResizeIntent, ResizablePaneOptions, useResizablePane } from './useResizablePane'; +import PaneResizeHandle from './PaneResizeHandle'; + +// The drag helper listens for POINTER events (touch fires no mouse events mid-drag). jsdom has no +// PointerEvent constructor, and the handlers only read MouseEvent fields - so a MouseEvent +// dispatched under the pointer event TYPE drives them exactly like a real pointer would. +const pointerMove = (clientY: number) => fireEvent(document, new MouseEvent('pointermove', { clientY })); +const pointerUp = () => fireEvent(document, new MouseEvent('pointerup')); + +type HarnessProps = { + paneId: string; + options: ResizablePaneOptions; + renderedSize?: number; +}; + +const keyboardIntents: PaneKeyboardResizeIntent[] = ['toward-start', 'toward-end', 'min', 'max']; + +function Harness({ paneId, options, renderedSize }: HarnessProps) { + const pane = useResizablePane(paneId, options); + + return ( + <div> + <button + data-testid="resize-handle" + data-size={pane.size} + onMouseDown={(event) => pane.startResize(event.clientY, renderedSize)} + > + {pane.size} + </button> + {keyboardIntents.map((intent) => ( + <button key={intent} data-testid={`kb-${intent}`} onClick={() => pane.keyboardResize(intent)} /> + ))} + </div> + ); +} + +const options: ResizablePaneOptions = { + defaultSize: 400, + minSize: 100, + maxSize: 800, + resizeAxis: 'vertical', + side: 'bottom', +}; + +describe('useResizablePane', () => { + beforeEach(() => { + window.localStorage.clear(); + }); + + afterEach(() => { + // Also releases listeners if a failed assertion interrupted a test before its own pointerup. + document.dispatchEvent(new MouseEvent('pointerup')); + cleanup(); + window.localStorage.clear(); + }); + + it.each([ + ['a value above the maximum', '9999', 800], + ['a value below the minimum', '-50', 100], + ['a non-finite value', 'null', 400], + ])('normalizes %s in both the rendered and persisted preference', async (_case, storedValue, expected) => { + const paneId = `normalization-${expected}-${storedValue}`; + const storageKey = `pane:${paneId}:size`; + window.localStorage.setItem(storageKey, storedValue); + + render(<Harness paneId={paneId} options={options} />); + + expect(screen.getByTestId('resize-handle').dataset.size).toBe(String(expected)); + await waitFor(() => expect(window.localStorage.getItem(storageKey)).toBe(String(expected))); + }); + + it('starts a bottom-panel drag from its responsive rendered height, not its larger stored preference', async () => { + const paneId = 'responsive-bottom-panel'; + const storageKey = `pane:${paneId}:size`; + window.localStorage.setItem(storageKey, '700'); + + render(<Harness paneId={paneId} options={options} renderedSize={300} />); + const handle = screen.getByTestId('resize-handle'); + expect(handle.dataset.size).toBe('700'); + + // Moving a bottom panel's top edge 50px upward grows the visible 300px panel to 350px. + // If the hook incorrectly started from the stored preference this would jump to 750px. + // The release flushes the final value regardless of the animation-frame queue. + fireEvent.mouseDown(handle, { clientY: 500 }); + pointerMove(450); + pointerUp(); + + await waitFor(() => expect(handle.dataset.size).toBe('350')); + expect(window.localStorage.getItem(storageKey)).toBe('350'); + }); + + it('applies drag movement through the animation-frame batch, before the pointer is released', () => { + // The path an actual drag uses: every pointermove within a frame only records the latest + // position, and ONE queued animation-frame callback commits it. An earlier version of this + // spec stubbed requestAnimationFrame to a no-op, so this path never ran and only the + // release-time flush above was covered. Jest's modern fake timers drive the frame. + jest.useFakeTimers(); + try { + render(<Harness paneId="raf-batch" options={options} />); + const handle = screen.getByTestId('resize-handle'); + + fireEvent.mouseDown(handle, { clientY: 500 }); + pointerMove(470); + pointerMove(450); + // Batched: nothing is applied until the frame fires. + expect(handle.dataset.size).toBe('400'); + + act(() => { + // Modern fake timers intercept requestAnimationFrame and run its callbacks on a 16 ms + // cadence (jest 29 has no advanceTimersToNextFrame yet). + jest.advanceTimersByTime(16); + }); + + // The one frame applies the LATEST move (up 50px grows the bottom pane to 450), not one + // update per move - and the mid-drag value is already persisted. + expect(handle.dataset.size).toBe('450'); + expect(window.localStorage.getItem('pane:raf-batch:size')).toBe('450'); + } finally { + act(() => { + pointerUp(); + }); + jest.useRealTimers(); + } + }); + + describe('keyboard resize', () => { + it('one step per press, with the splitter direction mapped to what it does to the pane', () => { + render(<Harness paneId="kb-bottom" options={options} />); + const handle = screen.getByTestId('resize-handle'); + expect(handle.dataset.size).toBe('400'); + + // A bottom pane's splitter is its TOP edge: moving it toward-start (up) makes the pane taller. + fireEvent.click(screen.getByTestId('kb-toward-start')); + expect(handle.dataset.size).toBe('416'); + + fireEvent.click(screen.getByTestId('kb-toward-end')); + expect(handle.dataset.size).toBe('400'); + expect(window.localStorage.getItem('pane:kb-bottom:size')).toBe('400'); + }); + + it('reverses for a left pane, whose splitter moves toward-end to grow it', () => { + render(<Harness paneId="kb-left" options={{ defaultSize: 400, minSize: 100, maxSize: 800, side: 'left' }} />); + + fireEvent.click(screen.getByTestId('kb-toward-end')); + expect(screen.getByTestId('resize-handle').dataset.size).toBe('416'); + }); + + it('Home/End land on the bounds, and steps clamp there', () => { + render(<Harness paneId="kb-bounds" options={options} />); + const handle = screen.getByTestId('resize-handle'); + + fireEvent.click(screen.getByTestId('kb-max')); + expect(handle.dataset.size).toBe('800'); + // Growing past the maximum stays at the maximum. + fireEvent.click(screen.getByTestId('kb-toward-start')); + expect(handle.dataset.size).toBe('800'); + + fireEvent.click(screen.getByTestId('kb-min')); + expect(handle.dataset.size).toBe('100'); + // Shrinking past the minimum stays at the minimum. + fireEvent.click(screen.getByTestId('kb-toward-end')); + expect(handle.dataset.size).toBe('100'); + }); + }); + + /** + * A viewport SMALLER than the display the preference was saved on. Both consumer-session panes + * are capped responsively - Tools to 85% of the viewport height, the message inspector to 85% of + * its width - so the pane the user can see is smaller than the stored number while the + * preference itself is deliberately kept. + * + * The pointer path already begins from the measured pane. The keyboard half is the same control + * and must behave the same way: an arrow moves the splitter from where it IS. Walking a stale + * 700 down toward a 300px pane instead means the first ~25 presses move nothing on screen, and + * the value announced to assistive tech describes a pane that is not there. + * + * Driven through the real handle, because the measurement is the handle's half of the job. + */ + describe('a pane capped below its stored preference', () => { + const stubbedRect = (size: number) => (el: HTMLDivElement | null) => { + if (el !== null) { + // jsdom lays nothing out; this is the responsive cap the browser would report. + el.getBoundingClientRect = () => ({ height: size, width: size } as DOMRect); + } + }; + + function ResponsiveHarness({ renderedSize }: { renderedSize: number }) { + const pane = useResizablePane('responsive-keyboard', options); + + return ( + <div ref={stubbedRect(renderedSize)} data-testid="pane"> + <PaneResizeHandle testId="handle" resizeAxis="vertical" onResizeStart={pane.startResize} pane={pane} /> + <span data-testid="size">{pane.size}</span> + </div> + ); + } + + const renderResponsive = (storedSize: string, renderedSize: number) => { + window.localStorage.setItem('pane:responsive-keyboard:size', storedSize); + render(<ResponsiveHarness renderedSize={renderedSize} />); + return { handle: screen.getByTestId('handle'), size: () => screen.getByTestId('size').textContent }; + }; + + it('an arrow moves the splitter from the pane on SCREEN, not from the stored preference', () => { + const { handle, size } = renderResponsive('700', 300); + + // A bottom pane's splitter is its top edge: ArrowUp grows it, by one step from the 300 it + // actually occupies. Starting from the stored 700 would set 716 and change nothing visible. + fireEvent.keyDown(handle, { key: 'ArrowUp' }); + expect(size()).toBe('316'); + + // ...and shrinking is immediate for the same reason - it does not have to walk 700 down first. + fireEvent.keyDown(handle, { key: 'ArrowDown' }); + expect(size()).toBe('284'); + }); + + it('announces the size the pane really has', () => { + const { handle } = renderResponsive('700', 300); + + act(() => handle.focus()); + + // aria-valuenow describes the separator's position, so it must be the measured pane; the + // bounds stay the ones a drag or an arrow can actually reach. + expect(handle.getAttribute('aria-valuenow')).toBe('300'); + expect(handle.getAttribute('aria-valuemin')).toBe('100'); + expect(handle.getAttribute('aria-valuemax')).toBe('800'); + }); + }); +}); diff --git a/ui/components/ui/resizable/useResizablePane.ts b/ui/components/ui/resizable/useResizablePane.ts index 874acc85d..7d20f832e 100644 --- a/ui/components/ui/resizable/useResizablePane.ts +++ b/ui/components/ui/resizable/useResizablePane.ts @@ -1,38 +1,101 @@ -import { useCallback, useRef } from 'react'; +import { useCallback, useEffect, useRef } from 'react'; import useLocalStorage from 'use-local-storage-state'; -import { beginHorizontalDragResize } from './dragResize'; +import { beginHorizontalDragResize, beginVerticalDragResize } from './dragResize'; + +/** + * A keyboard nudge on a resize handle, expressed in SCREEN direction - `toward-start` is toward + * the smaller client coordinate (left/up), `toward-end` the larger (right/down) - because that is + * what an arrow key means. The hook owns the pane's side, so only it can say whether moving the + * splitter toward the start grows the pane (a right/bottom pane) or shrinks it (a left/top one). + */ +export type PaneKeyboardResizeIntent = 'toward-start' | 'toward-end' | 'min' | 'max'; + +/** How far one arrow-key press moves the splitter. Held keys repeat, so fine beats fast. */ +const keyboardResizeStep = 16; + +/** + * What a resize begins from. A responsive layout may cap the RENDERED pane below its stored + * preference (both consumer-session panes are capped to 85% of the viewport), and a move has to + * start from what the user can actually see - otherwise the first presses/pixels are spent walking + * a number that is nowhere on screen, and the control feels inert. The preference itself is kept: + * only the starting point moves. + */ +function startFromRendered(renderedSize: number | undefined, storedSize: number): number { + return renderedSize !== undefined && Number.isFinite(renderedSize) && renderedSize > 0 ? renderedSize : storedSize; +} export type UseResizablePane = { size: number; - startResize: (startClientX: number) => void; + minSize: number; + maxSize: number; + startResize: (startClientPosition: number, renderedSize?: number) => void; + keyboardResize: (intent: PaneKeyboardResizeIntent, renderedSize?: number) => void; }; export type ResizablePaneOptions = { defaultSize: number; minSize: number; maxSize: number; - side?: 'left' | 'right'; + resizeAxis?: 'horizontal' | 'vertical'; + side?: 'left' | 'right' | 'top' | 'bottom'; }; export function useResizablePane(paneId: string, opts: ResizablePaneOptions): UseResizablePane { - const [size, setSize] = useLocalStorage<number>(`pane:${paneId}:size`, { defaultValue: opts.defaultSize }); + const [storedSize, setSize] = useLocalStorage<number>(`pane:${paneId}:size`, { defaultValue: opts.defaultSize }); + const fallbackSize = Math.min(opts.maxSize, Math.max(opts.minSize, opts.defaultSize)); + const size = Number.isFinite(storedSize) + ? Math.min(opts.maxSize, Math.max(opts.minSize, storedSize)) + : fallbackSize; + + // Old/corrupt preferences and values written on a larger layout must never make a pane + // permanently unreachable. Keep the persisted value inside the same bounds used while dragging. + useEffect(() => { + if (storedSize !== size) { + setSize(size); + } + }, [size, storedSize, setSize]); const sizeRef = useRef(size); sizeRef.current = size; const { minSize, maxSize } = opts; - const sign = opts.side === 'right' ? -1 : 1; + const resizeAxis = opts.resizeAxis ?? (opts.side === 'top' || opts.side === 'bottom' ? 'vertical' : 'horizontal'); + const sign: 1 | -1 = opts.side === 'right' || opts.side === 'bottom' ? -1 : 1; - const startResize = useCallback((startClientX: number) => { - beginHorizontalDragResize({ - startClientX, - startValue: sizeRef.current, + const startResize = useCallback((startClientPosition: number, renderedSize?: number) => { + const common = { + startValue: startFromRendered(renderedSize, sizeRef.current), min: minSize, max: maxSize, sign, onChange: setSize, - }); + }; + + if (resizeAxis === 'vertical') { + beginVerticalDragResize({ ...common, startClientY: startClientPosition }); + } else { + beginHorizontalDragResize({ ...common, startClientX: startClientPosition }); + } + }, [minSize, maxSize, resizeAxis, sign, setSize]); + + // The keyboard half of the handle: one press moves the splitter one step in screen direction, + // through the same sign, starting point and clamping the drag uses, so an arrow can never take + // the pane where a drag could not. Home/End are absolute and land on the declared bounds - the + // pair aria-valuemin/max report - so a responsive cap does not enter into them. + const keyboardResize = useCallback((intent: PaneKeyboardResizeIntent, renderedSize?: number) => { + if (intent === 'min') { + setSize(minSize); + return; + } + if (intent === 'max') { + setSize(maxSize); + return; + } + + const direction = intent === 'toward-start' ? -1 : 1; + const from = startFromRendered(renderedSize, sizeRef.current); + setSize(Math.min(maxSize, Math.max(minSize, from + sign * direction * keyboardResizeStep))); }, [minSize, maxSize, sign, setSize]); - return { size, startResize }; + return { size, minSize, maxSize, startResize, keyboardResize }; } diff --git a/ui/jest.config.js b/ui/jest.config.js index 7c8762ca8..f8d2e0003 100644 --- a/ui/jest.config.js +++ b/ui/jest.config.js @@ -16,7 +16,9 @@ module.exports = { /// component tests (jsdom) can render real components without a CSS/SVG transform. moduleNameMapper: { "\\.(css|less|scss|sass)$": "<rootDir>/__mocks__/styleMock.js", - "\\.(svg|png|jpg|jpeg|gif|webp|ttf|woff2?)$": "<rootDir>/__mocks__/fileMock.js", + // `.md` is bundled with esbuild's `text` loader (see build.js) exactly like `.svg`, so it is + // an asset here too - without the stub jest tries to PARSE the markdown as JavaScript. + "\\.(svg|png|jpg|jpeg|gif|webp|ttf|woff2?|md)$": "<rootDir>/__mocks__/fileMock.js", ...hq.get("jest"), }, testPathIgnorePatterns: ["/node_modules/", "/dist/", "/types/"],