From d1fde876f5a0e6299bd7e84448735ed1f8d6a242 Mon Sep 17 00:00:00 2001 From: Dickson Date: Sat, 3 Oct 2026 18:19:33 +0000 Subject: [PATCH] Report contract state before a failure with --snapshot Add opt-in --snapshot (config key `snapshot`). When a run fails, reset the session, replay the same seed up to the first failing run, and print the target contract's data variables before each call of that run. Passing runs do no extra work, as suggested in the issue, and nothing changes without the flag. If the replay does not fail at the same run, no state is reported. Refs #145. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 +++ app.ts | 2 + cli.tests.ts | 41 ++++++ cli.ts | 9 ++ config.tests.ts | 19 +++ config.ts | 10 ++ docs/chapter_6.md | 33 ++++- invariant.ts | 59 +++++++- invariant.types.ts | 3 + property.ts | 54 +++++++- property.types.ts | 3 + snapshot.tests.ts | 332 +++++++++++++++++++++++++++++++++++++++++++++ snapshot.ts | 136 +++++++++++++++++++ 13 files changed, 702 insertions(+), 15 deletions(-) create mode 100644 snapshot.tests.ts create mode 100644 snapshot.ts diff --git a/README.md b/README.md index e40f2b68..e748a20d 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,22 @@ npx rv - `--bail` – Stop after the first failure. - `--dial` – The path to a JavaScript file containing custom pre- and post-execution functions (dialers). +- `--snapshot` – After a failure, report the target contract's data variables + before each call of the failing run (opt-in). + +--- + +### Contract state on failure (opt-in) + +```bash +rv example counter invariant --snapshot +``` + +When a run fails, `--snapshot` resets the simnet session, replays the same +seed up to the first failing run, and prints the target contract's data +variables before each call of that run. Passing runs do no extra work. Maps, +constants and token balances are not included. The config file key is +`snapshot`. --- diff --git a/app.ts b/app.ts index 5cf124cd..4472d482 100644 --- a/app.ts +++ b/app.ts @@ -143,6 +143,7 @@ export const main = async () => { radio, eligibleAccounts, allAddresses, + runConfig.snapshot, ); break; } @@ -160,6 +161,7 @@ export const main = async () => { radio, eligibleAccounts, allAddresses, + runConfig.snapshot, ); break; } diff --git a/cli.tests.ts b/cli.tests.ts index 77e88216..7542f0f3 100644 --- a/cli.tests.ts +++ b/cli.tests.ts @@ -233,4 +233,45 @@ describe("CLI parsing with parseCli", () => { const config = parseCli(["./example", "counter", "test", "--seed=42"])!; expect(config.warnings).toEqual([]); }); + + it("keeps snapshots disabled unless --snapshot is passed", () => { + expect(parseCli(["./example", "counter", "test"])!.snapshot).toBe(false); + expect( + parseCli(["./example", "counter", "test", "--snapshot"])!.snapshot, + ).toBe(true); + }); + + it("takes snapshot only from the config when --config is provided", () => { + // Arrange + const tempDir = join(tmpdir(), "rendezvous-test-parsecli-snapshot"); + mkdirSync(tempDir, { recursive: true }); + const configPath = join(tempDir, "rv.config.json"); + writeFileSync(configPath, JSON.stringify({ snapshot: true }), "utf-8"); + + // Act + const fromConfig = parseCli([ + "./example", + "counter", + "test", + `--config=${configPath}`, + ])!; + writeFileSync(configPath, JSON.stringify({}), "utf-8"); + const ignored = parseCli([ + "./example", + "counter", + "test", + `--config=${configPath}`, + "--snapshot", + ])!; + + // Assert + expect(fromConfig.snapshot).toBe(true); + expect(ignored.snapshot).toBe(false); + expect(ignored.warnings).toEqual([ + "Warning: --snapshot ignored when --config is used.", + ]); + + // Teardown + rmSync(tempDir, { recursive: true, force: true }); + }); }); diff --git a/cli.ts b/cli.ts index 3683c647..d81c65ad 100644 --- a/cli.ts +++ b/cli.ts @@ -25,6 +25,7 @@ export const helpMessage = ` --seed= Seed for replay functionality --runs= Number of test iterations [default: 100] --dial= Path to custom dialers file + --snapshot Report contract state before a failure --regr Run regression tests only --bail Stop on first failure -h, --help Show this message @@ -44,6 +45,7 @@ export interface RunConfig { bail: boolean; regr: boolean; dial: string | undefined; + snapshot: boolean; accounts: ConfigAccount[] | undefined; accountsMode: AccountsMode; configPath: string | undefined; @@ -69,6 +71,7 @@ export const parseCli = (argv: string[]): RunConfig | undefined => { seed: { type: "string" }, runs: { type: "string" }, dial: { type: "string" }, + snapshot: { type: "boolean" }, bail: { type: "boolean" }, regr: { type: "boolean" }, help: { type: "boolean", short: "h" }, @@ -123,6 +126,7 @@ export const parseCli = (argv: string[]): RunConfig | undefined => { options.bail && "--bail", options.regr && "--regr", options.dial && "--dial", + options.snapshot && "--snapshot", ].filter(Boolean) as string[]; if (ignoredFlags.length > 0) { @@ -140,6 +144,7 @@ export const parseCli = (argv: string[]): RunConfig | undefined => { bail: fileConfig.bail ?? false, regr: fileConfig.regr ?? false, dial: fileConfig.dial, + snapshot: fileConfig.snapshot ?? false, accounts: fileConfig.accounts, accountsMode: fileConfig.accounts_mode ?? "overwrite", configPath: options.config, @@ -168,6 +173,7 @@ export const parseCli = (argv: string[]): RunConfig | undefined => { bail: options.bail ?? false, regr: options.regr ?? false, dial: options.dial, + snapshot: options.snapshot ?? false, accounts: undefined, accountsMode: "overwrite", configPath: undefined, @@ -202,6 +208,9 @@ export const logRunConfig = ( if (config.dial !== undefined) { radio.emit("logMessage", `Using dial path: ${config.dial}`); } + if (config.snapshot) { + radio.emit("logMessage", "Reporting contract state before a failure."); + } if (config.configPath) { radio.emit("logMessage", `Using config file: ${config.configPath}`); } diff --git a/config.tests.ts b/config.tests.ts index 9f65ca9b..a5cffe4f 100644 --- a/config.tests.ts +++ b/config.tests.ts @@ -193,6 +193,25 @@ describe("Config loading and validation", () => { // Act & Assert expect(() => loadConfig(filePath)).toThrow(`"bail" must be a boolean.`); }); + + it("reads snapshot as a boolean", () => { + // Arrange + const valid = createTempConfigFile( + "snapshot", + JSON.stringify({ snapshot: true }), + ); + const invalid = createTempConfigFile( + "snapshot-string", + JSON.stringify({ snapshot: "yes" }), + ); + + // Act & Assert + expect(loadConfig(valid)).toEqual({ + config: { snapshot: true }, + unknownKeys: [], + }); + expect(() => loadConfig(invalid)).toThrow(`"snapshot" must be a boolean.`); + }); }); describe("Account resolution", () => { diff --git a/config.ts b/config.ts index bcbd748b..d93556a8 100644 --- a/config.ts +++ b/config.ts @@ -38,6 +38,8 @@ export interface RendezvousConfig { regr?: boolean; /** Path to custom dialers file. */ dial?: string; + /** Report the contract state before a failure. */ + snapshot?: boolean; } /** @@ -149,6 +151,13 @@ const validateConfig = ( config.dial = obj.dial; } + if ("snapshot" in obj) { + if (typeof obj.snapshot !== "boolean") { + throw new Error(`"snapshot" must be a boolean.`); + } + config.snapshot = obj.snapshot; + } + const knownKeys = new Set([ "accounts", "accounts_mode", @@ -157,6 +166,7 @@ const validateConfig = ( "bail", "regr", "dial", + "snapshot", ]); const unknownKeys = Object.keys(obj).filter((k) => !knownKeys.has(k)); diff --git a/docs/chapter_6.md b/docs/chapter_6.md index 2686a5cc..dc6ca2d6 100644 --- a/docs/chapter_6.md +++ b/docs/chapter_6.md @@ -45,7 +45,7 @@ This chapter explains how to use Rendezvous in different situations. By the end, To run Rendezvous, use the following command: ```bash -rv [--config] [--seed] [--runs] [--regr] [--bail] [--dial] +rv [--config] [--seed] [--runs] [--regr] [--bail] [--dial] [--snapshot] ``` Let's break down each part of the command. @@ -389,11 +389,41 @@ A config file is a JSON object with optional fields: | `bail` | boolean | Stop on first failure. | | `regr` | boolean | Run regression tests only. | | `dial` | string | Path to custom dialers file. | +| `snapshot` | boolean | Report the contract state before a failure. | The `accounts` field lets you define custom accounts for testing. By default (`"overwrite"` mode), these replace the Devnet.toml accounts entirely. With `"concatenate"` mode, config accounts are merged with the existing Devnet accounts — if a name appears in both, the config account's address takes precedence. Rendezvous warns if the config file contains unrecognized keys (e.g. a typo like `"sedd"` instead of `"seed"`), and also warns if CLI flags are passed alongside `--config`. +**7. Reporting the Contract State Before a Failure** + +To see the contract state that led to a failure, use `--snapshot`: + +```bash +rv root contract invariant --snapshot +``` + +When a run fails, Rendezvous resets the simnet session and replays the same seed up to the first failing run. It then prints the target contract's data variables before each call of that run: + +``` +Replaying seed 1 up to run 5 to report the state before the first failure... +Simnet session reset. + +Contract state before each call of run 5, the first failing run (contract data variables only): + +₿ 12 Ӿ 16 before test-below-limit + counter: u4 + flag: false +``` + +For property-based tests, the state is read right before the failing test function is called. For invariant tests, it is read before each public function call of the failing run and before the invariant check, so you can follow how the state changed. + +Passing runs do no extra work; the replay only happens after a failure. Some details to keep in mind: + +- Only data variables (`define-data-var`) of the target contract are shown. Maps, constants, token balances and other contracts are not. +- The state comes from the first failing run, before shrinking. With `--bail`, this is the run in the failure report. Without it, the reported counterexample may be a shrunk version of that run. +- If the replay does not fail at the same run, for example because a dialer behaves differently, Rendezvous says so and reports no state. + ### Summary | Argument/Option | Description | Example | @@ -407,6 +437,7 @@ Rendezvous warns if the config file contains unrecognized keys (e.g. a typo like | `--bail` | Stop after the first failure. | `rv root contract test --bail` | | `--dial=` | Loads JavaScript dialers from a file for pre/post-processing. | `rv root contract test --dial=./custom-dialer.js` | | `--config=` | Uses a JSON config file for all run options. | `rv root contract test --config=rv.config.json` | +| `--snapshot` | Reports the contract's data variables before a failure. | `rv root contract test --snapshot` | --- diff --git a/invariant.ts b/invariant.ts index e4e2a353..e0147b57 100644 --- a/invariant.ts +++ b/invariant.ts @@ -14,7 +14,7 @@ import fc from "fast-check"; import { DialerRegistry, PostDialerError, PreDialerError } from "./dialer"; import { reporter } from "./heatstroke"; -import type { Statistics } from "./heatstroke.types"; +import type { RunDetails, Statistics } from "./heatstroke.types"; import type { InvariantTestConfig, InvariantTestContext, @@ -32,6 +32,7 @@ import { LOG_DIVIDER, } from "./shared"; import type { EnrichedContractInterfaceFunction } from "./shared.types"; +import { reportStateBeforeFailure } from "./snapshot"; import { buildTraitReferenceMap, enrichInterfaceWithTraitData, @@ -57,6 +58,7 @@ import { * @param radio The custom logging event emitter. * @param eligibleAccounts The resolved eligible accounts map. * @param allAddresses All resolved addresses for principal type generation. + * @param snapshot Whether to report the contract state before a failure. * @returns void */ export const checkInvariants = async ( @@ -72,6 +74,7 @@ export const checkInvariants = async ( radio: EventEmitter, eligibleAccounts: Map, allAddresses: string[], + snapshot = false, ) => { // The Rendezvous identifier is the first one in the list. Only one contract // can be fuzzed at a time. @@ -207,6 +210,31 @@ export const checkInvariants = async ( return; } + /** + * Runs an invariant test and, with `snapshot`, replays a failure to report + * the contract state before it. + */ + const runInvariantTest = async ( + config: InvariantTestConfig & InvariantTestContext, + ) => { + const runDetails = await invariantTest(config); + if (snapshot) { + await reportStateBeforeFailure( + runDetails, + rendezvousContractId, + resetSession, + (replayRadio, recorder) => + invariantTest({ + ...config, + bail: true, + radio: replayRadio, + recorder, + }), + radio, + ); + } + }; + if (regr) { // Run regression tests only. radio.emit( @@ -241,7 +269,7 @@ export const checkInvariants = async ( await resetSession(); - await invariantTest({ + await runInvariantTest({ simnet, targetContractName, rendezvousContractId, @@ -263,7 +291,7 @@ export const checkInvariants = async ( `Starting fresh round of invariant testing for the ${targetContractName} contract using user-provided configuration...\n`, ); - await invariantTest({ + await runInvariantTest({ simnet, targetContractName, rendezvousContractId, @@ -284,11 +312,12 @@ export const checkInvariants = async ( * Runs an invariant test. * @param config The union of the configuration and context for the invariant * test. - * @returns A promise that resolves when the invariant test is complete. + * @returns A promise that resolves to the run details when the invariant + * test is complete. */ const invariantTest = async ( config: InvariantTestConfig & InvariantTestContext, -) => { +): Promise => { const { simnet, targetContractName, @@ -302,6 +331,7 @@ const invariantTest = async ( allAddresses, functions, invariants, + recorder, } = config; // Pre-build one fast-check arbitrary per SUT and invariant function via @@ -350,11 +380,14 @@ const invariantTest = async ( statistics.invariant!.failed.set(functionInterface.name, 0); } + let result: RunDetails | undefined = undefined; const radioReporter = async (runDetails: any) => { + result = runDetails; reporter(runDetails, radio, "invariant", statistics); - // Persist failures for regression testing. - if (runDetails.failed) { + // Persist failures for regression testing. A replay to record the + // state repeats a failure that is already saved. + if (runDetails.failed && recorder === undefined) { persistFailure(runDetails, "invariant", rendezvousContractId, dial); } }; @@ -425,6 +458,7 @@ const invariantTest = async ( .map((burnBlocks) => ({ ...r, ...burnBlocks })), ), async (r) => { + recorder?.startRun(); for (const [index, selectedFunction] of r.selectedFunctions.entries()) { const [sutCallerWallet, sutCallerAddress] = r.sutCallers[index]; @@ -445,6 +479,11 @@ const invariantTest = async ( } try { + recorder?.record( + simnet, + r.rendezvousContractId, + selectedFunction.name, + ); const functionCall = simnet.callPublicFn( r.rendezvousContractId, selectedFunction.name, @@ -564,6 +603,11 @@ const invariantTest = async ( r.invariantCaller; try { + recorder?.record( + simnet, + r.rendezvousContractId, + r.selectedInvariant.name, + ); const { result: invariantCallResult } = simnet.callReadOnlyFn( r.rendezvousContractId, r.selectedInvariant.name, @@ -654,6 +698,7 @@ const invariantTest = async ( verbose: true, }, ); + return result!; }; /** diff --git a/invariant.types.ts b/invariant.types.ts index c24cbf0a..ad993d16 100644 --- a/invariant.types.ts +++ b/invariant.types.ts @@ -3,6 +3,7 @@ import type { EventEmitter } from "node:events"; import type { Simnet } from "@stacks/clarinet-sdk"; import type { EnrichedContractInterfaceFunction } from "./shared.types"; +import type { StateRecorder } from "./snapshot"; type ContractId = string; @@ -31,6 +32,8 @@ export interface InvariantTestConfig { radio: EventEmitter; eligibleAccounts: Map; allAddresses: string[]; + /** Records the contract state when replaying a failure. */ + recorder?: StateRecorder; } /** diff --git a/property.ts b/property.ts index 93a137ec..203343a6 100644 --- a/property.ts +++ b/property.ts @@ -8,7 +8,7 @@ import { dim, green, red, underline, yellow } from "ansicolor"; import fc from "fast-check"; import { reporter } from "./heatstroke"; -import type { Statistics } from "./heatstroke.types"; +import type { RunDetails, Statistics } from "./heatstroke.types"; import { strategyFor } from "./lib"; import { getFailureFilePath, @@ -21,6 +21,7 @@ import { getFunctionsListForContract, LOG_DIVIDER, } from "./shared"; +import { reportStateBeforeFailure } from "./snapshot"; import { buildTraitReferenceMap, enrichInterfaceWithTraitData, @@ -45,6 +46,7 @@ import { * @param radio The custom logging event emitter. * @param eligibleAccounts The resolved eligible accounts map (name to address). * @param allAddresses All resolved addresses for principal type generation. + * @param snapshot Whether to report the contract state before a failure. * @returns void */ export const checkProperties = async ( @@ -59,6 +61,7 @@ export const checkProperties = async ( radio: EventEmitter, eligibleAccounts: Map, allAddresses: string[], + snapshot = false, ) => { // A map where the keys are the test contract identifiers and the values are // arrays of their test functions. This map will be used to access the test @@ -195,6 +198,31 @@ export const checkProperties = async ( return; } + /** + * Runs a property test and, with `snapshot`, replays a failure to report + * the contract state before it. + */ + const runPropertyTest = async ( + config: PropertyTestConfig & PropertyTestContext, + ) => { + const runDetails = await propertyTest(config); + if (snapshot) { + await reportStateBeforeFailure( + runDetails, + rendezvousContractId, + resetSession, + (replayRadio, recorder) => + propertyTest({ + ...config, + bail: true, + radio: replayRadio, + recorder, + }), + radio, + ); + } + }; + if (regr) { // Run regression tests only. radio.emit( @@ -228,7 +256,7 @@ export const checkProperties = async ( await resetSession(); - await propertyTest({ + await runPropertyTest({ simnet, targetContractName, rendezvousContractId, @@ -252,7 +280,7 @@ export const checkProperties = async ( `Starting fresh round of property testing for the ${targetContractName} contract using user-provided configuration...\n`, ); - await propertyTest({ + await runPropertyTest({ simnet, targetContractName, rendezvousContractId, @@ -272,11 +300,12 @@ export const checkProperties = async ( * Runs a property test. * @param config The union of the configuration and context for the property * test. - * @returns A promise that resolves when the property test is complete. + * @returns A promise that resolves to the run details when the property test + * is complete. */ const propertyTest = async ( config: PropertyTestConfig & PropertyTestContext, -) => { +): Promise => { const { simnet, targetContractName, @@ -289,6 +318,7 @@ const propertyTest = async ( allAddresses, testFunctions, testContractsPairedFunctions, + recorder, } = config; // Pre-build one fast-check arbitrary per test function via the public @@ -313,11 +343,14 @@ const propertyTest = async ( statistics.test!.failed.set(functionInterface.name, 0); } + let result: RunDetails | undefined = undefined; const radioReporter = async (runDetails: any) => { + result = runDetails; reporter(runDetails, radio, "test", statistics); - // Persist failures for regression testing. - if (runDetails.failed) { + // Persist failures for regression testing. A replay to record the + // state repeats a failure that is already saved. + if (runDetails.failed && recorder === undefined) { persistFailure( runDetails, "test", @@ -372,6 +405,7 @@ const propertyTest = async ( .map((burnBlocks) => ({ ...r, ...burnBlocks })), ), async (r) => { + recorder?.startRun(); const printedTestFunctionArgs = r.functionArgs .map((cv) => cvToString(cv)) .join(" "); @@ -407,6 +441,11 @@ const propertyTest = async ( ); } else { try { + recorder?.record( + simnet, + r.rendezvousContractId, + r.selectedTestFunction.name, + ); // If the function call results in a runtime error, the error will // be caught and logged as a test failure in the catch block. const { result: testFunctionCallResult } = simnet.callPrivateFn( @@ -520,6 +559,7 @@ const propertyTest = async ( verbose: true, }, ); + return result!; }; /** diff --git a/property.types.ts b/property.types.ts index d19a7b2d..a57bfa32 100644 --- a/property.types.ts +++ b/property.types.ts @@ -3,6 +3,7 @@ import type { EventEmitter } from "node:events"; import type { Simnet } from "@stacks/clarinet-sdk"; import type { EnrichedContractInterfaceFunction } from "./shared.types"; +import type { StateRecorder } from "./snapshot"; /** * The configuration for a property test. @@ -17,6 +18,8 @@ export interface PropertyTestConfig { radio: EventEmitter; eligibleAccounts: Map; allAddresses: string[]; + /** Records the contract state when replaying a failure. */ + recorder?: StateRecorder; } /** diff --git a/snapshot.tests.ts b/snapshot.tests.ts new file mode 100644 index 00000000..6d59baa2 --- /dev/null +++ b/snapshot.tests.ts @@ -0,0 +1,332 @@ +import { EventEmitter } from "node:events"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { stripVTControlCharacters } from "node:util"; + +import type { Simnet } from "@stacks/clarinet-sdk"; +import { Cl } from "@stacks/transactions"; + +import { main } from "./app"; +import type { RunDetails } from "./heatstroke.types"; +import { reportStateBeforeFailure, StateRecorder } from "./snapshot"; + +const isolatedTestEnvPrefix = "rendezvous-test-snapshot-"; + +const fakeSimnet = () => + ({ + blockHeight: 3, + burnBlockHeight: 2, + getContractsInterfaces: () => + new Map([ + [ + "ST1.contract", + { + variables: [ + { name: "counter", access: "variable" }, + { name: "limit", access: "constant" }, + ], + }, + ], + ]), + getDataVar: vi.fn(() => Cl.uint(7)), + }) as unknown as Simnet & { getDataVar: ReturnType }; + +const failedRun = (numRuns: number) => + ({ failed: true, numRuns, seed: 1, counterexample: [] }) as RunDetails; + +describe("State recorder", () => { + it("records the data variables of the target run only", () => { + // Arrange + const simnet = fakeSimnet(); + const recorder = new StateRecorder(2); + + // Act + recorder.startRun(); + recorder.record(simnet, "ST1.contract", "test-a"); + recorder.startRun(); + recorder.record(simnet, "ST1.contract", "test-b"); + recorder.startRun(); + recorder.record(simnet, "ST1.contract", "test-c"); + + // Assert + expect(simnet.getDataVar).toHaveBeenCalledTimes(1); + expect(recorder.snapshots).toEqual([ + { + functionName: "test-b", + burnBlockHeight: 2, + blockHeight: 3, + dataVariables: [["counter", "u7"]], + }, + ]); + }); + + it("keeps a read error instead of throwing it", () => { + // Arrange + const simnet = fakeSimnet(); + simnet.getDataVar.mockImplementation(() => { + throw new Error("read failed"); + }); + const recorder = new StateRecorder(1); + recorder.startRun(); + + // Act & Assert + expect(() => + recorder.record(simnet, "ST1.contract", "test-a"), + ).not.toThrow(); + expect(recorder.error).toEqual(new Error("read failed")); + }); +}); + +describe("Reporting the state before a failure", () => { + const captureLogs = () => { + const logs: string[] = []; + const radio = new EventEmitter(); + radio.on("logMessage", (log) => logs.push(log)); + radio.on("logFailure", (log) => logs.push(log)); + return { logs, radio }; + }; + + it("does not replay a passing run", async () => { + // Arrange + const { logs, radio } = captureLogs(); + const resetSession = vi.fn(async () => {}); + const replay = vi.fn(); + + // Act + await reportStateBeforeFailure( + { ...failedRun(100), failed: false }, + "ST1.contract", + resetSession, + replay, + radio, + ); + + // Assert + expect(resetSession).not.toHaveBeenCalled(); + expect(replay).not.toHaveBeenCalled(); + expect(logs).toEqual([]); + }); + + it("reports no state when the replay fails at another run", async () => { + // Arrange + const { logs, radio } = captureLogs(); + + // Act + await reportStateBeforeFailure( + failedRun(3), + "ST1.contract", + async () => {}, + async (_radio, recorder) => { + recorder.startRun(); + recorder.record(fakeSimnet(), "ST1.contract", "test-a"); + return failedRun(1); + }, + radio, + ); + + // Assert + expect(logs).toContain( + "The replay did not fail at the same run, so no state is reported.\n", + ); + expect(logs.some((log) => log.includes("counter: u7"))).toBe(false); + }); + + it("reports a read error instead of the state", async () => { + // Arrange + const { logs, radio } = captureLogs(); + const simnet = fakeSimnet(); + simnet.getDataVar.mockImplementation(() => { + throw new Error("read failed"); + }); + + // Act + await reportStateBeforeFailure( + failedRun(1), + "ST1.contract", + async () => {}, + async (_radio, recorder) => { + recorder.startRun(); + recorder.record(simnet, "ST1.contract", "test-a"); + return failedRun(1); + }, + radio, + ); + + // Assert + expect(logs).toContain( + "Could not read the contract state: Error: read failed\n", + ); + }); +}); + +describe("--snapshot in the CLI", () => { + const initialArgv = process.argv; + const initialCwd = process.cwd(); + const initialExitCode = process.exitCode; + let project = ""; + let logs: string[] = []; + + const run = async (args: string[]) => { + logs = []; + for (const method of ["log", "error"] as const) { + vi.spyOn(console, method).mockImplementation((message: string) => { + logs.push(stripVTControlCharacters(String(message))); + }); + } + process.argv = ["node", "app.js", project, "counter", ...args]; + await main(); + vi.restoreAllMocks(); + return logs.join("\n"); + }; + + beforeEach(() => { + project = mkdtempSync(join(tmpdir(), isolatedTestEnvPrefix)); + mkdirSync(join(project, "contracts")); + cpSync( + resolve(__dirname, "example", "settings"), + join(project, "settings"), + { recursive: true }, + ); + writeFileSync( + join(project, "Clarinet.toml"), + `[project] +name = "snapshot" +telemetry = false + +[contracts.counter] +path = "contracts/counter.clar" +clarity_version = 3 +epoch = 3.0 +`, + ); + // The property and the invariant both fail once the counter reaches 5. + writeFileSync( + join(project, "contracts", "counter.clar"), + `(define-data-var counter uint u0) +(define-data-var flag bool false) +(define-constant LIMIT u5) + +(define-map context (string-ascii 100) { called: uint }) + +(define-private (update-context (function-name (string-ascii 100)) + (called uint)) + (ok (map-set context function-name { called: called }))) + +(define-public (increment) + (ok (var-set counter (+ (var-get counter) u1)))) + +(define-read-only (invariant-below-limit) (< (var-get counter) LIMIT)) + +(define-private (test-below-limit) + (begin + (var-set counter (+ (var-get counter) u1)) + (asserts! (< (var-get counter) LIMIT) (err u1)) + (ok true))) +`, + ); + // Keep saved regressions inside the temporary project. + process.chdir(project); + }); + + afterEach(() => { + process.chdir(initialCwd); + process.argv = initialArgv; + process.exitCode = initialExitCode; + vi.restoreAllMocks(); + rmSync(project, { recursive: true, force: true }); + }); + + it("reports the state before the failing property call", async () => { + // Act + const output = await run(["test", "--seed=1", "--snapshot"]); + + // Assert: the fifth call fails, after four passing ones. + expect(output).toContain("Error: Property failed after 5 tests."); + expect(output).toContain( + "Replaying seed 1 up to run 5 to report the state before the first " + + "failure...", + ); + expect(output).toContain( + "Contract state before each call of run 5, the first failing run " + + "(counter data variables only):", + ); + expect(output).toMatch( + /before test-below-limit\n {2}counter: u4\n {2}flag: false/, + ); + expect(output).not.toContain("LIMIT"); + expect(process.exitCode).toBe(1); + // The replay does not save the failure again. + const regressions = JSON.parse( + readFileSync( + join( + project, + ".rendezvous-regressions", + "ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM.counter.json", + ), + "utf8", + ), + ); + expect(regressions.test).toHaveLength(1); + }); + + it("reports the state before each call of the failing invariant run", async () => { + // Act + const output = await run(["invariant", "--seed=1", "--snapshot"]); + + // Assert + const states = [ + ...output.matchAll(/before (\S+)\n {2}counter: u(\d+)/g), + ].map(([, functionName, counter]) => ({ + functionName, + counter: Number(counter), + })); + expect(states.length).toBeGreaterThan(1); + // The run starts below the limit and each increment adds one. + expect(states[0].counter).toBeLessThan(5); + states.forEach(({ functionName, counter }, index) => { + expect(functionName).toBe( + index < states.length - 1 ? "increment" : "invariant-below-limit", + ); + expect(counter).toBe(states[0].counter + index); + }); + expect(states[states.length - 1].counter).toBeGreaterThanOrEqual(5); + expect(process.exitCode).toBe(1); + }); + + it("does not replay or report state for a passing run", async () => { + // Arrange + writeFileSync( + join(project, "contracts", "counter.clar"), + `(define-data-var counter uint u0) + +(define-private (test-passes) (ok true)) +`, + ); + + // Act + const output = await run(["test", "--seed=1", "--runs=10", "--snapshot"]); + + // Assert + expect(output).toContain("OK, properties passed after 10 runs."); + expect(output).not.toContain("Replaying"); + expect(output).not.toContain("Contract state"); + }); + + it("does not report state without --snapshot", async () => { + // Act + const output = await run(["test", "--seed=1"]); + + // Assert + expect(output).toContain("Error: Property failed after 5 tests."); + expect(output).not.toContain("Replaying"); + expect(output).not.toContain("Contract state"); + }); +}); diff --git a/snapshot.ts b/snapshot.ts new file mode 100644 index 00000000..ad7012bc --- /dev/null +++ b/snapshot.ts @@ -0,0 +1,136 @@ +import { EventEmitter } from "node:events"; + +import type { Simnet } from "@stacks/clarinet-sdk"; +import { cvToString } from "@stacks/transactions"; + +import type { RunDetails } from "./heatstroke.types"; +import { getContractNameFromContractId } from "./shared"; + +/** + * The target contract's data variables before a function call. + */ +export interface StateSnapshot { + /** The function about to be called. */ + functionName: string; + burnBlockHeight: number; + blockHeight: number; + /** Data variable values as Clarity strings. */ + dataVariables: [string, string][]; +} + +/** + * Records the state before each call of a single run, chosen by its number. + * Other runs are not recorded, so replaying up to a failure stays cheap. + */ +export class StateRecorder { + readonly snapshots: StateSnapshot[] = []; + error: unknown = undefined; + private readonly targetRun: number; + private run = 0; + + constructor(targetRun: number) { + this.targetRun = targetRun; + } + + /** Call at the start of each run. */ + startRun() { + this.run++; + } + + /** Records the data variables before `functionName` is called. */ + record(simnet: Simnet, contractId: string, functionName: string) { + if (this.run !== this.targetRun || this.error !== undefined) { + return; + } + // Do not let a failed read change how the replayed run behaves. + try { + const variables = simnet + .getContractsInterfaces() + .get(contractId)! + .variables.filter(({ access }) => access === "variable"); + this.snapshots.push({ + functionName, + burnBlockHeight: simnet.burnBlockHeight, + blockHeight: simnet.blockHeight, + dataVariables: variables.map(({ name }) => [ + name, + cvToString(simnet.getDataVar(contractId, name)), + ]), + }); + } catch (error: unknown) { + this.error = error; + } + } +} + +/** + * Replays a failed run from a fresh session and reports the target + * contract's data variables before each call of the first failing run. + * Nothing is replayed or read when the run passed. + * @param runDetails The details of the original run. + * @param contractId The target contract identifier. + * @param resetSession Resets the simnet session to a clean state. + * @param replay Runs the test again with the same configuration, stopping + * at the first failure, logging to the given radio and recording states. + * @param radio The custom logging event emitter. + */ +export const reportStateBeforeFailure = async ( + runDetails: RunDetails, + contractId: string, + resetSession: () => Promise, + replay: (radio: EventEmitter, recorder: StateRecorder) => Promise, + radio: EventEmitter, +) => { + if (!runDetails.failed) { + return; + } + + const { numRuns, seed } = runDetails; + radio.emit( + "logMessage", + `Replaying seed ${seed} up to run ${numRuns} to report the state ` + + `before the first failure...`, + ); + await resetSession(); + + const recorder = new StateRecorder(numRuns); + // The replay output is not shown; only its recorded state is reported. + const replayed = await replay(new EventEmitter(), recorder); + + if (recorder.error !== undefined) { + radio.emit( + "logFailure", + `Could not read the contract state: ${String(recorder.error)}\n`, + ); + return; + } + if (!replayed.failed || replayed.numRuns !== numRuns) { + radio.emit( + "logFailure", + "The replay did not fail at the same run, so no state is reported.\n", + ); + return; + } + + const contractName = getContractNameFromContractId(contractId); + radio.emit( + "logMessage", + `\nContract state before each call of run ${numRuns}, the first ` + + `failing run (${contractName} data variables only):`, + ); + for (const snapshot of recorder.snapshots) { + radio.emit( + "logMessage", + `\n₿ ${snapshot.burnBlockHeight.toString().padStart(8)} ` + + `Ӿ ${snapshot.blockHeight.toString().padStart(8)} ` + + `before ${snapshot.functionName}`, + ); + if (snapshot.dataVariables.length === 0) { + radio.emit("logMessage", " (no data variables)"); + } + for (const [name, value] of snapshot.dataVariables) { + radio.emit("logMessage", ` ${name}: ${value}`); + } + } + radio.emit("logMessage", "\n"); +};