From d4c6cfc03386cfe9748ce666c52af0deb18d507e Mon Sep 17 00:00:00 2001 From: Atanas Oreshkov Date: Fri, 6 Nov 2026 08:24:56 +0200 Subject: [PATCH 1/2] Pin what a program prints with golden files assertSurface(root) renders a command tree, so it can't show what a Program adds: the argument-file line in help, the @* case in completion scripts, or the configuration it reads. assertSurface(program) runs the program and compares what it printed. Program.root becomes public so the testing module can walk the commands. Co-Authored-By: Claude Opus 5.5 --- cli-kit-testing/api/cli-kit-testing.api | 2 + .../oreshkov/clikit/testing/GoldenFiles.kt | 36 ++++++++++ .../clikit/testing/GoldenFilesTest.kt | 70 +++++++++++++++++++ cli-kit/CHANGELOG.md | 5 ++ cli-kit/README.md | 10 +-- cli-kit/api/cli-kit.api | 1 + .../kotlin/app/oreshkov/clikit/Program.kt | 3 +- 7 files changed, 122 insertions(+), 5 deletions(-) diff --git a/cli-kit-testing/api/cli-kit-testing.api b/cli-kit-testing/api/cli-kit-testing.api index 4c0c711..1071658 100644 --- a/cli-kit-testing/api/cli-kit-testing.api +++ b/cli-kit-testing/api/cli-kit-testing.api @@ -5,8 +5,10 @@ public final class app/oreshkov/clikit/testing/GoldenFiles { public final fun assertNoOrphans (Ljava/util/Set;)V public final fun assertSurface (Lapp/oreshkov/clikit/CommandSpec;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;)V public final fun assertSurface (Lapp/oreshkov/clikit/CommandSpec;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;Lapp/oreshkov/clikit/Settings;)V + public final fun assertSurface (Lapp/oreshkov/clikit/Program;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;)V public static synthetic fun assertSurface$default (Lapp/oreshkov/clikit/testing/GoldenFiles;Lapp/oreshkov/clikit/CommandSpec;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;ILjava/lang/Object;)V public static synthetic fun assertSurface$default (Lapp/oreshkov/clikit/testing/GoldenFiles;Lapp/oreshkov/clikit/CommandSpec;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;Lapp/oreshkov/clikit/Settings;ILjava/lang/Object;)V + public static synthetic fun assertSurface$default (Lapp/oreshkov/clikit/testing/GoldenFiles;Lapp/oreshkov/clikit/Program;Lapp/oreshkov/clikit/Styler;Ljava/util/Set;ILjava/lang/Object;)V public final fun existing ()Ljava/util/Set; } diff --git a/cli-kit-testing/src/jvmMain/kotlin/app/oreshkov/clikit/testing/GoldenFiles.kt b/cli-kit-testing/src/jvmMain/kotlin/app/oreshkov/clikit/testing/GoldenFiles.kt index 8730de9..03d2094 100644 --- a/cli-kit-testing/src/jvmMain/kotlin/app/oreshkov/clikit/testing/GoldenFiles.kt +++ b/cli-kit-testing/src/jvmMain/kotlin/app/oreshkov/clikit/testing/GoldenFiles.kt @@ -2,10 +2,13 @@ package app.oreshkov.clikit.testing import app.oreshkov.clikit.CommandSpec import app.oreshkov.clikit.CompletionGenerator +import app.oreshkov.clikit.ExitStatus import app.oreshkov.clikit.ExperimentalCliKitApi import app.oreshkov.clikit.HelpRenderer +import app.oreshkov.clikit.Program import app.oreshkov.clikit.Settings import app.oreshkov.clikit.Shell +import app.oreshkov.clikit.StandardOptions import app.oreshkov.clikit.Styler import app.oreshkov.clikit.TableRenderer import app.oreshkov.clikit.paths @@ -111,7 +114,40 @@ public class GoldenFiles( val completions = CompletionGenerator() val expected = root.paths().associate { helpFileName(it) to { help.render(it) } } + Shell.entries.associate { completionFileName(it) to { completions.generate(it, root) } } + assertAll(expected, extra) + } + + /** + * Pins what [program] prints, by running it: one golden file per help page, its `--help` at each command, and, if + * its tree has [StandardOptions.completion], one per completion script, its `completion `. So the files show + * what a command tree alone can't say: the configuration it reads, and that it has argument files on. A page it + * doesn't print, because the run exits with another status than [ExitStatus.OK], fails and names its command line. + * + * The other files and failures work as in the [assertSurface] that takes a command tree. + */ + public fun assertSurface(program: Program, styler: Styler = Styler.Plain, extra: Set = emptySet()) { + fun printed(words: List): String { + val result = program.record(*words.toTypedArray(), io = RecordingTerminalIo(styler = styler)) + if (result.status != ExitStatus.OK) { + fail("'${(listOf(program.root.name) + words).joinToString(" ")}' exited ${result.status}: ${result.stderr}") + } + return result.stdout + } + + val paths = program.root.paths().toList() + val pages = paths.associate { path -> helpFileName(path) to { printed(path.words() + "--help") } } + val completion = paths.firstOrNull { it.last() === StandardOptions.completion } + val scripts = completion?.let { path -> + Shell.entries.associate { shell -> completionFileName(shell) to { printed(path.words() + shell.id) } } + } + assertAll(pages + scripts.orEmpty(), extra) + } + + /** The words that name the command at the end of this path, below the root. */ + private fun List.words(): List = drop(1).map { it.name } + /** Compares every file in [expected] before failing, then fails for one on disk in neither it nor [extra]. */ + private fun assertAll(expected: Map String>, extra: Set) { val failures = expected.mapNotNull { (name, render) -> try { assertMatches(name, render()) diff --git a/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt b/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt index 50980ba..8b21545 100644 --- a/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt +++ b/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt @@ -1,13 +1,17 @@ package app.oreshkov.clikit.testing +import app.oreshkov.clikit.CommandHandler import app.oreshkov.clikit.CommandSpec import app.oreshkov.clikit.Converters +import app.oreshkov.clikit.ExitStatus import app.oreshkov.clikit.ExperimentalCliKitApi import app.oreshkov.clikit.Option +import app.oreshkov.clikit.Program import app.oreshkov.clikit.Setting import app.oreshkov.clikit.Settings import app.oreshkov.clikit.Shell import app.oreshkov.clikit.StandardOptions +import app.oreshkov.clikit.withArgumentFiles import java.nio.file.Path import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.createTempDirectory @@ -87,6 +91,72 @@ class GoldenFilesTest { assertFailsWith { golden.assertSurface(tree) } } + private val program = Program(root, "demo 1.0", mapOf(get to CommandHandler { ExitStatus.OK })) + + @Test + fun `a program prints the pages its tree renders, so both forms read the same files`() { + write { golden.assertSurface(program) } + assertEquals(surface, files()) + + golden.assertSurface(root) + golden.assertSurface(program) + } + + @Test + @OptIn(ExperimentalCliKitApi::class) + fun `a program's files show that it has argument files on, which its tree can't say`() { + val withFiles = program.withArgumentFiles() + write { golden.assertSurface(withFiles) } + val footer = "Start with @file to read arguments from it, one per line; @@ for a literal @." + assertContains(directory.resolve("help-store-get.txt").readText(), footer) + assertContains(directory.resolve("completion-bash.sh").readText(), "-*|@*)") + + golden.assertSurface(withFiles) + assertFailsWith { golden.assertSurface(program) } + assertFailsWith { golden.assertSurface(root) } + } + + @Test + @OptIn(ExperimentalCliKitApi::class) + fun `a program's files show the configuration it reads, with nothing more to pass`() { + val settings = object : Settings { + override val locations: List = listOf("the user file") + + override fun read(): List = emptyList() + } + val key = Option.Single("--store", Converters.string, config = "store") + val tree = CommandSpec("demo", options = listOf(key), subcommands = listOf(get, StandardOptions.completion)) + val reading = Program(tree, "demo 1.0", mapOf(get to CommandHandler { ExitStatus.OK }), settings = settings) + write { golden.assertSurface(reading) } + assertContains(directory.resolve("help-root.txt").readText(), "Configuration:\n store Same as --store\n") + + golden.assertSurface(tree, settings = settings, extra = emptySet()) + } + + @Test + fun `a program without the completion command has no script to pin, and one left on disk is an orphan`() { + val tree = CommandSpec("demo", subcommands = listOf(get)) + val plain = Program(tree, "demo 1.0", mapOf(get to CommandHandler { ExitStatus.OK })) + write { golden.assertSurface(plain) } + assertEquals(setOf("help-root.txt", "help-get.txt"), files()) + + directory.resolve("completion-bash.sh").writeText("gone\n") + val failure = assertFailsWith { golden.assertSurface(plain) } + assertContains(failure.message.orEmpty(), "completion-bash.sh") + } + + @Test + fun `a script the program won't print fails, naming its command line, and the others are reported with it`() { + val required = Option.Single("--store", Converters.string, required = true) + val tree = CommandSpec("demo", options = listOf(required), subcommands = listOf(get, StandardOptions.completion)) + val refusing = Program(tree, "demo 1.0", mapOf(get to CommandHandler { ExitStatus.OK })) + + val failure = assertFailsWith { write { golden.assertSurface(refusing) } } + assertContains(failure.message.orEmpty(), "'demo completion bash' exited ${ExitStatus.USAGE}") + assertContains(failure.message.orEmpty(), "--store") + assertEquals(Shell.entries.size - 1, failure.suppressed.size) + } + @Test fun `a changed page fails, and every other changed page is reported with it`() { write { golden.assertSurface(root) } diff --git a/cli-kit/CHANGELOG.md b/cli-kit/CHANGELOG.md index 08ec904..cfd7b34 100644 --- a/cli-kit/CHANGELOG.md +++ b/cli-kit/CHANGELOG.md @@ -8,6 +8,11 @@ listed in [Delva's changelog](../CHANGELOG.md#010---2026-10-15). ## [Unreleased] +### Added + +- `GoldenFiles.assertSurface(program)` pins what a program prints by running it, so the files show its configuration + and its argument files, which a command tree can't say; `Program.root` is the tree it runs. + ### Changed - Argument files (experimental): only the first argument can name one. Every later word is taken as typed, so an diff --git a/cli-kit/README.md b/cli-kit/README.md index 69de859..f5602f7 100644 --- a/cli-kit/README.md +++ b/cli-kit/README.md @@ -267,7 +267,7 @@ an error that names the file and line, and so is a value the converter refuses: read only when a value is wanted from them, so `--help`, `help`, `--version`, `completion` and a command with no configurable option work whatever they say. A key that no option of the tree declares is warned about, with the nearest key, and `--quiet` silences the warning. Help tags the option `[config: store]` and lists the command's keys, -and where they're read, in a Configuration section; `GoldenFiles.assertSurface(root, settings = …)` pins those pages. +and where they're read, in a Configuration section; `GoldenFiles.assertSurface(program)` pins those pages. Anyone who can write a file can set every key, so give no key to an option that takes a secret, a command to run or code to load, and `readSecret` never reads one. The project file is off unless the program asks for it, since a @@ -358,13 +358,15 @@ command line. An absent option never matches, because the parser can't see the h The companion `cli-kit-testing` (`app.oreshkov.clikit:cli-kit-testing`, a test dependency) runs a program in-process against a recording terminal — with typed answers to prompts, standard input and interrupts — and pins every help -page and completion script of a command tree with golden files in one call (its KDoc is on -[javadoc.io](https://javadoc.io/doc/app.oreshkov.clikit/cli-kit-testing)): +page and completion script of a program with golden files in one call (its KDoc is on +[javadoc.io](https://javadoc.io/doc/app.oreshkov.clikit/cli-kit-testing)). It runs the program, so the files hold what +the program prints, its configuration and argument files included; `assertSurface(root)` renders a command tree alone, +for a test with no program to run: ```kotlin @Test fun `every help page and completion script`() { - GoldenFiles(Path.of("src", "test", "resources", "golden"), "golden.update").assertSurface(root) + GoldenFiles(Path.of("src", "test", "resources", "golden"), "golden.update").assertSurface(program) } @Test diff --git a/cli-kit/api/cli-kit.api b/cli-kit/api/cli-kit.api index 54b7323..ed6d2e1 100644 --- a/cli-kit/api/cli-kit.api +++ b/cli-kit/api/cli-kit.api @@ -537,6 +537,7 @@ public final class app/oreshkov/clikit/Program { public synthetic fun (Lapp/oreshkov/clikit/CommandSpec;Ljava/lang/String;Ljava/util/Map;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public fun (Lapp/oreshkov/clikit/CommandSpec;Ljava/lang/String;Ljava/util/Map;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;Lapp/oreshkov/clikit/Settings;)V public synthetic fun (Lapp/oreshkov/clikit/CommandSpec;Ljava/lang/String;Ljava/util/Map;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;Lapp/oreshkov/clikit/Settings;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getRoot ()Lapp/oreshkov/clikit/CommandSpec; public final fun run ([Ljava/lang/String;Lapp/oreshkov/clikit/TerminalIo;)I } diff --git a/cli-kit/src/commonMain/kotlin/app/oreshkov/clikit/Program.kt b/cli-kit/src/commonMain/kotlin/app/oreshkov/clikit/Program.kt index fa2385f..512dc2b 100644 --- a/cli-kit/src/commonMain/kotlin/app/oreshkov/clikit/Program.kt +++ b/cli-kit/src/commonMain/kotlin/app/oreshkov/clikit/Program.kt @@ -293,7 +293,8 @@ public class CommandFailure( * `withArgumentFiles()`. */ public class Program @ExperimentalCliKitApi constructor( - private val root: CommandSpec, + /** The command tree this program runs. */ + public val root: CommandSpec, /** What `--version` prints: `demo 0.1.0`. */ private val version: String, private val handlers: Map, From 4ef7b171b15c08e53df59a4823215331dcb8b1d9 Mon Sep 17 00:00:00 2001 From: Atanas Oreshkov Date: Fri, 6 Nov 2026 08:26:44 +0200 Subject: [PATCH 2/2] Test a page a program won't print with a name no command line can type Co-Authored-By: Claude Opus 5.5 --- .../oreshkov/clikit/testing/GoldenFilesTest.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt b/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt index 8b21545..bc81d57 100644 --- a/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt +++ b/cli-kit-testing/src/jvmTest/kotlin/app/oreshkov/clikit/testing/GoldenFilesTest.kt @@ -146,15 +146,16 @@ class GoldenFilesTest { } @Test - fun `a script the program won't print fails, naming its command line, and the others are reported with it`() { - val required = Option.Single("--store", Converters.string, required = true) - val tree = CommandSpec("demo", options = listOf(required), subcommands = listOf(get, StandardOptions.completion)) - val refusing = Program(tree, "demo 1.0", mapOf(get to CommandHandler { ExitStatus.OK })) + fun `a page the program won't print fails, naming its command line, and the others are reported with it`() { + // A name that reads as an option: the tree can hold the command, but no command line can select it. + val untypable = listOf(CommandSpec("-x"), CommandSpec("-y")) + val tree = CommandSpec("demo", subcommands = untypable) + val refusing = Program(tree, "demo 1.0", untypable.associateWith { CommandHandler { ExitStatus.OK } }) val failure = assertFailsWith { write { golden.assertSurface(refusing) } } - assertContains(failure.message.orEmpty(), "'demo completion bash' exited ${ExitStatus.USAGE}") - assertContains(failure.message.orEmpty(), "--store") - assertEquals(Shell.entries.size - 1, failure.suppressed.size) + assertContains(failure.message.orEmpty(), "'demo -x --help' exited ${ExitStatus.USAGE}") + assertEquals(1, failure.suppressed.size) + assertContains(failure.suppressed.single().message.orEmpty(), "'demo -y --help'") } @Test