Skip to content

Commit 2761a35

Browse files
sukujgrgsukujgrg
committed
Test CLI help semantically
1 parent 83283bd commit 2761a35

3 files changed

Lines changed: 40 additions & 133 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,10 @@ reuses the production typed parser, including offline static-JWKS validation.
195195
failures.
196196

197197
The command root and every implemented subcommand support both `-h` and
198-
`--help`. Help exits successfully, writes only its stable usage text to stdout,
199-
and is handled before configuration access, verification, clock use or audit
200-
emission. Go and C++ emit byte-identical help for their shared
201-
`credbind-ssh-authorized-keys` surface.
198+
`--help`. Help exits successfully, writes its implementation-owned usage text
199+
to stdout, and is handled before configuration access, verification, clock use
200+
or audit emission. Tests cover commands and options semantically; help
201+
presentation is not a cross-language byte contract.
202202

203203
## Sanitizer and fuzz gates
204204

‎tests/cli/adapters_test.py‎

Lines changed: 14 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env python3
2-
"""Byte-level tests for the specified C++ command adapter surface."""
2+
"""Semantic tests for the specified C++ command adapter surface."""
33

44
from __future__ import annotations
55

@@ -22,66 +22,14 @@ def require(condition: bool, message: str) -> None:
2222

2323

2424
HELP_TOPICS = (
25-
((), b"""Usage:
26-
credbind-ssh-authorized-keys version
27-
credbind-ssh-authorized-keys config init --policy-input PATH [OPTIONS]
28-
credbind-ssh-authorized-keys config init --deny-all [OPTIONS]
29-
credbind-ssh-authorized-keys config check --config PATH
30-
credbind-ssh-authorized-keys sshd-config render --config PATH --verifier PATH --command-user USER
31-
credbind-ssh-authorized-keys verify --config PATH --user USER --key KEY --key-type TYPE
32-
33-
Options:
34-
-h, --help Show help.
35-
"""),
36-
(("version",), b"""Usage:
37-
credbind-ssh-authorized-keys version
38-
39-
Print version metadata as JSON.
40-
"""),
41-
(("config",), b"""Usage:
42-
credbind-ssh-authorized-keys config init --help
43-
credbind-ssh-authorized-keys config check --help
44-
"""),
45-
(("config", "init"), b"""Usage:
46-
credbind-ssh-authorized-keys config init --policy-input PATH [OPTIONS]
47-
credbind-ssh-authorized-keys config init --deny-all [OPTIONS]
48-
49-
Options:
50-
--policy-input PATH Initialize from explicit trust and account policy.
51-
--deny-all Initialize an explicit deny-all policy.
52-
--clock-skew DURATION Set verifier clock skew.
53-
--total-verification-deadline DURATION Set the total verification deadline.
54-
--max-token-bytes INTEGER Set the token byte limit.
55-
--max-evidence-bytes INTEGER Set the evidence byte limit.
56-
--max-ssh-certificate-bytes INTEGER Set the SSH certificate byte limit.
57-
--max-offered-key-chars INTEGER Set the offered-key character limit.
58-
--max-authorized-keys-output-chars INTEGER Set the authorized-keys output limit.
59-
--issuer-key-cache-directory PATH Set the issuer-key cache directory.
60-
--issuer-key-cache-maximum-freshness DURATION
61-
Set the issuer-key cache freshness limit.
62-
--logging-facility FACILITY Set the local syslog facility.
63-
--output PATH Atomically write instead of using stdout.
64-
--force Replace an existing regular output file.
65-
-h, --help Show help.
66-
"""),
67-
(("config", "check"), b"""Usage:
68-
credbind-ssh-authorized-keys config check --config PATH
69-
70-
Validate configuration offline without changing it.
71-
"""),
72-
(("sshd-config",), b"""Usage:
73-
credbind-ssh-authorized-keys sshd-config render --help
74-
"""),
75-
(("sshd-config", "render"), b"""Usage:
76-
credbind-ssh-authorized-keys sshd-config render --config PATH --verifier PATH --command-user USER
77-
78-
Render the minimal OpenSSH AuthorizedKeysCommand fragment without installing it.
79-
"""),
80-
(("verify",), b"""Usage:
81-
credbind-ssh-authorized-keys verify --config PATH --user USER --key KEY --key-type TYPE
82-
83-
Verify one OpenSSH certificate request. Denial produces empty stdout and exit status 0.
84-
"""),
25+
((), (b"credbind-ssh-authorized-keys", b"config", b"verify")),
26+
(("version",), (b"credbind-ssh-authorized-keys version",)),
27+
(("config",), (b"config init", b"config check")),
28+
(("config", "init"), (b"config init", b"--policy-input", b"--deny-all")),
29+
(("config", "check"), (b"config check", b"--config")),
30+
(("sshd-config",), (b"sshd-config render",)),
31+
(("sshd-config", "render"), (b"sshd-config render", b"--command-user")),
32+
(("verify",), (b"verify", b"--key-type")),
8533
)
8634

8735

@@ -164,12 +112,13 @@ def main() -> int:
164112
config.write_bytes(sample_config())
165113
config.chmod(0o600)
166114

167-
for prefix, expected_help in HELP_TOPICS:
115+
for prefix, required_terms in HELP_TOPICS:
168116
for help_flag in ("-h", "--help"):
169117
result = invoke(binary, *prefix, help_flag)
170-
require((result.returncode, result.stdout, result.stderr) ==
171-
(0, expected_help, b""),
172-
f"exact side-effect-free help for {prefix!r} {help_flag}")
118+
require(result.returncode == 0 and result.stdout.endswith(b"\n") and
119+
result.stderr == b"" and
120+
all(term in result.stdout for term in required_terms),
121+
f"semantic side-effect-free help for {prefix!r} {help_flag}")
173122

174123
result = invoke(binary, "config", "check", "--config", str(config))
175124
require((result.returncode, result.stdout, result.stderr) ==

‎tests/unit/adapters_test.cpp‎

Lines changed: 22 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -335,71 +335,23 @@ void test_command() {
335335
}
336336

337337
void test_help() {
338-
const std::vector<std::pair<std::vector<std::string_view>, std::string_view>> topics{
339-
{{}, R"(Usage:
340-
credbind-ssh-authorized-keys version
341-
credbind-ssh-authorized-keys config init --policy-input PATH [OPTIONS]
342-
credbind-ssh-authorized-keys config init --deny-all [OPTIONS]
343-
credbind-ssh-authorized-keys config check --config PATH
344-
credbind-ssh-authorized-keys sshd-config render --config PATH --verifier PATH --command-user USER
345-
credbind-ssh-authorized-keys verify --config PATH --user USER --key KEY --key-type TYPE
346-
347-
Options:
348-
-h, --help Show help.
349-
)"},
350-
{{"version"}, R"(Usage:
351-
credbind-ssh-authorized-keys version
352-
353-
Print version metadata as JSON.
354-
)"},
355-
{{"config"}, R"(Usage:
356-
credbind-ssh-authorized-keys config init --help
357-
credbind-ssh-authorized-keys config check --help
358-
)"},
359-
{{"config", "init"}, R"(Usage:
360-
credbind-ssh-authorized-keys config init --policy-input PATH [OPTIONS]
361-
credbind-ssh-authorized-keys config init --deny-all [OPTIONS]
362-
363-
Options:
364-
--policy-input PATH Initialize from explicit trust and account policy.
365-
--deny-all Initialize an explicit deny-all policy.
366-
--clock-skew DURATION Set verifier clock skew.
367-
--total-verification-deadline DURATION Set the total verification deadline.
368-
--max-token-bytes INTEGER Set the token byte limit.
369-
--max-evidence-bytes INTEGER Set the evidence byte limit.
370-
--max-ssh-certificate-bytes INTEGER Set the SSH certificate byte limit.
371-
--max-offered-key-chars INTEGER Set the offered-key character limit.
372-
--max-authorized-keys-output-chars INTEGER Set the authorized-keys output limit.
373-
--issuer-key-cache-directory PATH Set the issuer-key cache directory.
374-
--issuer-key-cache-maximum-freshness DURATION
375-
Set the issuer-key cache freshness limit.
376-
--logging-facility FACILITY Set the local syslog facility.
377-
--output PATH Atomically write instead of using stdout.
378-
--force Replace an existing regular output file.
379-
-h, --help Show help.
380-
)"},
381-
{{"config", "check"}, R"(Usage:
382-
credbind-ssh-authorized-keys config check --config PATH
383-
384-
Validate configuration offline without changing it.
385-
)"},
386-
{{"sshd-config"}, R"(Usage:
387-
credbind-ssh-authorized-keys sshd-config render --help
388-
)"},
389-
{{"sshd-config", "render"}, R"(Usage:
390-
credbind-ssh-authorized-keys sshd-config render --config PATH --verifier PATH --command-user USER
391-
392-
Render the minimal OpenSSH AuthorizedKeysCommand fragment without installing it.
393-
)"},
394-
{{"verify"}, R"(Usage:
395-
credbind-ssh-authorized-keys verify --config PATH --user USER --key KEY --key-type TYPE
396-
397-
Verify one OpenSSH certificate request. Denial produces empty stdout and exit status 0.
398-
)"},
338+
struct HelpTopic {
339+
std::vector<std::string_view> arguments;
340+
std::vector<std::string_view> required_terms;
341+
};
342+
const std::vector<HelpTopic> topics{
343+
{{}, {"credbind-ssh-authorized-keys", "config", "verify"}},
344+
{{"version"}, {"credbind-ssh-authorized-keys version"}},
345+
{{"config"}, {"config init", "config check"}},
346+
{{"config", "init"}, {"config init", "--policy-input", "--deny-all"}},
347+
{{"config", "check"}, {"config check", "--config"}},
348+
{{"sshd-config"}, {"sshd-config render"}},
349+
{{"sshd-config", "render"}, {"sshd-config render", "--command-user"}},
350+
{{"verify"}, {"verify", "--key-type"}},
399351
};
400352
for (const auto& topic : topics) {
401353
for (const std::string_view flag : {"-h", "--help"}) {
402-
auto arguments = topic.first;
354+
auto arguments = topic.arguments;
403355
arguments.push_back(flag);
404356
FakeLogger logger;
405357
ThrowClock clock;
@@ -408,8 +360,14 @@ Verify one OpenSSH certificate request. Denial produces empty stdout and exit st
408360
require(credbind::command::run(arguments, output, diagnostics,
409361
logger, clock) == 0,
410362
"help exits zero");
411-
require(output.str() == topic.second && diagnostics.str().empty(),
412-
"help exact stdout and empty stderr");
363+
const std::string rendered = output.str();
364+
require(!rendered.empty() && rendered.back() == '\n' &&
365+
diagnostics.str().empty(),
366+
"help uses stdout and ends with newline");
367+
for (const std::string_view term : topic.required_terms) {
368+
require(rendered.find(term) != std::string::npos,
369+
"help contains semantic command and option terms");
370+
}
413371
require(logger.calls == 0, "help does not emit audit events");
414372
}
415373
}

0 commit comments

Comments
 (0)