From 6f308095d8c5880696d30ff74d0d6095a154bd82 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:32:01 -0500 Subject: [PATCH 1/9] --min-time is not part of benchmark_base, state min-time used to be stdrel-stopping-criterion parameter, but was being used by hot measuring loop (see #408). This PR add m_min_time data member to benchmark_base and state classes as measurement option, same as `--min-samples`. Setter/getter methods are added. Measuring loops have access to this parameter, but delegate what to use with measurement options (--min-time and --min-samples) to individual stopping criteria. Stopping criteria acquired `bool is_elibigible_to_stop(stopping_context)` method with stopping_context populated with number-of-samples collected, min-samples, total-sample-time, min-time. The method allows stopping criteria to customize treatment of measurement options: * stdrel: honors both --min-time and --min-samples * entropy: honors --min-samples, ignores --min-time * target-samples: ignores both Closes #408. CLI option `--min-time` is now always recognized, irrespective of which stopping-criterion is current during option parsing. --- docs/cli_help.md | 35 ++++++++------- examples/CMakeLists.txt | 6 +-- nvbench/benchmark_base.cuh | 10 +++++ nvbench/benchmark_base.cxx | 1 + nvbench/detail/measure_cold.cu | 30 +++++++------ nvbench/detail/measure_cold.cuh | 1 + nvbench/detail/measure_cpu_only.cuh | 1 + nvbench/detail/measure_cpu_only.cxx | 18 +++++--- nvbench/detail/measure_hot.cu | 4 +- nvbench/detail/measure_timeout_warnings.cuh | 25 +++++++++-- nvbench/detail/sample_count_criterion.cuh | 1 + nvbench/detail/sample_count_criterion.cxx | 5 +++ nvbench/detail/stdrel_criterion.cuh | 1 + nvbench/detail/stdrel_criterion.cxx | 17 +++---- nvbench/json_printer.cu | 2 + nvbench/option_parser.cu | 9 +++- nvbench/state.cuh | 6 +++ nvbench/state.cxx | 2 + nvbench/stopping_criterion.cuh | 40 +++++++++++++++++ testing/entropy_criterion.cu | 17 +++++++ testing/exception_safety.cu | 2 +- testing/measure_timeout_warnings.cu | 24 ++++++++-- testing/option_parser.cu | 49 ++++++++++++++++++++- testing/sample_count_criterion.cu | 18 ++++++++ testing/state_generator.cu | 3 ++ testing/stdrel_criterion.cu | 44 +++++++++++------- 26 files changed, 290 insertions(+), 81 deletions(-) diff --git a/docs/cli_help.md b/docs/cli_help.md index 94eb7cb4..afd13d24 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -155,11 +155,23 @@ * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. +* `--min-time ` + * Require at least `` of accumulated measurement time before stopping + criteria that use this limit may stop. + * The default `stdrel` stopping criterion uses this limit. Other stopping + criteria may ignore it. + * Batched measurements use this limit directly because they are not controlled + by a stopping criterion. + * Default is 0.5 seconds. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + ## Stopping Criteria * `--stopping-criterion ` - * After `--min-samples` is satisfied, use `` to detect if enough - samples were collected. + * Use `` to detect if enough samples were collected. + * Each stopping criterion decides which measurement collection limits must be + satisfied before its stopping decision is applied. * Only applies to Cold and CPU-only measurements. * If both GPU and CPU times are gathered, GPU time is used for stopping analysis. @@ -174,18 +186,6 @@ ### "stdrel" Stopping Criterion Parameters -* `--min-time ` - * Require at least `` of accumulated execution time before `stdrel` - can stop based on the relative standard deviation estimate, either because - it falls below `--max-noise` or because it stabilizes above that threshold. - To avoid running indefinitely when relative standard deviation cannot be - computed reliably, NVBench may also stop earlier after repeated non-finite - noise estimates. - * Only applies to `stdrel` stopping criterion. - * Default is 0.5 seconds. - * Applies to the most recent `--benchmark`, or all benchmarks if specified - before any `--benchmark` arguments. - * `--max-noise ` * Target relative standard deviation (stdev/mean). `stdrel` stops once the estimate falls below this value, or once the estimate has stabilized above @@ -197,6 +197,9 @@ ### "entropy" Stopping Criterion Parameters +The `entropy` criterion honors `--min-samples` before applying its convergence +test, but does not require `--min-time`. + * `--max-angle ` * Maximum linear regression angle of cumulative entropy. * Smaller values give more accurate results. @@ -217,7 +220,7 @@ * `--target-samples ` * Stop after at least `` samples are collected. * Default is 100 samples. - * The total number of collected samples is - `max(--min-samples, --target-samples)`. + * `sample-count` does not require global `--min-samples` or `--min-time` + before stopping. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2abe3c7d..e95424d0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -29,11 +29,7 @@ function (nvbench_add_examples_target target_prefix cuda_std) target_link_libraries(${example_name} PRIVATE nvbench::main) set_target_properties(${example_name} PROPERTIES COMPILE_FEATURES cuda_std_${cuda_std}) - set(example_args --timeout 0.1) - # The custom_criterion example doesn't support the --min-time argument: - if (NOT "${example_src}" STREQUAL "custom_criterion.cu") - list(APPEND example_args --min-time 1e-5) - endif() + set(example_args --timeout 0.1 --min-time 1e-5) add_test(NAME ${example_name} COMMAND "$" ${example_args}) diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index 9ceea815..268466d1 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -169,6 +169,15 @@ struct benchmark_base } /// @} + /// Accumulate at least this much measurement time when the stopping criterion requires it. @{ + [[nodiscard]] nvbench::float64_t get_min_time() const { return m_min_time; } + benchmark_base &set_min_time(nvbench::float64_t min_time) + { + m_min_time = min_time; + return *this; + } + /// @} + /// Execute this many warmup runs before collecting cold measurement samples. @{ [[nodiscard]] nvbench::int64_t get_cold_warmup_runs() const { return m_cold_warmup_runs; } benchmark_base &set_cold_warmup_runs(nvbench::int64_t cold_warmup_runs) @@ -348,6 +357,7 @@ protected: nvbench::int64_t m_min_samples{10}; nvbench::int64_t m_cold_warmup_runs{1}; + nvbench::float64_t m_min_time{0.5}; nvbench::float64_t m_cold_max_warmup_walltime{-1.}; nvbench::float64_t m_skip_time{-1.}; nvbench::float64_t m_timeout{15.}; diff --git a/nvbench/benchmark_base.cxx b/nvbench/benchmark_base.cxx index 06db10bb..8187dbc4 100644 --- a/nvbench/benchmark_base.cxx +++ b/nvbench/benchmark_base.cxx @@ -50,6 +50,7 @@ std::unique_ptr benchmark_base::clone() const result->m_min_samples = m_min_samples; result->m_cold_warmup_runs = m_cold_warmup_runs; + result->m_min_time = m_min_time; result->m_cold_max_warmup_walltime = m_cold_max_warmup_walltime; result->m_skip_time = m_skip_time; diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index eb63d40d..d8272862 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -51,6 +51,7 @@ measure_cold_base::measure_cold_base(state &exec_state) , m_check_throttling(!exec_state.get_run_once()) , m_min_samples{exec_state.get_min_samples()} , m_cold_warmup_runs{exec_state.get_cold_warmup_runs()} + , m_min_time{exec_state.get_min_time()} , m_cold_max_warmup_walltime{exec_state.get_cold_max_warmup_walltime()} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} @@ -179,13 +180,13 @@ bool measure_cold_base::is_finished() return true; } - // Check that we've gathered enough samples: - if (m_total_samples >= m_min_samples) + const nvbench::stopping_context context{m_total_samples, + m_total_cuda_time, + m_min_samples, + m_min_time}; + if (m_stopping_criterion.is_finished(context)) { - if (m_stopping_criterion.is_finished()) - { - return true; - } + return true; } // Check for timeouts: @@ -211,13 +212,16 @@ void measure_cold_base::log_timeout_warnings(nvbench::printer_base &printer, const auto timeout = m_walltime_timer.get_duration(); - log_measurement_timeout_warnings(printer, - m_criterion_params, - timeout, - m_total_samples, - m_min_samples, - m_total_cuda_time, - cuda_stdev_noise); + log_measurement_timeout_warnings( + printer, + m_criterion_params, + timeout, + m_total_samples, + m_min_samples, + m_min_time, + min_time_can_block_stop(m_stopping_criterion, m_total_samples, m_total_cuda_time, m_min_time), + m_total_cuda_time, + cuda_stdev_noise); } void measure_cold_base::generate_summaries() diff --git a/nvbench/detail/measure_cold.cuh b/nvbench/detail/measure_cold.cuh index df245193..e2c505ae 100644 --- a/nvbench/detail/measure_cold.cuh +++ b/nvbench/detail/measure_cold.cuh @@ -123,6 +123,7 @@ protected: nvbench::int64_t m_min_samples{}; nvbench::int64_t m_cold_warmup_runs{1}; + nvbench::float64_t m_min_time{}; nvbench::float64_t m_cold_max_warmup_walltime{}; nvbench::float64_t m_skip_time{}; nvbench::float64_t m_timeout{}; diff --git a/nvbench/detail/measure_cpu_only.cuh b/nvbench/detail/measure_cpu_only.cuh index d8facfae..9c4e1e31 100644 --- a/nvbench/detail/measure_cpu_only.cuh +++ b/nvbench/detail/measure_cpu_only.cuh @@ -81,6 +81,7 @@ protected: nvbench::int64_t m_min_samples{}; + nvbench::float64_t m_min_time{}; nvbench::float64_t m_skip_time{}; nvbench::float64_t m_timeout{}; diff --git a/nvbench/detail/measure_cpu_only.cxx b/nvbench/detail/measure_cpu_only.cxx index d8efdf38..db7d8e8c 100644 --- a/nvbench/detail/measure_cpu_only.cxx +++ b/nvbench/detail/measure_cpu_only.cxx @@ -44,6 +44,7 @@ measure_cpu_only_base::measure_cpu_only_base(state &exec_state) exec_state.get_stopping_criterion())} , m_run_once{exec_state.get_run_once()} , m_min_samples{exec_state.get_min_samples()} + , m_min_time{exec_state.get_min_time()} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} { @@ -96,13 +97,13 @@ bool measure_cpu_only_base::is_finished() return true; } - // Check that we've gathered enough samples: - if (m_total_samples >= m_min_samples) + const nvbench::stopping_context context{m_total_samples, + m_total_cpu_time, + m_min_samples, + m_min_time}; + if (m_stopping_criterion.is_finished(context)) { - if (m_stopping_criterion.is_finished()) - { - return true; - } + return true; } // Check for timeouts: @@ -272,6 +273,11 @@ void measure_cpu_only_base::generate_summaries() timeout, m_total_samples, m_min_samples, + m_min_time, + min_time_can_block_stop(m_stopping_criterion, + m_total_samples, + m_total_cpu_time, + m_min_time), m_total_cpu_time, cpu_stdev_noise); } diff --git a/nvbench/detail/measure_hot.cu b/nvbench/detail/measure_hot.cu index fe36a57e..37583046 100644 --- a/nvbench/detail/measure_hot.cu +++ b/nvbench/detail/measure_hot.cu @@ -39,9 +39,7 @@ measure_hot_base::measure_hot_base(state &exec_state) : m_state{exec_state} , m_launch{exec_state.get_cuda_stream()} , m_min_samples{exec_state.get_min_samples()} - , m_min_time{exec_state.get_criterion_params().has_value("min-time") - ? exec_state.get_criterion_params().get_float64("min-time") - : 0.5} + , m_min_time{exec_state.get_min_time()} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} { diff --git a/nvbench/detail/measure_timeout_warnings.cuh b/nvbench/detail/measure_timeout_warnings.cuh index 7eb14518..100e3577 100644 --- a/nvbench/detail/measure_timeout_warnings.cuh +++ b/nvbench/detail/measure_timeout_warnings.cuh @@ -52,16 +52,35 @@ get_float64_criterion_param(const nvbench::criterion_params ¶ms, const std:: return params.get_float64(name); } +inline bool min_time_can_block_stop(const nvbench::stopping_criterion_base &criterion, + nvbench::int64_t total_samples, + nvbench::float64_t accumulated_time, + nvbench::float64_t min_time) +{ + const nvbench::stopping_context min_samples_satisfied_context{total_samples, + accumulated_time, + total_samples, + min_time}; + const nvbench::stopping_context min_time_satisfied_context{total_samples, + accumulated_time, + total_samples, + accumulated_time}; + + return !criterion.is_eligible_to_stop(min_samples_satisfied_context) && + criterion.is_eligible_to_stop(min_time_satisfied_context); +} + inline void log_measurement_timeout_warnings(nvbench::printer_base &printer, const nvbench::criterion_params &criterion_params, nvbench::float64_t timeout, nvbench::int64_t total_samples, nvbench::int64_t min_samples, + nvbench::float64_t min_time, + bool can_min_time_block_stop, nvbench::float64_t accumulated_time, std::optional stdev_noise) { const auto max_noise = get_float64_criterion_param(criterion_params, "max-noise"); - const auto min_time = get_float64_criterion_param(criterion_params, "min-time"); const auto enough_samples_for_noise = statistics::has_enough_samples_for_noise_estimate(total_samples); @@ -102,7 +121,7 @@ inline void log_measurement_timeout_warnings(nvbench::printer_base &printer, total_samples, min_samples)); } - if (min_time && accumulated_time < *min_time) + if (can_min_time_block_stop && accumulated_time < min_time) { printer.log(nvbench::log_level::warn, fmt::format("Current measurement timed out ({:0.2f}s) " @@ -110,7 +129,7 @@ inline void log_measurement_timeout_warnings(nvbench::printer_base &printer, "{:0.2f}s)", timeout, accumulated_time, - *min_time)); + min_time)); } } diff --git a/nvbench/detail/sample_count_criterion.cuh b/nvbench/detail/sample_count_criterion.cuh index d7f44f0e..a10a6656 100644 --- a/nvbench/detail/sample_count_criterion.cuh +++ b/nvbench/detail/sample_count_criterion.cuh @@ -45,6 +45,7 @@ protected: virtual void do_initialize() override; virtual void do_add_measurement(nvbench::float64_t measurement) override; virtual bool do_is_finished() override; + virtual bool do_is_eligible_to_stop(const nvbench::stopping_context &context) const override; }; } // namespace nvbench::detail diff --git a/nvbench/detail/sample_count_criterion.cxx b/nvbench/detail/sample_count_criterion.cxx index 241b7755..5b304317 100644 --- a/nvbench/detail/sample_count_criterion.cxx +++ b/nvbench/detail/sample_count_criterion.cxx @@ -45,4 +45,9 @@ bool sample_count_criterion::do_is_finished() return m_total_samples >= m_params.get_int64("target-samples"); } +bool sample_count_criterion::do_is_eligible_to_stop(const nvbench::stopping_context &) const +{ + return true; +} + } // namespace nvbench::detail diff --git a/nvbench/detail/stdrel_criterion.cuh b/nvbench/detail/stdrel_criterion.cuh index ec1b4e2e..c0440537 100644 --- a/nvbench/detail/stdrel_criterion.cuh +++ b/nvbench/detail/stdrel_criterion.cuh @@ -50,6 +50,7 @@ protected: virtual void do_initialize() override; virtual void do_add_measurement(nvbench::float64_t measurement) override; virtual bool do_is_finished() override; + virtual bool do_is_eligible_to_stop(const nvbench::stopping_context &context) const override; }; } // namespace nvbench::detail diff --git a/nvbench/detail/stdrel_criterion.cxx b/nvbench/detail/stdrel_criterion.cxx index c36f8f97..38e63b16 100644 --- a/nvbench/detail/stdrel_criterion.cxx +++ b/nvbench/detail/stdrel_criterion.cxx @@ -32,9 +32,7 @@ namespace nvbench::detail constexpr nvbench::int64_t invalid_noise_estimate_limit = 64; stdrel_criterion::stdrel_criterion() - : stopping_criterion_base{"stdrel", - {{"max-noise", 0.005}, // 0.5% stdrel - {"min-time", 0.5}}} // 0.5 seconds + : stopping_criterion_base{"stdrel", {{"max-noise", 0.005}}} // 0.5% stdrel {} void stdrel_criterion::do_initialize() @@ -76,14 +74,6 @@ bool stdrel_criterion::do_is_finished() return true; } - const auto total_measured_time = - m_measurements_summary.get_mean() * - static_cast(m_measurements_summary.get_size()); - if (total_measured_time <= m_params.get_float64("min-time")) - { - return false; - } - if (m_noise_tracker.empty()) { return false; @@ -138,4 +128,9 @@ bool stdrel_criterion::do_is_finished() return false; } +bool stdrel_criterion::do_is_eligible_to_stop(const nvbench::stopping_context &context) const +{ + return !context.needs_more_samples() && !context.needs_more_time(); +} + } // namespace nvbench::detail diff --git a/nvbench/json_printer.cu b/nvbench/json_printer.cu index 9871eaed..d20d3590 100644 --- a/nvbench/json_printer.cu +++ b/nvbench/json_printer.cu @@ -474,6 +474,7 @@ void json_printer::do_print_benchmark_results(const benchmark_vector &benches) bench["index"] = bench_index; bench["min_samples"] = bench_ptr->get_min_samples(); + bench["min_time"] = bench_ptr->get_min_time(); bench["cold_warmup_runs"] = bench_ptr->get_cold_warmup_runs(); bench["cold_max_warmup_walltime"] = bench_ptr->get_cold_max_warmup_walltime(); bench["skip_time"] = bench_ptr->get_skip_time(); @@ -533,6 +534,7 @@ void json_printer::do_print_benchmark_results(const benchmark_vector &benches) st["name"] = exec_state.get_axis_values_as_string(); st["min_samples"] = exec_state.get_min_samples(); + st["min_time"] = exec_state.get_min_time(); st["cold_warmup_runs"] = exec_state.get_cold_warmup_runs(); st["cold_max_warmup_walltime"] = exec_state.get_cold_max_warmup_walltime(); st["skip_time"] = exec_state.get_skip_time(); diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 239a0498..249c12c8 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -557,8 +557,9 @@ void option_parser::parse_range(option_parser::arg_iterator_t first, this->update_int64_prop(first[0], first[1]); first += 2; } - else if (arg == "--skip-time" || arg == "--timeout" || arg == "--cold-max-warmup-walltime" || - arg == "--throttle-threshold" || arg == "--throttle-recovery-delay") + else if (arg == "--min-time" || arg == "--skip-time" || arg == "--timeout" || + arg == "--cold-max-warmup-walltime" || arg == "--throttle-threshold" || + arg == "--throttle-recovery-delay") { check_params(1); this->update_float64_prop(first[0], first[1]); @@ -1130,6 +1131,10 @@ try { bench.set_skip_time(value); } + else if (prop_arg == "--min-time") + { + bench.set_min_time(value); + } else if (prop_arg == "--timeout") { bench.set_timeout(value); diff --git a/nvbench/state.cuh b/nvbench/state.cuh index fb506e8d..49f0a4c2 100644 --- a/nvbench/state.cuh +++ b/nvbench/state.cuh @@ -155,6 +155,11 @@ struct state void set_min_samples(nvbench::int64_t min_samples) { m_min_samples = min_samples; } /// @} + /// Accumulate at least this much measurement time when the stopping criterion requires it. @{ + [[nodiscard]] nvbench::float64_t get_min_time() const { return m_min_time; } + void set_min_time(nvbench::float64_t min_time) { m_min_time = min_time; } + /// @} + /// Execute this many warmup runs before collecting cold measurement samples. @{ [[nodiscard]] nvbench::int64_t get_cold_warmup_runs() const { return m_cold_warmup_runs; } void set_cold_warmup_runs(nvbench::int64_t cold_warmup_runs) @@ -357,6 +362,7 @@ private: nvbench::int64_t m_min_samples; nvbench::int64_t m_cold_warmup_runs; + nvbench::float64_t m_min_time; nvbench::float64_t m_cold_max_warmup_walltime; nvbench::float64_t m_skip_time; nvbench::float64_t m_timeout; diff --git a/nvbench/state.cxx b/nvbench/state.cxx index a81fe693..4f032695 100644 --- a/nvbench/state.cxx +++ b/nvbench/state.cxx @@ -47,6 +47,7 @@ state::state(const benchmark_base &bench) , m_stopping_criterion(bench.get_stopping_criterion()) , m_min_samples{bench.get_min_samples()} , m_cold_warmup_runs{bench.get_cold_warmup_runs()} + , m_min_time{bench.get_min_time()} , m_cold_max_warmup_walltime{bench.get_cold_max_warmup_walltime()} , m_skip_time{bench.get_skip_time()} , m_timeout{bench.get_timeout()} @@ -71,6 +72,7 @@ state::state(const benchmark_base &bench, , m_stopping_criterion(bench.get_stopping_criterion()) , m_min_samples{bench.get_min_samples()} , m_cold_warmup_runs{bench.get_cold_warmup_runs()} + , m_min_time{bench.get_min_time()} , m_cold_max_warmup_walltime{bench.get_cold_max_warmup_walltime()} , m_skip_time{bench.get_skip_time()} , m_timeout{bench.get_timeout()} diff --git a/nvbench/stopping_criterion.cuh b/nvbench/stopping_criterion.cuh index 9742350c..e35dc6c2 100644 --- a/nvbench/stopping_criterion.cuh +++ b/nvbench/stopping_criterion.cuh @@ -45,6 +45,17 @@ namespace detail inline std::string default_stopping_criterion() { return "stdrel"; } } // namespace detail +struct stopping_context +{ + nvbench::int64_t total_samples{}; + nvbench::float64_t total_time{}; + nvbench::int64_t min_samples{}; + nvbench::float64_t min_time{}; + + [[nodiscard]] bool needs_more_samples() const { return total_samples < min_samples; } + [[nodiscard]] bool needs_more_time() const { return total_time < min_time; } +}; + /** * Stores all the parameters for stopping criterion in use */ @@ -121,6 +132,23 @@ public: */ bool is_finished() { return this->do_is_finished(); } + /** + * Check if the criterion is eligible to stop under the current measurement context and has + * been met for all measurements processed by `add_measurement`. + */ + bool is_finished(const stopping_context &context) + { + return this->do_is_eligible_to_stop(context) && this->do_is_finished(); + } + + /** + * Check if framework-level measurement limits permit stopping. + */ + bool is_eligible_to_stop(const stopping_context &context) const + { + return this->do_is_eligible_to_stop(context); + } + protected: /** * Initialize the criterion after updating the parameters @@ -136,6 +164,18 @@ protected: * Check if the criterion has been met for all measurements processed by `add_measurement` */ virtual bool do_is_finished() = 0; + + /** + * Check if framework-level measurement limits permit stopping. + * + * The default preserves the historical behavior for custom criteria: honor min_samples before + * applying the criterion's own stopping decision, but do not impose min_time unless the + * criterion opts in. + */ + virtual bool do_is_eligible_to_stop(const stopping_context &context) const + { + return !context.needs_more_samples(); + } }; } // namespace nvbench diff --git a/testing/entropy_criterion.cu b/testing/entropy_criterion.cu index 795e58f2..9ca0ff3f 100644 --- a/testing/entropy_criterion.cu +++ b/testing/entropy_criterion.cu @@ -39,6 +39,22 @@ void test_const() ASSERT(criterion.is_finished()); } +void test_context_requires_min_samples_only() +{ + nvbench::criterion_params params; + nvbench::detail::entropy_criterion criterion; + + criterion.initialize(params); + for (int i = 0; i < 6; i++) + { + criterion.add_measurement(42.0); + } + + ASSERT(criterion.is_finished()); + ASSERT(!criterion.is_finished(nvbench::stopping_context{5, 1.0, 6, 0.0})); + ASSERT(criterion.is_finished(nvbench::stopping_context{6, 0.0, 6, 1.0})); +} + void produce_entropy_arch(nvbench::detail::entropy_criterion &criterion) { /* @@ -87,5 +103,6 @@ void test_entropy_arch() int main() { test_const(); + test_context_requires_min_samples_only(); test_entropy_arch(); } diff --git a/testing/exception_safety.cu b/testing/exception_safety.cu index f5f0c38f..f01166e1 100644 --- a/testing/exception_safety.cu +++ b/testing/exception_safety.cu @@ -130,7 +130,7 @@ void run_benchmark(test_control &control, bool add_axis = false) bench.add_device(0); bench.set_min_samples(1); bench.set_timeout(0.01); - bench.set_criterion_param_float64("min-time", 1e-6); + bench.set_min_time(1e-6); if (add_axis) { bench.add_int64_axis("Case", {0, 1, 2}); diff --git a/testing/measure_timeout_warnings.cu b/testing/measure_timeout_warnings.cu index 58372f39..d2b46e07 100644 --- a/testing/measure_timeout_warnings.cu +++ b/testing/measure_timeout_warnings.cu @@ -63,6 +63,8 @@ void check_noise_warning( total_samples, 1, 1.0, + true, + 1.0, stdev_noise); ASSERT(printer.logs.size() == 1); @@ -88,7 +90,15 @@ void test_min_samples_timeout_warning() recording_printer printer{stream}; nvbench::criterion_params params; - nvbench::detail::log_measurement_timeout_warnings(printer, params, 1.0, 4, 5, 1.0, std::nullopt); + nvbench::detail::log_measurement_timeout_warnings(printer, + params, + 1.0, + 4, + 5, + 1.0, + false, + 1.0, + std::nullopt); ASSERT(printer.logs.size() == 1); ASSERT(printer.logs[0].first == nvbench::log_level::warn); @@ -100,9 +110,15 @@ void test_min_time_timeout_warning() std::ostringstream stream; recording_printer printer{stream}; nvbench::criterion_params params; - params.set_float64("min-time", 2.0); - - nvbench::detail::log_measurement_timeout_warnings(printer, params, 1.0, 5, 1, 1.5, std::nullopt); + nvbench::detail::log_measurement_timeout_warnings(printer, + params, + 1.0, + 5, + 1, + 2.0, + true, + 1.5, + std::nullopt); ASSERT(printer.logs.size() == 1); ASSERT(printer.logs[0].first == nvbench::log_level::warn); diff --git a/testing/option_parser.cu b/testing/option_parser.cu index ec1354df..e00c6302 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1205,6 +1205,16 @@ void test_min_samples() ASSERT(states[0].get_min_samples() == 12345); } +void test_min_time() +{ + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench", "--min-time", "12345e-2"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(std::abs(states[0].get_min_time() - 12345e-2) < 1e-12); +} + void test_cold_warmup_runs() { { @@ -1355,7 +1365,7 @@ void test_stopping_criterion() ASSERT(criterion_params.get_float64("max-angle") == 0.42); ASSERT(criterion_params.get_float64("min-r2") == 0.6); } - { // Global params to default criterion should work: + { // Global params to default criterion and global measurement options should work: nvbench::option_parser parser; parser.parse({ "--max-noise", @@ -1374,6 +1384,7 @@ void test_stopping_criterion() const auto &states = parser_to_states(parser); ASSERT(states.size() == 1); + ASSERT(states[0].get_min_time() == 0.1); ASSERT(states[0].get_stopping_criterion() == "entropy"); const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); @@ -1422,6 +1433,41 @@ void test_stopping_criterion() ASSERT(criterion_params.has_value("target-samples")); ASSERT(criterion_params.get_int64("target-samples") == 123); } + { // min-time is a measurement option, not a sample-count criterion parameter: + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "sample-count", + "--target-samples", + "3", + "--min-time", + "1e-5", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "sample-count"); + ASSERT(states[0].get_criterion_params().get_int64("target-samples") == 3); + ASSERT(states[0].get_min_time() == 1e-5); + } + { // min-time is also independent from entropy criterion parameters: + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--min-time", + "1e-5", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "entropy"); + ASSERT(states[0].get_min_time() == 1e-5); + } { // Global sample-count params apply to later benchmarks and per-benchmark params override: nvbench::option_parser parser; parser.parse({ @@ -1644,6 +1690,7 @@ try test_axis_before_benchmark(); test_min_samples(); + test_min_time(); test_cold_warmup_runs(); test_skip_time(); test_cold_max_warmup_walltime(); diff --git a/testing/sample_count_criterion.cu b/testing/sample_count_criterion.cu index 2d259e30..87bd18e8 100644 --- a/testing/sample_count_criterion.cu +++ b/testing/sample_count_criterion.cu @@ -68,6 +68,23 @@ void test_target_samples_one() ASSERT(criterion.is_finished()); } +void test_context_ignores_global_floors() +{ + nvbench::criterion_params params; + params.set_int64("target-samples", 3); + + nvbench::detail::sample_count_criterion criterion; + criterion.initialize(params); + + for (int i = 0; i < 3; ++i) + { + criterion.add_measurement(1.0); + } + + ASSERT(criterion.is_finished()); + ASSERT(criterion.is_finished(nvbench::stopping_context{3, 0.0, 10, 1.0})); +} + void test_non_positive_target_samples() { for (const auto target_samples : {0, -1}) @@ -85,5 +102,6 @@ int main() test_default_target_samples(); test_custom_target_samples(); test_target_samples_one(); + test_context_ignores_global_floors(); test_non_positive_target_samples(); } diff --git a/testing/state_generator.cu b/testing/state_generator.cu index 7c78cb1e..42d48502 100644 --- a/testing/state_generator.cu +++ b/testing/state_generator.cu @@ -764,6 +764,7 @@ void test_termination_criteria() { const nvbench::int64_t min_samples = 1000; const nvbench::int64_t cold_warmup_runs = 7; + const nvbench::float64_t min_time = 2000; const nvbench::float64_t cold_max_warmup_walltime = 3000; const nvbench::float64_t skip_time = 4000; const nvbench::float64_t timeout = 5000; @@ -774,6 +775,7 @@ void test_termination_criteria() dummy_bench bench; bench.set_devices(std::vector{}); bench.set_min_samples(min_samples); + bench.set_min_time(min_time); bench.set_cold_warmup_runs(cold_warmup_runs); bench.set_cold_max_warmup_walltime(cold_max_warmup_walltime); bench.set_skip_time(skip_time); @@ -783,6 +785,7 @@ void test_termination_criteria() ASSERT(states.size() == 1); ASSERT(min_samples == states[0].get_min_samples()); + ASSERT(within_one(min_time, states[0].get_min_time())); ASSERT(cold_warmup_runs == states[0].get_cold_warmup_runs()); ASSERT(within_one(cold_max_warmup_walltime, states[0].get_cold_max_warmup_walltime())); ASSERT(within_one(skip_time, states[0].get_skip_time())); diff --git a/testing/stdrel_criterion.cu b/testing/stdrel_criterion.cu index e0c1f68b..fc4bbfcd 100644 --- a/testing/stdrel_criterion.cu +++ b/testing/stdrel_criterion.cu @@ -30,13 +30,10 @@ constexpr nvbench::int64_t max_invalid_measurements_cap = 1024; -nvbench::int64_t count_invalid_measurements_until_finished(nvbench::float64_t min_time) +nvbench::int64_t count_invalid_measurements_until_finished() { - nvbench::criterion_params params; - params.set_float64("min-time", min_time); - nvbench::detail::stdrel_criterion criterion; - criterion.initialize(params); + criterion.initialize(nvbench::criterion_params{}); // freshly initialized criterion starts as not is_finished ASSERT(!criterion.is_finished()); @@ -71,7 +68,6 @@ void test_stdrel() nvbench::criterion_params params; params.set_float64("max-noise", max_noise); - params.set_float64("min-time", 0.0); nvbench::detail::stdrel_criterion criterion; criterion.initialize(params); @@ -104,11 +100,8 @@ void test_stdrel() void test_stdrel_needs_enough_samples() { - nvbench::criterion_params params; - params.set_float64("min-time", 0.0); - nvbench::detail::stdrel_criterion criterion; - criterion.initialize(params); + criterion.initialize(nvbench::criterion_params{}); using nvbench::detail::statistics::min_samples_for_noise_estimate; for (nvbench::int64_t i = 1; i < min_samples_for_noise_estimate; ++i) @@ -138,7 +131,6 @@ void test_stdrel_uses_sample_standard_deviation() nvbench::criterion_params params; params.set_float64("max-noise", 0.5 * (biased_noise + unbiased_noise)); - params.set_float64("min-time", 0.0); nvbench::detail::stdrel_criterion criterion; criterion.initialize(params); @@ -154,14 +146,34 @@ void test_stdrel_uses_sample_standard_deviation() void test_stdrel_finishes_with_persistently_invalid_noise() { - const auto count = count_invalid_measurements_until_finished(0.0); + const auto count = count_invalid_measurements_until_finished(); ASSERT(count > 1); } -void test_stdrel_invalid_noise_bypasses_min_time() +void test_stdrel_context_requires_min_samples_and_min_time() { - const auto count = count_invalid_measurements_until_finished(1.0); - ASSERT(count > 0); + nvbench::detail::stdrel_criterion criterion; + criterion.initialize(nvbench::criterion_params{}); + + using nvbench::detail::statistics::min_samples_for_noise_estimate; + for (nvbench::int64_t i = 0; i < min_samples_for_noise_estimate; ++i) + { + criterion.add_measurement(42.0); + } + + ASSERT(criterion.is_finished()); + ASSERT(!criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate - 1, + 42.0, + min_samples_for_noise_estimate, + 0.0})); + ASSERT(!criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate, + 0.1, + min_samples_for_noise_estimate, + 1.0})); + ASSERT(criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate, + 1.0, + min_samples_for_noise_estimate, + 1.0})); } int main() @@ -171,5 +183,5 @@ int main() test_stdrel_needs_enough_samples(); test_stdrel_uses_sample_standard_deviation(); test_stdrel_finishes_with_persistently_invalid_noise(); - test_stdrel_invalid_noise_bypasses_min_time(); + test_stdrel_context_requires_min_samples_and_min_time(); } From 4249cb53caf7f746142c180f637f0982c21ccc58 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:48:43 -0500 Subject: [PATCH 2/9] Add Python API to new setter/getter of min-time to benchmark/state --- python/cuda/bench/__init__.pyi | 5 +++++ python/cuda/bench/_decorators.py | 10 ++++++++++ python/src/py_nvbench.cpp | 33 ++++++++++++++++++++++++++++++++ python/test/test_cuda_bench.py | 12 +++++++++++- 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/python/cuda/bench/__init__.pyi b/python/cuda/bench/__init__.pyi index f8c59179..3003e0df 100644 --- a/python/cuda/bench/__init__.pyi +++ b/python/cuda/bench/__init__.pyi @@ -63,6 +63,7 @@ class Benchmark: def set_criterion_param_int64(self, name: str, value: SupportsInt) -> Self: ... def set_criterion_param_string(self, name: str, value: str) -> Self: ... def set_min_samples(self, count: SupportsInt) -> Self: ... + def set_min_time(self, duration_seconds: SupportsFloat) -> Self: ... def set_cold_warmup_runs(self, count: SupportsInt) -> Self: ... def set_cold_max_warmup_walltime(self, duration_seconds: SupportsFloat) -> Self: ... def set_is_cpu_only(self, is_cpu_only: bool) -> Self: ... @@ -106,6 +107,8 @@ class State: def set_throttle_threshold(self, threshold_fraction: SupportsFloat) -> None: ... def get_min_samples(self) -> int: ... def set_min_samples(self, min_samples_count: SupportsInt) -> None: ... + def get_min_time(self) -> float: ... + def set_min_time(self, duration_seconds: SupportsFloat) -> None: ... def get_cold_warmup_runs(self) -> int: ... def set_cold_warmup_runs(self, cold_warmup_runs: SupportsInt) -> None: ... def get_cold_max_warmup_walltime(self) -> float: ... @@ -212,6 +215,8 @@ class _OptionDecorators: ) -> Callable[[_F], _F]: ... def min_samples(self, count: SupportsInt) -> Callable[[_F], _F]: ... def set_min_samples(self, count: SupportsInt) -> Callable[[_F], _F]: ... + def min_time(self, duration_seconds: SupportsFloat) -> Callable[[_F], _F]: ... + def set_min_time(self, duration_seconds: SupportsFloat) -> Callable[[_F], _F]: ... def cold_warmup_runs(self, count: SupportsInt) -> Callable[[_F], _F]: ... def set_cold_warmup_runs(self, count: SupportsInt) -> Callable[[_F], _F]: ... def cold_max_warmup_walltime( diff --git a/python/cuda/bench/_decorators.py b/python/cuda/bench/_decorators.py index a45f0e92..300aedbd 100644 --- a/python/cuda/bench/_decorators.py +++ b/python/cuda/bench/_decorators.py @@ -292,6 +292,16 @@ def set_min_samples(self, count: int) -> Callable[[_F], _F]: lambda benchmark: benchmark.set_min_samples(count) ) + def min_time(self, duration_seconds: float) -> Callable[[_F], _F]: + """Set the minimum measurement time to collect.""" + return self.set_min_time(duration_seconds) + + def set_min_time(self, duration_seconds: float) -> Callable[[_F], _F]: + """Set the minimum measurement time to collect.""" + return _append_benchmark_action( + lambda benchmark: benchmark.set_min_time(duration_seconds) + ) + def cold_warmup_runs(self, count: int) -> Callable[[_F], _F]: """Set the number of cold measurement warmup runs.""" return self.set_cold_warmup_runs(count) diff --git a/python/src/py_nvbench.cpp b/python/src/py_nvbench.cpp index 82894e1e..1f656e84 100644 --- a/python/src/py_nvbench.cpp +++ b/python/src/py_nvbench.cpp @@ -472,6 +472,7 @@ static void def_class_Benchmark(py::module_ m) // nvbench::benchmark_base::set_criterion_param_float64 // nvbench::benchmark_base::set_criterion_param_string // nvbench::benchmark_base::set_min_samples + // nvbench::benchmark_base::set_min_time // nvbench::benchmark_base::set_cold_warmup_runs // nvbench::benchmark_base::set_cold_max_warmup_walltime @@ -734,6 +735,21 @@ Set minimal samples count before stopping criterion applies py::return_value_policy::reference, py::arg("min_samples_count")); + // method Benchmark.set_min_time + auto method_set_min_time_impl = [](nvbench::benchmark_base &self, + nvbench::float64_t duration_seconds) { + self.set_min_time(duration_seconds); + return std::ref(self); + }; + static constexpr const char *method_set_min_time_doc = R"XXXX( +Set minimal measurement time before the stopping criterion applies, in seconds +)XXXX"; + py_benchmark_cls.def("set_min_time", + method_set_min_time_impl, + method_set_min_time_doc, + py::return_value_policy::reference, + py::arg("duration_seconds")); + // method Benchmark.set_cold_warmup_runs auto method_set_cold_warmup_runs_impl = [](nvbench::benchmark_base &self, nvbench::int64_t count) { @@ -795,6 +811,8 @@ void def_class_State(py::module_ m) // nvbench::state::get_skip_reason // nvbench::state::get_min_samples // nvbench::state::set_min_samples + // nvbench::state::get_min_time + // nvbench::state::set_min_time // nvbench::state::get_cold_warmup_runs // nvbench::state::set_cold_warmup_runs // nvbench::state::get_cold_max_warmup_walltime @@ -1059,6 +1077,21 @@ Set the number of benchmark timings for NVBench to perform before stopping crite method_set_min_samples_doc, py::arg("min_samples_count")); + // method State.get_min_time + static constexpr const char *method_get_min_time_doc = R"XXXX( +Get the accumulated measurement time required before stopping criterion begins being used +)XXXX"; + pystate_cls.def("get_min_time", &nvbench::state::get_min_time, method_get_min_time_doc); + + // method State.set_min_time + static constexpr const char *method_set_min_time_doc = R"XXXX( +Set the accumulated measurement time required before stopping criterion begins being used +)XXXX"; + pystate_cls.def("set_min_time", + &nvbench::state::set_min_time, + method_set_min_time_doc, + py::arg("duration_seconds")); + // method State.get_cold_warmup_runs static constexpr const char *method_get_cold_warmup_runs_doc = R"XXXX( Get the number of cold measurement warmup runs diff --git a/python/test/test_cuda_bench.py b/python/test/test_cuda_bench.py index de8a1dfe..fa0622ad 100644 --- a/python/test/test_cuda_bench.py +++ b/python/test/test_cuda_bench.py @@ -152,6 +152,8 @@ def test_decorator_docstrings(): obj_has_docstring_check(bench.option.set_criterion_param_string) obj_has_docstring_check(bench.option.min_samples) obj_has_docstring_check(bench.option.set_min_samples) + obj_has_docstring_check(bench.option.min_time) + obj_has_docstring_check(bench.option.set_min_time) obj_has_docstring_check(bench.option.cold_warmup_runs) obj_has_docstring_check(bench.option.set_cold_warmup_runs) obj_has_docstring_check(bench.option.cold_max_warmup_walltime) @@ -173,6 +175,10 @@ def set_min_samples(self, count): self.calls.append(("min_samples", count)) return self + def set_min_time(self, duration_seconds): + self.calls.append(("min_time", duration_seconds)) + return self + def set_cold_warmup_runs(self, count): self.calls.append(("cold_warmup_runs", count)) return self @@ -193,6 +199,7 @@ def fake_register(fn): @bench.register() @bench.axis.int64("Elements", [1, 2, 3]) @bench.option.min_samples(11) + @bench.option.min_time(0.125) @bench.option.cold_warmup_runs(7) @bench.option.cold_max_warmup_walltime(0.25) def decorated(state: bench.State): @@ -202,6 +209,7 @@ def decorated(state: bench.State): assert fake_benchmark.calls == [ ("int64", "Elements", [1, 2, 3]), ("min_samples", 11), + ("min_time", 0.125), ("cold_warmup_runs", 7), ("cold_max_warmup_walltime", 0.25), ] @@ -279,7 +287,7 @@ def decorated(state: bench.State): pass with pytest.raises(RuntimeError, match="must be placed below"): - bench.option.min_samples(3)(decorated) + bench.option.min_time(0.1)(decorated) def test_axis_decorators_reject_wrong_order(monkeypatch): @@ -312,6 +320,8 @@ def test_State_doc(): obj_has_docstring_check(cl.get_string) obj_has_docstring_check(cl.get_cold_warmup_runs) obj_has_docstring_check(cl.set_cold_warmup_runs) + obj_has_docstring_check(cl.get_min_time) + obj_has_docstring_check(cl.set_min_time) obj_has_docstring_check(cl.get_cold_max_warmup_walltime) obj_has_docstring_check(cl.set_cold_max_warmup_walltime) obj_has_docstring_check(cl.skip) From 9b09b69de37bfbd7ea9c95dfffef7aadfc6eefbf Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:51:01 -0500 Subject: [PATCH 3/9] Extend testing of --min-time option --- testing/option_parser.cu | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/testing/option_parser.cu b/testing/option_parser.cu index e00c6302..05586420 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1450,6 +1450,27 @@ void test_stopping_criterion() ASSERT(states.size() == 1); ASSERT(states[0].get_stopping_criterion() == "sample-count"); ASSERT(states[0].get_criterion_params().get_int64("target-samples") == 3); + ASSERT(!states[0].get_criterion_params().has_value("min-time")); + ASSERT(states[0].get_min_time() == 1e-5); + } + { // min-time remains a measurement option when it precedes the criterion: + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--min-time", + "1e-5", + "--stopping-criterion", + "sample-count", + "--target-samples", + "3", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "sample-count"); + ASSERT(states[0].get_criterion_params().get_int64("target-samples") == 3); + ASSERT(!states[0].get_criterion_params().has_value("min-time")); ASSERT(states[0].get_min_time() == 1e-5); } { // min-time is also independent from entropy criterion parameters: From e2a6f9b2cd6e5ca1cd7a05c65289e8c90e485ef6 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:00:09 -0500 Subject: [PATCH 4/9] Add validation of --min-time option value in benchmark_base and state, add tests --- docs/cli_help.md | 2 ++ nvbench/benchmark_base.cuh | 2 ++ nvbench/detail/validate_min_time.cuh | 47 ++++++++++++++++++++++++++++ nvbench/state.cuh | 7 ++++- testing/benchmark.cu | 17 ++++++++++ testing/option_parser.cu | 36 ++++++++++++++++++--- testing/state.cu | 19 +++++++++++ 7 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 nvbench/detail/validate_min_time.cuh diff --git a/docs/cli_help.md b/docs/cli_help.md index afd13d24..db4b8e75 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -162,6 +162,8 @@ criteria may ignore it. * Batched measurements use this limit directly because they are not controlled by a stopping criterion. + * Must be finite and non-negative. Use `0` to disable this minimum-time + requirement. * Default is 0.5 seconds. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index 268466d1..25e10a1e 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -29,6 +29,7 @@ #endif #include +#include #include #include #include @@ -173,6 +174,7 @@ struct benchmark_base [[nodiscard]] nvbench::float64_t get_min_time() const { return m_min_time; } benchmark_base &set_min_time(nvbench::float64_t min_time) { + nvbench::detail::validate_min_time(min_time); m_min_time = min_time; return *this; } diff --git a/nvbench/detail/validate_min_time.cuh b/nvbench/detail/validate_min_time.cuh new file mode 100644 index 00000000..014cb0a7 --- /dev/null +++ b/nvbench/detail/validate_min_time.cuh @@ -0,0 +1,47 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#if defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_GCC) +#pragma GCC system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_CLANG) +#pragma clang system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_MSVC) +#pragma system_header +#endif + +#include + +#include +#include + +namespace nvbench::detail +{ + +inline void validate_min_time(nvbench::float64_t min_time) +{ + if (!std::isfinite(min_time) || min_time < nvbench::float64_t{0}) + { + throw std::invalid_argument{"min_time must be finite and non-negative."}; + } +} + +} // namespace nvbench::detail diff --git a/nvbench/state.cuh b/nvbench/state.cuh index 49f0a4c2..e9a7bfe9 100644 --- a/nvbench/state.cuh +++ b/nvbench/state.cuh @@ -29,6 +29,7 @@ #endif #include +#include #include #include #include @@ -157,7 +158,11 @@ struct state /// Accumulate at least this much measurement time when the stopping criterion requires it. @{ [[nodiscard]] nvbench::float64_t get_min_time() const { return m_min_time; } - void set_min_time(nvbench::float64_t min_time) { m_min_time = min_time; } + void set_min_time(nvbench::float64_t min_time) + { + nvbench::detail::validate_min_time(min_time); + m_min_time = min_time; + } /// @} /// Execute this many warmup runs before collecting cold measurement samples. @{ diff --git a/testing/benchmark.cu b/testing/benchmark.cu index 7cb9f5c5..f85781f4 100644 --- a/testing/benchmark.cu +++ b/testing/benchmark.cu @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -238,6 +239,21 @@ void test_run() ASSERT(bench.get_states().size() == 1); } +void test_min_time() +{ + no_types_bench bench; + + bench.set_min_time(0.); + ASSERT(bench.get_min_time() == 0.); + + bench.set_min_time(1.25); + ASSERT(bench.get_min_time() == 1.25); + + ASSERT_THROWS_ANY(bench.set_min_time(-1.)); + ASSERT_THROWS_ANY(bench.set_min_time(std::numeric_limits::quiet_NaN())); + ASSERT_THROWS_ANY(bench.set_min_time(std::numeric_limits::infinity())); +} + void test_clone() { lots_of_types_bench bench; @@ -309,6 +325,7 @@ int main() test_int64_power_of_two_axes(); test_string_axes(); test_run(); + test_min_time(); test_clone(); test_get_config_count(); } diff --git a/testing/option_parser.cu b/testing/option_parser.cu index fda3f477..a6eacf23 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1207,12 +1207,38 @@ void test_min_samples() void test_min_time() { - nvbench::option_parser parser; - parser.parse({"--benchmark", "DummyBench", "--min-time", "12345e-2"}); - const auto &states = parser_to_states(parser); + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench", "--min-time", "12345e-2"}); + const auto &states = parser_to_states(parser); - ASSERT(states.size() == 1); - ASSERT(std::abs(states[0].get_min_time() - 12345e-2) < 1e-12); + ASSERT(states.size() == 1); + ASSERT(std::abs(states[0].get_min_time() - 12345e-2) < 1e-12); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench", "--min-time", "0"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_min_time() == 0.); + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--min-time", "-1"})); + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--min-time", "nan"})); + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--min-time", "inf"})); + } } void test_cold_warmup_runs() diff --git a/testing/state.cu b/testing/state.cu index 51989437..17612761 100644 --- a/testing/state.cu +++ b/testing/state.cu @@ -22,6 +22,8 @@ #include #include +#include + #include "test_asserts.cuh" // Mock up a benchmark for testing: @@ -128,10 +130,27 @@ void test_defaults() ASSERT(state.get_string_or_default("Bar", "Kramble") == "Kramble"); } +void test_min_time() +{ + dummy_bench bench; + state_tester state{bench}; + + state.set_min_time(0.); + ASSERT(state.get_min_time() == 0.); + + state.set_min_time(1.25); + ASSERT(state.get_min_time() == 1.25); + + ASSERT_THROWS_ANY(state.set_min_time(-1.)); + ASSERT_THROWS_ANY(state.set_min_time(std::numeric_limits::quiet_NaN())); + ASSERT_THROWS_ANY(state.set_min_time(std::numeric_limits::infinity())); +} + int main() { test_streams(); test_params(); test_summaries(); test_defaults(); + test_min_time(); } From be66c19c04a422e4696b314f8eac4c1b4287cac8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:01:15 -0500 Subject: [PATCH 5/9] Update docstring to state that min-time is finite, non-negative --- python/cuda/bench/_decorators.py | 4 ++-- python/src/py_nvbench.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/cuda/bench/_decorators.py b/python/cuda/bench/_decorators.py index 300aedbd..4b1780e6 100644 --- a/python/cuda/bench/_decorators.py +++ b/python/cuda/bench/_decorators.py @@ -293,11 +293,11 @@ def set_min_samples(self, count: int) -> Callable[[_F], _F]: ) def min_time(self, duration_seconds: float) -> Callable[[_F], _F]: - """Set the minimum measurement time to collect.""" + """Set the finite, non-negative minimum measurement time to collect.""" return self.set_min_time(duration_seconds) def set_min_time(self, duration_seconds: float) -> Callable[[_F], _F]: - """Set the minimum measurement time to collect.""" + """Set the finite, non-negative minimum measurement time to collect.""" return _append_benchmark_action( lambda benchmark: benchmark.set_min_time(duration_seconds) ) diff --git a/python/src/py_nvbench.cpp b/python/src/py_nvbench.cpp index 1f656e84..9dcb319a 100644 --- a/python/src/py_nvbench.cpp +++ b/python/src/py_nvbench.cpp @@ -742,7 +742,7 @@ Set minimal samples count before stopping criterion applies return std::ref(self); }; static constexpr const char *method_set_min_time_doc = R"XXXX( -Set minimal measurement time before the stopping criterion applies, in seconds +Set the finite, non-negative minimum measurement time, in seconds )XXXX"; py_benchmark_cls.def("set_min_time", method_set_min_time_impl, @@ -1079,13 +1079,13 @@ Set the number of benchmark timings for NVBench to perform before stopping crite // method State.get_min_time static constexpr const char *method_get_min_time_doc = R"XXXX( -Get the accumulated measurement time required before stopping criterion begins being used +Get the minimum accumulated measurement time, in seconds )XXXX"; pystate_cls.def("get_min_time", &nvbench::state::get_min_time, method_get_min_time_doc); // method State.set_min_time static constexpr const char *method_set_min_time_doc = R"XXXX( -Set the accumulated measurement time required before stopping criterion begins being used +Set the finite, non-negative minimum measurement time, in seconds )XXXX"; pystate_cls.def("set_min_time", &nvbench::state::set_min_time, From bb129eeb93161430b6aa628e2208a2b03ac1ea3e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:03:51 -0500 Subject: [PATCH 6/9] Tweaked wording to accommodate possibility for stopping-criterion to override --min-samples --- docs/cli_help.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli_help.md b/docs/cli_help.md index db4b8e75..aa63fa5d 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -149,8 +149,8 @@ before any `--benchmark` arguments. * `--min-samples ` - * Gather at least `` samples per measurement before checking any - other stopping criterion besides the timeout. + * Require at least `` samples per measurement before stopping criteria + that use this limit may stop. * Default is 10 samples. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. From 881263432e9133af13434d8f6b285abfe701a879 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:15:03 -0500 Subject: [PATCH 7/9] expand compact namespace declaration --- nvbench/detail/validate_min_time.cuh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nvbench/detail/validate_min_time.cuh b/nvbench/detail/validate_min_time.cuh index 014cb0a7..93ddfcd4 100644 --- a/nvbench/detail/validate_min_time.cuh +++ b/nvbench/detail/validate_min_time.cuh @@ -33,7 +33,9 @@ #include #include -namespace nvbench::detail +namespace nvbench +{ +namespace detail { inline void validate_min_time(nvbench::float64_t min_time) @@ -44,4 +46,5 @@ inline void validate_min_time(nvbench::float64_t min_time) } } -} // namespace nvbench::detail +} // namespace detail +} // namespace nvbench From 2186c3f6cb97fff1d9629b605a5d15ea3de0fe9c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:48:47 -0500 Subject: [PATCH 8/9] Add tests exercising is_finished(ctx) --- testing/entropy_criterion.cu | 14 +++++- testing/sample_count_criterion.cu | 7 ++- testing/stdrel_criterion.cu | 73 ++++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/testing/entropy_criterion.cu b/testing/entropy_criterion.cu index 9ca0ff3f..10968672 100644 --- a/testing/entropy_criterion.cu +++ b/testing/entropy_criterion.cu @@ -51,8 +51,18 @@ void test_context_requires_min_samples_only() } ASSERT(criterion.is_finished()); - ASSERT(!criterion.is_finished(nvbench::stopping_context{5, 1.0, 6, 0.0})); - ASSERT(criterion.is_finished(nvbench::stopping_context{6, 0.0, 6, 1.0})); + ASSERT(!criterion.is_finished(nvbench::stopping_context{ + 5, // total_samples + 1.0, // total_time + 6, // min_samples + 0.0 // min_time + })); + ASSERT(criterion.is_finished(nvbench::stopping_context{ + 6, // total_samples + 0.0, // total_time + 6, // min_samples + 1.0 // min_time + })); } void produce_entropy_arch(nvbench::detail::entropy_criterion &criterion) diff --git a/testing/sample_count_criterion.cu b/testing/sample_count_criterion.cu index 87bd18e8..776fe890 100644 --- a/testing/sample_count_criterion.cu +++ b/testing/sample_count_criterion.cu @@ -82,7 +82,12 @@ void test_context_ignores_global_floors() } ASSERT(criterion.is_finished()); - ASSERT(criterion.is_finished(nvbench::stopping_context{3, 0.0, 10, 1.0})); + ASSERT(criterion.is_finished(nvbench::stopping_context{ + 3, // total_samples + 0.0, // total_time + 10, // min_samples + 1.0 // min_time + })); } void test_non_positive_target_samples() diff --git a/testing/stdrel_criterion.cu b/testing/stdrel_criterion.cu index fc4bbfcd..27410646 100644 --- a/testing/stdrel_criterion.cu +++ b/testing/stdrel_criterion.cu @@ -150,6 +150,39 @@ void test_stdrel_finishes_with_persistently_invalid_noise() ASSERT(count > 1); } +void test_stdrel_invalid_noise_context_requires_min_time() +{ + nvbench::detail::stdrel_criterion criterion; + criterion.initialize(nvbench::criterion_params{}); + + const auto invalid_measurement = nvbench::float64_t{0}; + nvbench::int64_t total_invalid_measurements = 0; + while (!criterion.is_finished() && total_invalid_measurements < max_invalid_measurements_cap) + { + criterion.add_measurement(invalid_measurement); + ++total_invalid_measurements; + } + ASSERT(criterion.is_finished()); + + const auto min_time = nvbench::float64_t{1}; + { + const auto total_samples = total_invalid_measurements; + const auto total_time = min_time / nvbench::float64_t{2}; + const auto min_samples = total_invalid_measurements; + const auto context = + nvbench::stopping_context{total_samples, total_time, min_samples, min_time}; + ASSERT(!criterion.is_finished(context)); + } + { + const auto total_samples = total_invalid_measurements; + const auto total_time = min_time; + const auto min_samples = total_invalid_measurements; + const auto context = + nvbench::stopping_context{total_samples, total_time, min_samples, min_time}; + ASSERT(criterion.is_finished(context)); + } +} + void test_stdrel_context_requires_min_samples_and_min_time() { nvbench::detail::stdrel_criterion criterion; @@ -162,18 +195,33 @@ void test_stdrel_context_requires_min_samples_and_min_time() } ASSERT(criterion.is_finished()); - ASSERT(!criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate - 1, - 42.0, - min_samples_for_noise_estimate, - 0.0})); - ASSERT(!criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate, - 0.1, - min_samples_for_noise_estimate, - 1.0})); - ASSERT(criterion.is_finished(nvbench::stopping_context{min_samples_for_noise_estimate, - 1.0, - min_samples_for_noise_estimate, - 1.0})); + { + const auto total_samples = min_samples_for_noise_estimate - 1; + const auto total_time = nvbench::float64_t{42}; + const auto min_samples = min_samples_for_noise_estimate; + const auto min_time = nvbench::float64_t{0}; + const auto context = + nvbench::stopping_context{total_samples, total_time, min_samples, min_time}; + ASSERT(!criterion.is_finished(context)); + } + { + const auto total_samples = min_samples_for_noise_estimate; + const auto total_time = nvbench::float64_t{0.1}; + const auto min_samples = min_samples_for_noise_estimate; + const auto min_time = nvbench::float64_t{1}; + const auto context = + nvbench::stopping_context{total_samples, total_time, min_samples, min_time}; + ASSERT(!criterion.is_finished(context)); + } + { + const auto total_samples = min_samples_for_noise_estimate; + const auto total_time = nvbench::float64_t{1}; + const auto min_samples = min_samples_for_noise_estimate; + const auto min_time = nvbench::float64_t{1}; + const auto context = + nvbench::stopping_context{total_samples, total_time, min_samples, min_time}; + ASSERT(criterion.is_finished(context)); + } } int main() @@ -183,5 +231,6 @@ int main() test_stdrel_needs_enough_samples(); test_stdrel_uses_sample_standard_deviation(); test_stdrel_finishes_with_persistently_invalid_noise(); + test_stdrel_invalid_noise_context_requires_min_time(); test_stdrel_context_requires_min_samples_and_min_time(); } From 0e4748d3306079251effda275102ee1d8987b61b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:31:03 -0500 Subject: [PATCH 9/9] Keep hot batch sizing stable when min-time is disabled Add a separate hot batch target time with a 100 us floor so --min-time 0, or very small positive min-time values, do not produce degenerate one-launch hot batches. Keep the user-provided min-time unchanged for stopping and timeout diagnostics; only the internal batch-size estimate uses the clamped target. --- nvbench/detail/measure_hot.cu | 16 ++++++++++++---- nvbench/detail/measure_hot.cuh | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nvbench/detail/measure_hot.cu b/nvbench/detail/measure_hot.cu index 37583046..b79b73c6 100644 --- a/nvbench/detail/measure_hot.cu +++ b/nvbench/detail/measure_hot.cu @@ -35,11 +35,16 @@ namespace nvbench::detail { +// Keep hot batches large enough to avoid one-launch batches when --min-time is +// disabled or set to an extremely small positive value. +constexpr nvbench::float64_t smallest_hot_batch_target_time = 100e-6; + measure_hot_base::measure_hot_base(state &exec_state) : m_state{exec_state} , m_launch{exec_state.get_cuda_stream()} , m_min_samples{exec_state.get_min_samples()} , m_min_time{exec_state.get_min_time()} + , m_batch_target_time{m_min_time} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} { @@ -53,16 +58,19 @@ measure_hot_base::measure_hot_base(state &exec_state) // If the cold measurement ran successfully, disable skip_time. It'd just // be annoying to skip now. m_skip_time = -1; + + m_batch_target_time = std::max(m_min_time, smallest_hot_batch_target_time); } catch (...) { // If the above threw an exception, we don't have a cold measurement to use. - // Estimate a target_time between m_min_time and m_timeout. - // Use the average of the min_time and timeout, but don't go over 5x - // min_time in case timeout is huge. + // Estimate an internal batch target from min_time and timeout, then apply + // the minimum floor below. Use the average of min_time and timeout, but + // don't go over 5x min_time in case timeout is huge. // We could expose a `target_time` property on benchmark_base/state if // needed. - m_min_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5); + const auto fallback_batch_target_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5); + m_batch_target_time = std::max(smallest_hot_batch_target_time, fallback_batch_target_time); } } diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index c8fb808a..75453365 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -94,6 +94,7 @@ protected: nvbench::int64_t m_min_samples{}; nvbench::float64_t m_min_time{}; + nvbench::float64_t m_batch_target_time{}; nvbench::float64_t m_skip_time{}; nvbench::float64_t m_timeout{}; @@ -147,7 +148,7 @@ private: // The .95 factor here pads the batch_size a bit to avoid needing a second // batch due to noise. const auto time_estimate = m_cuda_timer.get_duration() * 0.95; - auto batch_size = static_cast(m_min_time / time_estimate); + auto batch_size = static_cast(m_batch_target_time / time_estimate); do { @@ -199,7 +200,7 @@ private: // Predict number of remaining iterations: batch_size = static_cast( - (m_min_time - m_total_cuda_time) / + (m_batch_target_time - m_total_cuda_time) / (m_total_cuda_time / static_cast(m_total_samples))); if (m_total_cuda_time > m_min_time && // min time okay