Min time becomes measurement option

docs/cli_help.md

@@ -149,17 +149,31 @@
     before any `--benchmark` arguments.
 
 * `--min-samples <count>`
-  * Gather at least `<count>` samples per measurement before checking any
-    other stopping criterion besides the timeout.
+  * Require at least `<count>` samples per measurement before stopping criteria
+    that use this limit may stop.
   * Default is 10 samples.
   * Applies to the most recent `--benchmark`, or all benchmarks if specified
     before any `--benchmark` arguments.
 
+* `--min-time <seconds>`
+  * Require at least `<seconds>` of accumulated measurement time before stopping
+    criteria that use this limit may stop.
+  * The default `stdrel` stopping criterion uses this limit. Other stopping
+    criteria may ignore it.
+  * Batched measurements use this limit directly because they are not controlled
+    by a stopping criterion.
+  * Must be finite and non-negative. Use `0` to disable this minimum-time
+    requirement.
+  * Default is 0.5 seconds.
+  * Applies to the most recent `--benchmark`, or all benchmarks if specified
+    before any `--benchmark` arguments.
+
 ## Stopping Criteria
 
 * `--stopping-criterion <criterion>`
-  * After `--min-samples` is satisfied, use `<criterion>` to detect if enough
-    samples were collected.
+  * Use `<criterion>` to detect if enough samples were collected.
+  * Each stopping criterion decides which measurement collection limits must be
+    satisfied before its stopping decision is applied.
   * Only applies to Cold and CPU-only measurements.
   * If both GPU and CPU times are gathered, GPU time is used for stopping
     analysis.
@@ -174,18 +188,6 @@
 
 ### "stdrel" Stopping Criterion Parameters
 
-* `--min-time <seconds>`
-  * Require at least `<seconds>` of accumulated execution time before `stdrel`
-    can stop based on the relative standard deviation estimate, either because
-    it falls below `--max-noise` or because it stabilizes above that threshold.
-    To avoid running indefinitely when relative standard deviation cannot be
-    computed reliably, NVBench may also stop earlier after repeated non-finite
-    noise estimates.
-  * Only applies to `stdrel` stopping criterion.
-  * Default is 0.5 seconds.
-  * Applies to the most recent `--benchmark`, or all benchmarks if specified
-    before any `--benchmark` arguments.
-
 * `--max-noise <value>`
   * Target relative standard deviation (stdev/mean). `stdrel` stops once the
     estimate falls below this value, or once the estimate has stabilized above
@@ -197,6 +199,9 @@
 
 ### "entropy" Stopping Criterion Parameters
 
+The `entropy` criterion honors `--min-samples` before applying its convergence
+test, but does not require `--min-time`.
+
 * `--max-angle <value>`
   * Maximum linear regression angle of cumulative entropy.
   * Smaller values give more accurate results.
@@ -217,7 +222,7 @@
 * `--target-samples <count>`
   * Stop after at least `<count>` samples are collected.
   * Default is 100 samples.
-  * The total number of collected samples is
-    `max(--min-samples, --target-samples)`.
+  * `sample-count` does not require global `--min-samples` or `--min-time`
+    before stopping.
   * Applies to the most recent `--benchmark`, or all benchmarks if specified
     before any `--benchmark` arguments.

examples/CMakeLists.txt

@@ -29,11 +29,7 @@ function (nvbench_add_examples_target target_prefix cuda_std)
     target_link_libraries(${example_name} PRIVATE nvbench::main)
     set_target_properties(${example_name} PROPERTIES COMPILE_FEATURES cuda_std_${cuda_std})
 
-    set(example_args --timeout 0.1)
-    # The custom_criterion example doesn't support the --min-time argument:
-    if (NOT "${example_src}" STREQUAL "custom_criterion.cu")
-      list(APPEND example_args --min-time 1e-5)
-    endif()
+    set(example_args --timeout 0.1 --min-time 1e-5)
 
     add_test(NAME ${example_name}
       COMMAND "$<TARGET_FILE:${example_name}>" ${example_args})

nvbench/benchmark_base.cuh

@@ -29,6 +29,7 @@
 #endif
 
 #include <nvbench/axes_metadata.cuh>
+#include <nvbench/detail/validate_min_time.cuh>
 #include <nvbench/device_info.cuh>
 #include <nvbench/state.cuh>
 #include <nvbench/stopping_criterion.cuh>
@@ -169,6 +170,16 @@ struct benchmark_base
   }
   /// @}
 
+  /// Accumulate at least this much measurement time when the stopping criterion requires it. @{
+  [[nodiscard]] nvbench::float64_t get_min_time() const { return m_min_time; }
+  benchmark_base &set_min_time(nvbench::float64_t min_time)
+  {
+    nvbench::detail::validate_min_time(min_time);
+    m_min_time = min_time;
+    return *this;
+  }
+  /// @}
+
   /// Execute this many warmup runs before collecting cold measurement samples. @{
   [[nodiscard]] nvbench::int64_t get_cold_warmup_runs() const { return m_cold_warmup_runs; }
   benchmark_base &set_cold_warmup_runs(nvbench::int64_t cold_warmup_runs)
@@ -348,6 +359,7 @@ protected:
   nvbench::int64_t m_min_samples{10};
   nvbench::int64_t m_cold_warmup_runs{1};
 
+  nvbench::float64_t m_min_time{0.5};
   nvbench::float64_t m_cold_max_warmup_walltime{-1.};
   nvbench::float64_t m_skip_time{-1.};
   nvbench::float64_t m_timeout{15.};

nvbench/benchmark_base.cxx

@@ -50,6 +50,7 @@ std::unique_ptr<benchmark_base> benchmark_base::clone() const
 
   result->m_min_samples              = m_min_samples;
   result->m_cold_warmup_runs         = m_cold_warmup_runs;
+  result->m_min_time                 = m_min_time;
   result->m_cold_max_warmup_walltime = m_cold_max_warmup_walltime;
 
   result->m_skip_time = m_skip_time;

nvbench/detail/measure_cold.cu

@@ -51,6 +51,7 @@ measure_cold_base::measure_cold_base(state &exec_state)
     , m_check_throttling(!exec_state.get_run_once())
     , m_min_samples{exec_state.get_min_samples()}
     , m_cold_warmup_runs{exec_state.get_cold_warmup_runs()}
+    , m_min_time{exec_state.get_min_time()}
     , m_cold_max_warmup_walltime{exec_state.get_cold_max_warmup_walltime()}
     , m_skip_time{exec_state.get_skip_time()}
     , m_timeout{exec_state.get_timeout()}
@@ -179,13 +180,13 @@ bool measure_cold_base::is_finished()
     return true;
   }
 
-  // Check that we've gathered enough samples:
-  if (m_total_samples >= m_min_samples)
+  const nvbench::stopping_context context{m_total_samples,
+                                          m_total_cuda_time,
+                                          m_min_samples,
+                                          m_min_time};
+  if (m_stopping_criterion.is_finished(context))
   {
-    if (m_stopping_criterion.is_finished())
-    {
-      return true;
-    }
+    return true;
   }
 
   // Check for timeouts:
@@ -211,13 +212,16 @@ void measure_cold_base::log_timeout_warnings(nvbench::printer_base &printer,
 
   const auto timeout = m_walltime_timer.get_duration();
 
-  log_measurement_timeout_warnings(printer,
-                                   m_criterion_params,
-                                   timeout,
-                                   m_total_samples,
-                                   m_min_samples,
-                                   m_total_cuda_time,
-                                   cuda_stdev_noise);
+  log_measurement_timeout_warnings(
+    printer,
+    m_criterion_params,
+    timeout,
+    m_total_samples,
+    m_min_samples,
+    m_min_time,
+    min_time_can_block_stop(m_stopping_criterion, m_total_samples, m_total_cuda_time, m_min_time),
+    m_total_cuda_time,
+    cuda_stdev_noise);
 }
 
 void measure_cold_base::generate_summaries()

nvbench/detail/measure_cold.cuh

@@ -123,6 +123,7 @@ protected:
   nvbench::int64_t m_min_samples{};
   nvbench::int64_t m_cold_warmup_runs{1};
 
+  nvbench::float64_t m_min_time{};
   nvbench::float64_t m_cold_max_warmup_walltime{};
   nvbench::float64_t m_skip_time{};
   nvbench::float64_t m_timeout{};

nvbench/detail/measure_cpu_only.cuh

@@ -81,6 +81,7 @@ protected:
 
   nvbench::int64_t m_min_samples{};
 
+  nvbench::float64_t m_min_time{};
   nvbench::float64_t m_skip_time{};
   nvbench::float64_t m_timeout{};
 

nvbench/detail/measure_cpu_only.cxx

@@ -44,6 +44,7 @@ measure_cpu_only_base::measure_cpu_only_base(state &exec_state)
         exec_state.get_stopping_criterion())}
     , m_run_once{exec_state.get_run_once()}
     , m_min_samples{exec_state.get_min_samples()}
+    , m_min_time{exec_state.get_min_time()}
     , m_skip_time{exec_state.get_skip_time()}
     , m_timeout{exec_state.get_timeout()}
 {
@@ -96,13 +97,13 @@ bool measure_cpu_only_base::is_finished()
     return true;
   }
 
-  // Check that we've gathered enough samples:
-  if (m_total_samples >= m_min_samples)
+  const nvbench::stopping_context context{m_total_samples,
+                                          m_total_cpu_time,
+                                          m_min_samples,
+                                          m_min_time};
+  if (m_stopping_criterion.is_finished(context))
   {
-    if (m_stopping_criterion.is_finished())
-    {
-      return true;
-    }
+    return true;
   }
 
   // Check for timeouts:
@@ -272,6 +273,11 @@ void measure_cpu_only_base::generate_summaries()
                                        timeout,
                                        m_total_samples,
                                        m_min_samples,
+                                       m_min_time,
+                                       min_time_can_block_stop(m_stopping_criterion,
+                                                               m_total_samples,
+                                                               m_total_cpu_time,
+                                                               m_min_time),
                                        m_total_cpu_time,
                                        cpu_stdev_noise);
     }

nvbench/detail/measure_hot.cu

@@ -35,13 +35,16 @@
 namespace nvbench::detail
 {
 
+// Keep hot batches large enough to avoid one-launch batches when --min-time is
+// disabled or set to an extremely small positive value.
+constexpr nvbench::float64_t smallest_hot_batch_target_time = 100e-6;
+
 measure_hot_base::measure_hot_base(state &exec_state)
     : m_state{exec_state}
     , m_launch{exec_state.get_cuda_stream()}
     , m_min_samples{exec_state.get_min_samples()}
-    , m_min_time{exec_state.get_criterion_params().has_value("min-time")
-                   ? exec_state.get_criterion_params().get_float64("min-time")
-                   : 0.5}
+    , m_min_time{exec_state.get_min_time()}
+    , m_batch_target_time{m_min_time}
     , m_skip_time{exec_state.get_skip_time()}
     , m_timeout{exec_state.get_timeout()}
 {
@@ -55,16 +58,19 @@ measure_hot_base::measure_hot_base(state &exec_state)
     // If the cold measurement ran successfully, disable skip_time. It'd just
     // be annoying to skip now.
     m_skip_time = -1;
+
+    m_batch_target_time = std::max(m_min_time, smallest_hot_batch_target_time);
   }
   catch (...)
   {
     // If the above threw an exception, we don't have a cold measurement to use.
-    // Estimate a target_time between m_min_time and m_timeout.
-    // Use the average of the min_time and timeout, but don't go over 5x
-    // min_time in case timeout is huge.
+    // Estimate an internal batch target from min_time and timeout, then apply
+    // the minimum floor below. Use the average of min_time and timeout, but
+    // don't go over 5x min_time in case timeout is huge.
     // We could expose a `target_time` property on benchmark_base/state if
     // needed.
-    m_min_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5);
+    const auto fallback_batch_target_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5);
+    m_batch_target_time = std::max(smallest_hot_batch_target_time, fallback_batch_target_time);
   }
 }
 

nvbench/detail/measure_hot.cuh

@@ -94,6 +94,7 @@ protected:
 
   nvbench::int64_t m_min_samples{};
   nvbench::float64_t m_min_time{};
+  nvbench::float64_t m_batch_target_time{};
 
   nvbench::float64_t m_skip_time{};
   nvbench::float64_t m_timeout{};
@@ -147,7 +148,7 @@ private:
     // The .95 factor here pads the batch_size a bit to avoid needing a second
     // batch due to noise.
     const auto time_estimate = m_cuda_timer.get_duration() * 0.95;
-    auto batch_size          = static_cast<nvbench::int64_t>(m_min_time / time_estimate);
+    auto batch_size          = static_cast<nvbench::int64_t>(m_batch_target_time / time_estimate);
 
     do
     {
@@ -199,7 +200,7 @@ private:
 
       // Predict number of remaining iterations:
       batch_size = static_cast<nvbench::int64_t>(
-        (m_min_time - m_total_cuda_time) /
+        (m_batch_target_time - m_total_cuda_time) /
         (m_total_cuda_time / static_cast<nvbench::float64_t>(m_total_samples)));
 
       if (m_total_cuda_time > m_min_time && // min time okay

nvbench/detail/measure_timeout_warnings.cuh

@@ -52,16 +52,35 @@ get_float64_criterion_param(const nvbench::criterion_params &params, const std::
   return params.get_float64(name);
 }
 
+inline bool min_time_can_block_stop(const nvbench::stopping_criterion_base &criterion,
+                                    nvbench::int64_t total_samples,
+                                    nvbench::float64_t accumulated_time,
+                                    nvbench::float64_t min_time)
+{
+  const nvbench::stopping_context min_samples_satisfied_context{total_samples,
+                                                                accumulated_time,
+                                                                total_samples,
+                                                                min_time};
+  const nvbench::stopping_context min_time_satisfied_context{total_samples,
+                                                             accumulated_time,
+                                                             total_samples,
+                                                             accumulated_time};
+
+  return !criterion.is_eligible_to_stop(min_samples_satisfied_context) &&
+         criterion.is_eligible_to_stop(min_time_satisfied_context);
+}
+
 inline void log_measurement_timeout_warnings(nvbench::printer_base &printer,
                                              const nvbench::criterion_params &criterion_params,
                                              nvbench::float64_t timeout,
                                              nvbench::int64_t total_samples,
                                              nvbench::int64_t min_samples,
+                                             nvbench::float64_t min_time,
+                                             bool can_min_time_block_stop,
                                              nvbench::float64_t accumulated_time,
                                              std::optional<nvbench::float64_t> stdev_noise)
 {
   const auto max_noise = get_float64_criterion_param(criterion_params, "max-noise");
-  const auto min_time  = get_float64_criterion_param(criterion_params, "min-time");
 
   const auto enough_samples_for_noise =
     statistics::has_enough_samples_for_noise_estimate(total_samples);
@@ -102,15 +121,15 @@ inline void log_measurement_timeout_warnings(nvbench::printer_base &printer,
                             total_samples,
                             min_samples));
   }
-  if (min_time && accumulated_time < *min_time)
+  if (can_min_time_block_stop && accumulated_time < min_time)
   {
     printer.log(nvbench::log_level::warn,
                 fmt::format("Current measurement timed out ({:0.2f}s) "
                             "before accumulating min_time ({:0.2f}s < "
                             "{:0.2f}s)",
                             timeout,
                             accumulated_time,
-                            *min_time));
+                            min_time));
   }
 }
 

nvbench/detail/sample_count_criterion.cuh

@@ -45,6 +45,7 @@ protected:
   virtual void do_initialize() override;
   virtual void do_add_measurement(nvbench::float64_t measurement) override;
   virtual bool do_is_finished() override;
+  virtual bool do_is_eligible_to_stop(const nvbench::stopping_context &context) const override;
 };
 
 } // namespace nvbench::detail

nvbench/detail/sample_count_criterion.cxx

@@ -45,4 +45,9 @@ bool sample_count_criterion::do_is_finished()
   return m_total_samples >= m_params.get_int64("target-samples");
 }
 
+bool sample_count_criterion::do_is_eligible_to_stop(const nvbench::stopping_context &) const
+{
+  return true;
+}
+
 } // namespace nvbench::detail

nvbench/detail/stdrel_criterion.cuh

@@ -50,6 +50,7 @@ protected:
   virtual void do_initialize() override;
   virtual void do_add_measurement(nvbench::float64_t measurement) override;
   virtual bool do_is_finished() override;
+  virtual bool do_is_eligible_to_stop(const nvbench::stopping_context &context) const override;
 };
 
 } // namespace nvbench::detail