From 44497fa7467ce32a3790033a0ef41df59e92cf4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuna=20K=C4=B1l=C4=B1=C3=A7?= Date: Fri, 16 Oct 2026 23:20:15 +0300 Subject: [PATCH 1/2] fix(warmup): honor per-benchmark warmup without MinTime --- src/benchmark_runner.cc | 11 ++-- test/CMakeLists.txt | 1 + test/benchmark_warmup_gtest.cc | 102 +++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 test/benchmark_warmup_gtest.cc diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 0d267a38d5..6d889f55aa 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -291,12 +291,11 @@ BenchmarkRunner::BenchmarkRunner( min_time(FLAGS_benchmark_dry_run ? 0 : ComputeMinTime(b_, parsed_benchtime_flag)), - min_warmup_time( - FLAGS_benchmark_dry_run - ? 0 - : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time)), + min_warmup_time(FLAGS_benchmark_dry_run + ? 0 + : (b.min_warmup_time() > 0.0 + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time)), warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)), repeats(FLAGS_benchmark_dry_run ? 1 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f33209cdb6..6ed0e1f510 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -251,6 +251,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) endmacro() add_gtest(benchmark_gtest) + add_gtest(benchmark_warmup_gtest) add_gtest(benchmark_name_gtest) add_gtest(benchmark_random_interleaving_gtest) add_gtest(commandlineflags_gtest) diff --git a/test/benchmark_warmup_gtest.cc b/test/benchmark_warmup_gtest.cc new file mode 100644 index 0000000000..8111d5f29f --- /dev/null +++ b/test/benchmark_warmup_gtest.cc @@ -0,0 +1,102 @@ +#include +#include + +#include "../src/commandlineflags.h" +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" + +namespace benchmark { + +BM_DECLARE_bool(benchmark_dry_run); +BM_DECLARE_string(benchmark_min_time); +BM_DECLARE_double(benchmark_min_warmup_time); + +namespace { + +class CapturingReporter : public BenchmarkReporter { + public: + bool ReportContext(const Context&) override { return true; } + void ReportRuns(const std::vector& reports) override { + runs.insert(runs.end(), reports.begin(), reports.end()); + } + + std::vector runs; +}; + +class WarmupTest : public testing::Test { + protected: + void SetUp() override { + FLAGS_benchmark_dry_run = false; + FLAGS_benchmark_min_time = "0.01s"; + FLAGS_benchmark_min_warmup_time = 0.0; + benchmark_ = RegisterBenchmark("Warmup", [this](State& state) { + ++invocations_; + // Manual time makes calibration independent of the clock. The first + // invocation is deliberately slower so we can detect a reported warmup. + for (auto _ : state) { + state.SetIterationTime(invocations_ == 1 ? 2.0 : 1.0); + } + }); + benchmark_->UseManualTime()->Repetitions(1); + } + + void TearDown() override { + ClearRegisteredBenchmarks(); + FLAGS_benchmark_dry_run = saved_dry_run_; + FLAGS_benchmark_min_time = saved_min_time_; + FLAGS_benchmark_min_warmup_time = saved_min_warmup_time_; + } + + void ExpectRun(int expected_invocations, double expected_time) { + CapturingReporter reporter; + ASSERT_EQ(RunSpecifiedBenchmarks(&reporter, "^Warmup/"), 1u); + EXPECT_EQ(invocations_, expected_invocations); + ASSERT_EQ(reporter.runs.size(), 1u); + const auto& run = reporter.runs.front(); + EXPECT_EQ(run.skipped, internal::NotSkipped); + EXPECT_EQ(run.iterations, 1); + EXPECT_EQ(run.repetitions, 1); + EXPECT_DOUBLE_EQ(run.real_accumulated_time, expected_time); + } + + Benchmark* benchmark_ = nullptr; + + private: + const bool saved_dry_run_ = FLAGS_benchmark_dry_run; + const std::string saved_min_time_ = FLAGS_benchmark_min_time; + const double saved_min_warmup_time_ = FLAGS_benchmark_min_warmup_time; + int invocations_ = 0; +}; + +TEST_F(WarmupTest, LocalWarmupWithoutMinTime) { + benchmark_->MinWarmUpTime(0.01); + ExpectRun(2, 1.0); +} + +TEST_F(WarmupTest, LocalWarmupWithMinTime) { + benchmark_->MinWarmUpTime(0.01)->MinTime(0.01); + ExpectRun(2, 1.0); +} + +TEST_F(WarmupTest, LocalWarmupOverridesGlobalWithoutMinTime) { + FLAGS_benchmark_min_warmup_time = 3.0; + benchmark_->MinWarmUpTime(0.01); + ExpectRun(2, 1.0); +} + +TEST_F(WarmupTest, GlobalWarmupFallback) { + FLAGS_benchmark_min_warmup_time = 0.01; + ExpectRun(2, 1.0); +} + +TEST_F(WarmupTest, NoWarmup) { ExpectRun(1, 2.0); } + +TEST_F(WarmupTest, DryRunSkipsWarmup) { + FLAGS_benchmark_dry_run = true; + FLAGS_benchmark_min_warmup_time = 3.0; + benchmark_->MinWarmUpTime(0.01)->MinTime(3.0)->Repetitions(3); + ExpectRun(1, 2.0); +} + +} // namespace +} // namespace benchmark From ffadc528143ef06f0c44aa9d2b50a08a00dad11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuna=20K=C4=B1l=C4=B1=C3=A7?= Date: Sat, 17 Oct 2026 00:16:43 +0300 Subject: [PATCH 2/2] fix: remove refression tests --- test/CMakeLists.txt | 1 - test/benchmark_warmup_gtest.cc | 102 --------------------------------- 2 files changed, 103 deletions(-) delete mode 100644 test/benchmark_warmup_gtest.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6ed0e1f510..f33209cdb6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -251,7 +251,6 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) endmacro() add_gtest(benchmark_gtest) - add_gtest(benchmark_warmup_gtest) add_gtest(benchmark_name_gtest) add_gtest(benchmark_random_interleaving_gtest) add_gtest(commandlineflags_gtest) diff --git a/test/benchmark_warmup_gtest.cc b/test/benchmark_warmup_gtest.cc deleted file mode 100644 index 8111d5f29f..0000000000 --- a/test/benchmark_warmup_gtest.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include -#include - -#include "../src/commandlineflags.h" -#include "benchmark/benchmark.h" -#include "gtest/gtest.h" - -namespace benchmark { - -BM_DECLARE_bool(benchmark_dry_run); -BM_DECLARE_string(benchmark_min_time); -BM_DECLARE_double(benchmark_min_warmup_time); - -namespace { - -class CapturingReporter : public BenchmarkReporter { - public: - bool ReportContext(const Context&) override { return true; } - void ReportRuns(const std::vector& reports) override { - runs.insert(runs.end(), reports.begin(), reports.end()); - } - - std::vector runs; -}; - -class WarmupTest : public testing::Test { - protected: - void SetUp() override { - FLAGS_benchmark_dry_run = false; - FLAGS_benchmark_min_time = "0.01s"; - FLAGS_benchmark_min_warmup_time = 0.0; - benchmark_ = RegisterBenchmark("Warmup", [this](State& state) { - ++invocations_; - // Manual time makes calibration independent of the clock. The first - // invocation is deliberately slower so we can detect a reported warmup. - for (auto _ : state) { - state.SetIterationTime(invocations_ == 1 ? 2.0 : 1.0); - } - }); - benchmark_->UseManualTime()->Repetitions(1); - } - - void TearDown() override { - ClearRegisteredBenchmarks(); - FLAGS_benchmark_dry_run = saved_dry_run_; - FLAGS_benchmark_min_time = saved_min_time_; - FLAGS_benchmark_min_warmup_time = saved_min_warmup_time_; - } - - void ExpectRun(int expected_invocations, double expected_time) { - CapturingReporter reporter; - ASSERT_EQ(RunSpecifiedBenchmarks(&reporter, "^Warmup/"), 1u); - EXPECT_EQ(invocations_, expected_invocations); - ASSERT_EQ(reporter.runs.size(), 1u); - const auto& run = reporter.runs.front(); - EXPECT_EQ(run.skipped, internal::NotSkipped); - EXPECT_EQ(run.iterations, 1); - EXPECT_EQ(run.repetitions, 1); - EXPECT_DOUBLE_EQ(run.real_accumulated_time, expected_time); - } - - Benchmark* benchmark_ = nullptr; - - private: - const bool saved_dry_run_ = FLAGS_benchmark_dry_run; - const std::string saved_min_time_ = FLAGS_benchmark_min_time; - const double saved_min_warmup_time_ = FLAGS_benchmark_min_warmup_time; - int invocations_ = 0; -}; - -TEST_F(WarmupTest, LocalWarmupWithoutMinTime) { - benchmark_->MinWarmUpTime(0.01); - ExpectRun(2, 1.0); -} - -TEST_F(WarmupTest, LocalWarmupWithMinTime) { - benchmark_->MinWarmUpTime(0.01)->MinTime(0.01); - ExpectRun(2, 1.0); -} - -TEST_F(WarmupTest, LocalWarmupOverridesGlobalWithoutMinTime) { - FLAGS_benchmark_min_warmup_time = 3.0; - benchmark_->MinWarmUpTime(0.01); - ExpectRun(2, 1.0); -} - -TEST_F(WarmupTest, GlobalWarmupFallback) { - FLAGS_benchmark_min_warmup_time = 0.01; - ExpectRun(2, 1.0); -} - -TEST_F(WarmupTest, NoWarmup) { ExpectRun(1, 2.0); } - -TEST_F(WarmupTest, DryRunSkipsWarmup) { - FLAGS_benchmark_dry_run = true; - FLAGS_benchmark_min_warmup_time = 3.0; - benchmark_->MinWarmUpTime(0.01)->MinTime(3.0)->Repetitions(3); - ExpectRun(1, 2.0); -} - -} // namespace -} // namespace benchmark