Merge pull request #3298 from dolthub/test/retire-wide-fetch-hotspots

Retire two wide-fetch hotspots to nightly regression seeds

Author: timsehn

…rformance-hotspot-corpus/issue_3251.json …erformance-hotspot-seeds/issue_3251.jsontest/performance-hotspot-corpus/issue_3251.json renamed to test/performance-hotspot-seeds/issue_3251.json test/performance-hotspot-corpus/issue_3251.json renamed to test/performance-hotspot-seeds/issue_3251.json

@@ -38,5 +38,32 @@
   "fingerprint": "0a60021380f251f8dc7d69fc",
   "family": "9dba6267f4d215e47221145d",
   "issue": 3251,
-  "category": "wide_fetches"
+  "category": "wide_fetches",
+  "discovery": {
+    "source_commit": "1066294abad4c33a71ee6ba6615a6460fa32fec9",
+    "median_ratio": 3.9541806448308536,
+    "minimum_ratio": 3.922268107218506,
+    "pairs": [
+      {
+        "doltlite_ms": 772.034,
+        "sqlite_ms": 195.245
+      },
+      {
+        "doltlite_ms": 772.173,
+        "sqlite_ms": 196.869
+      },
+      {
+        "doltlite_ms": 776.307,
+        "sqlite_ms": 194.524
+      },
+      {
+        "doltlite_ms": 773.528,
+        "sqlite_ms": 195.892
+      },
+      {
+        "doltlite_ms": 773.62,
+        "sqlite_ms": 194.39
+      }
+    ]
+  }
 }

…rformance-hotspot-corpus/issue_3252.json …erformance-hotspot-seeds/issue_3252.jsontest/performance-hotspot-corpus/issue_3252.json renamed to test/performance-hotspot-seeds/issue_3252.json test/performance-hotspot-corpus/issue_3252.json renamed to test/performance-hotspot-seeds/issue_3252.json

@@ -38,5 +38,32 @@
   "fingerprint": "d7719c8ea14f6955fbf79c63",
   "family": "695846a28e78b94f29e8a5ae",
   "issue": 3252,
-  "category": "wide_fetches"
+  "category": "wide_fetches",
+  "discovery": {
+    "source_commit": "1066294abad4c33a71ee6ba6615a6460fa32fec9",
+    "median_ratio": 3.339452294215503,
+    "minimum_ratio": 3.3283229433667714,
+    "pairs": [
+      {
+        "doltlite_ms": 631.07,
+        "sqlite_ms": 189.606
+      },
+      {
+        "doltlite_ms": 634.425,
+        "sqlite_ms": 188.976
+      },
+      {
+        "doltlite_ms": 634.349,
+        "sqlite_ms": 189.956
+      },
+      {
+        "doltlite_ms": 636.29,
+        "sqlite_ms": 189.723
+      },
+      {
+        "doltlite_ms": 628.064,
+        "sqlite_ms": 188.41
+      }
+    ]
+  }
 }

test/performance_hotspot_seeds_test.py

@@ -70,8 +70,8 @@ def test_index_cache_warmup_precedes_transaction_and_timer(self):
 
     def test_retired_corpus_moves_to_search_and_random_exploration_continues(self):
         specs = list(seeds.specs())
-        self.assertEqual(len(specs), 16)
-        self.assertEqual(sum(len(cases) for _, cases, _ in specs), 35)
+        self.assertEqual(len(specs), 17)
+        self.assertEqual(sum(len(cases) for _, cases, _ in specs), 37)
         names = {path.stem for path in seeds.SEED_DIR.glob('*.json')}
         wide_cases = {'create_index', 'delete_batch', 'distinct', 'index_fetch',
                       'join_pk', 'point_payload', 'point_pk', 'range_pk',
@@ -87,6 +87,7 @@ def test_retired_corpus_moves_to_search_and_random_exploration_continues(self):
                                   'scan_after_update_wide_text_pk',
                                   'issue_3249', 'issue_3250',
                                   'issue_3263', 'issue_3264', 'issue_3266',
+                                  'issue_3251', 'issue_3252',
                                   'bulk_deletes_delete_plain'})
         wide = [(p, cases, setup) for p, cases, setup in specs if p.payload==16384]
         self.assertEqual(len(wide), 1)
@@ -108,7 +109,7 @@ def test_retired_corpus_moves_to_search_and_random_exploration_continues(self):
             for index, spec in enumerate(specs):
                 self.assertEqual(next(generated), (index, *spec, 'retired'))
             index, profile, cases, setup, origin = next(generated)
-            self.assertEqual(index, 16)
+            self.assertEqual(index, 17)
             self.assertEqual(len(cases), 4)
             self.assertEqual({case.recipe['operator'] for case in cases}, {'update_blob'})
             self.assertIn('CREATE TABLE u', setup)

test/performance_hotspots_test.py

@@ -112,7 +112,7 @@ def test_sql_failure_is_not_a_timing(self):
             hotspots.run(["sh", "-c", "echo 'error' >&2"])
 
     def test_main_measures_only_remaining_workloads_for_all_arms(self):
-        self.assertEqual(len(list((hotspots.TEST_DIR/'performance-hotspot-corpus').glob('*.json'))), 7)
+        self.assertEqual(len(list((hotspots.TEST_DIR/'performance-hotspot-corpus').glob('*.json'))), 5)
 
         with tempfile.TemporaryDirectory() as directory:
             result = Path(directory) / "results.tsv"
@@ -159,10 +159,10 @@ def test_main_measures_only_remaining_workloads_for_all_arms(self):
             self.assertIn("Add Column With Default", report.getvalue())
             self.assertEqual(report.getvalue().count("### "), 4)
             self.assertNotIn("### Integer Keys", report.getvalue())
-            self.assertTrue(all(len(call.args[2]) == 7 for call in measure_retained.call_args_list))
-            self.assertEqual(len(result.read_text().splitlines()), 8)
+            self.assertTrue(all(len(call.args[2]) == 5 for call in measure_retained.call_args_list))
+            self.assertEqual(len(result.read_text().splitlines()), 6)
             self.assertIn('add_column\tadd_column_default\t100000\t100000\n', result.read_text())
-            self.assertEqual(len(raw.read_text().splitlines()), 17)
+            self.assertEqual(len(raw.read_text().splitlines()), 13)
 
     def test_medians_raw_samples_and_stock_report(self):
         with tempfile.TemporaryDirectory() as directory:
@@ -254,7 +254,7 @@ def test_retired_corpus_categories_and_shared_fixtures(self):
                 Path(directory), hotspots.TEST_DIR/'performance-hotspot-seeds')
         counts = {category: 0 for category in ('wide_rows', 'narrow_rows', 'zero_row_updates',
                                               'in_transaction_mutations', 'integer_keys',
-                                              'bulk_deletes', 'small_cache')}
+                                              'bulk_deletes', 'small_cache', 'wide_fetches')}
         for name, bundle, databases in fixtures:
             category = hotspots.section_of(name)
             counts[category] += 1
@@ -274,13 +274,13 @@ def test_retired_corpus_categories_and_shared_fixtures(self):
                 self.assertTrue(bundle['expected'].startswith('0|'))
         self.assertEqual(counts, {'wide_rows': 12, 'narrow_rows': 5, 'zero_row_updates': 2,
                                  'in_transaction_mutations': 4, 'integer_keys': 2,
-                                 'bulk_deletes': 1, 'small_cache': 3})
-        self.assertEqual(sql.call_count, 30)
+                                 'bulk_deletes': 1, 'small_cache': 3, 'wide_fetches': 2})
+        self.assertEqual(sql.call_count, 33)
         grouped = {}
         for name, bundle, databases in fixtures:
             previous = grouped.setdefault(bundle['setup_sql'], databases)
             self.assertEqual(databases, previous)
-        self.assertEqual(len(grouped), 13)
+        self.assertEqual(len(grouped), 14)
 
     def test_retained_gate_measures_fixed_batches(self):
         bundle = json.loads((hotspots.TEST_DIR/'performance-hotspot-seeds/narrow_rows_point_pk.json').read_text())
@@ -335,7 +335,7 @@ def test_discovered_sections_are_reported_and_gated(self):
 
     def test_remaining_issue_themes_are_reported_and_gated(self):
         expected = {
-            'wide_fetches': {3251, 3252, 3253},
+            'wide_fetches': {3253},
             'small_cache': {3265},
             'after_deletes': {3258, 3270, 3271},
         }