fix: shard oversized topic scan by timestamp

Author: aiwarelife

scripts/sync_topic_plugins.py

@@ -312,21 +312,28 @@ def collect_query(self, query_text: str, first: dict[str, Any]) -> list[dict[str
             f"search shard did not converge: latest={latest_total}, unique={len(seen)}"
         )
 
-    def collect_date_range(
+    def collect_time_range(
         self,
-        start: dt.date,
-        end: dt.date,
+        start: dt.datetime,
+        end: dt.datetime,
     ) -> list[dict[str, Any]]:
-        query_text = f"topic:{TOPIC} created:{start.isoformat()}..{end.isoformat()}"
+        def timestamp(value: dt.datetime) -> str:
+            return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        query_text = f"topic:{TOPIC} created:{timestamp(start)}..{timestamp(end)}"
         first = self.search_page(query_text, 1)
         total = int(first.get("total_count", 0))
         if total <= 1000:
             return self.collect_query(query_text, first)
         if start >= end:
-            raise SyncError(f"more than 1,000 topic repositories were created on {start}")
-        midpoint = start + (end - start) // 2
-        return self.collect_date_range(start, midpoint) + self.collect_date_range(
-            midpoint + dt.timedelta(days=1), end
+            raise SyncError(
+                f"more than 1,000 topic repositories were created at {timestamp(start)}"
+            )
+        midpoint = start + dt.timedelta(
+            seconds=int((end - start).total_seconds()) // 2
+        )
+        return self.collect_time_range(start, midpoint) + self.collect_time_range(
+            midpoint + dt.timedelta(seconds=1), end
         )
 
     def search(self, max_pages: int | None = None) -> tuple[list[dict[str, Any]], int]:
@@ -344,9 +351,10 @@ def search(self, max_pages: int | None = None) -> tuple[list[dict[str, Any]], in
         if total <= 1000:
             return self.collect_query(base_query, first), total
 
-        today = dt.datetime.now(dt.timezone.utc).date()
-        repositories = self.collect_date_range(dt.date(2008, 1, 1), today - dt.timedelta(days=1))
-        repositories += self.collect_date_range(today, today)
+        now = dt.datetime.now(dt.timezone.utc).replace(microsecond=0)
+        repositories = self.collect_time_range(
+            dt.datetime(2008, 1, 1, tzinfo=dt.timezone.utc), now
+        )
         deduplicated = {int(item["id"]): item for item in repositories}
         if len(deduplicated) != len(repositories):
             raise SyncError("date-sharded topic search returned duplicate repository IDs")

tests/test_topic_sync.py

@@ -413,6 +413,31 @@ def search_page(self, query_text, page):
         items = GrowingClient().collect_query("topic:dsh-plugin", first)
         self.assertEqual(len(items), 104)
 
+    def test_oversized_day_is_split_by_timestamp(self):
+        class TimestampClient(topic_sync.GitHubClient):
+            def __init__(self):
+                self.queries = []
+
+            def search_page(self, query_text, page):
+                self.queries.append(query_text)
+                if "00:00:00Z..2026-08-14T23:59:59Z" in query_text:
+                    return {"total_count": 1500, "items": []}
+                if "00:00:00Z..2026-08-14T11:59:59Z" in query_text:
+                    return {"total_count": 1, "items": [{"id": 1}]}
+                if "12:00:00Z..2026-08-14T23:59:59Z" in query_text:
+                    return {"total_count": 1, "items": [{"id": 2}]}
+                raise AssertionError(query_text)
+
+        client = TimestampClient()
+        items = client.collect_time_range(
+            topic_sync.dt.datetime(2026, 8, 14, tzinfo=topic_sync.dt.timezone.utc),
+            topic_sync.dt.datetime(
+                2026, 8, 14, 23, 59, 59, tzinfo=topic_sync.dt.timezone.utc
+            ),
+        )
+        self.assertEqual([item["id"] for item in items], [1, 2])
+        self.assertEqual(len(client.queries), 3)
+
     def test_changelog_and_report_are_idempotent(self):
         promoted = [
             {