PR-7501 counting granules and exit condition for prepare loop

Author: benbart

ctorm/ctorm/config.py

@@ -24,6 +24,9 @@ class CtormConfig:
     source_buckets: list
     granules_sqs_queue_url: str
 
+    # This is the number of granules that will be prepared for test.
+    granule_goal: int = 2000000
+
     @classmethod
     def from_file(
         cls,

ctorm/ctorm/prepare.py

@@ -63,10 +63,10 @@ class CtormPrepare:
 
     def __init__(self, cfg: CtormConfig):
         self.cfg = cfg
+        self.total_granules = 0
 
     def prepare(self):
-        goal = 50
-        while goal > 0:  # TODO: replace with while true and an exit condition
+        while self.total_granules < self.cfg.granule_goal:
             # This is the loop that creates a SQS message from multiple objects.
             sqs_msg = CtormSqsMessage(self.cfg)
             for b in self.cfg.source_buckets:
@@ -86,10 +86,10 @@ def prepare(self):
                     file = self.process_ummg(ummg, b)
                     if file:
                         sqs_msg.add_file(file)
-                    goal -= 1
+                    self.total_granules += 1
                 b.next_cont_token = page.get("NextContinuationToken")
 
-            log.debug("goal: %d", goal)
+            log.debug("tot granules: %d/%d", self.total_granules, self.cfg.granule_goal)
             log.debug("sqs_msg size: %d", len(sqs_msg.to_json()))
             log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size())
             if sqs_msg.check_message_size():