From 69f26f8de8ce7a0b8d6c27d475afef45adceac0f Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Tue, 14 Jul 2020 21:19:47 +0530
Subject: [PATCH 1/8] making the option --limit in pull usable.

---
 lieer/gmailieer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 269d04b..59c58f9 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -604,8 +604,8 @@ def full_pull (self):
     self.bar_close ()
 
     if self.local.config.remove_local_messages:
-      if self.limit and not self.dry_run:
-        raise ValueError('--limit with "remove_local_messages" will cause lots of messages to be deleted')
+      if self.limit and self.local.state.last_historyId != 0:
+        raise AttributeError('Previous synchronization state detected, remove the --limit tag.')
 
       # removing files that have been deleted remotely
       all_remote = set (message_gids)

From a68b96d46a2d39823e99b03a205883705102005f Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Wed, 15 Jul 2020 17:34:31 +0530
Subject: [PATCH 2/8] Added limit to the permanent variables in set.

---
 lieer/gmailieer.py | 7 ++++++-
 lieer/local.py     | 6 ++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 59c58f9..6219258 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -193,7 +193,8 @@ def main (self):
         help = 'Remove messages that have been deleted on the remote (default is on)')
     parser_set.add_argument ('--no-remove-local-messages', action = 'store_true', default = False,
         help = 'Do not remove messages that have been deleted on the remote')
-
+    parser_set.add_argument ('--limit', type = int, default = None,
+        help = 'Maximum number of messages to pull (soft limit, GMail may return more), note that this may upset the tally of synchronized messages.')
     parser_set.set_defaults (func = self.set)
 
 
@@ -785,6 +786,9 @@ def set (self, args):
     if args.remove_local_messages:
       self.local.config.set_remove_local_messages (True)
 
+    if args.limit is not None:
+   	  self.local.config.set_limit (args.limit)
+
     if args.no_remove_local_messages:
       self.local.config.set_remove_local_messages (False)
 
@@ -802,6 +806,7 @@ def set (self, args):
     print ("historyId .........: %d" % self.local.state.last_historyId)
     print ("lastmod ...........: %d" % self.local.state.lastmod)
     print ("Timeout ...........: %f" % self.local.config.timeout)
+    print ("Limit .............: %d" % self.local.config.limit)
     print ("File extension ....: %s" % self.local.config.file_extension)
     print ("Remove local messages .....:", self.local.config.remove_local_messages)
     print ("Drop non existing labels...:", self.local.config.drop_non_existing_label)
diff --git a/lieer/local.py b/lieer/local.py
index 41d1d36..9bfacf0 100644
--- a/lieer/local.py
+++ b/lieer/local.py
@@ -102,6 +102,7 @@ def __init__ (self, config_f):
       self.ignore_tags = set(self.json.get ('ignore_tags', []))
       self.ignore_remote_labels = set(self.json.get ('ignore_remote_labels', Remote.DEFAULT_IGNORE_LABELS))
       self.file_extension = self.json.get ('file_extension', '')
+      self.limit = self.json.get ('limit', None)
 
     def write (self):
       self.json = {}
@@ -115,6 +116,7 @@ def write (self):
       self.json['ignore_remote_labels'] = list(self.ignore_remote_labels)
       self.json['remove_local_messages'] = self.remove_local_messages
       self.json['file_extension'] = self.file_extension
+      self.json['limit'] = self.limit
 
       if os.path.exists (self.config_f):
         shutil.copyfile (self.config_f, self.config_f + '.bak')
@@ -174,6 +176,10 @@ def set_file_extension (self, t):
         print ("Failed creating test file with file extension: " + t + ", not set.")
         raise
 
+    def set_limit (self,l):
+        self.limit = l
+        self.write()
+
 
   class State:
     # last historyid of last synchronized message, anything that has happened

From efd619ee5fcf205d86f3d6e36f41916cf6395505 Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Wed, 15 Jul 2020 18:31:51 +0530
Subject: [PATCH 3/8] changed self.limit to self.local.config.limit

---
 lieer/gmailieer.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 6219258..627ad50 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -276,7 +276,6 @@ def setup (self, args, dry_run = False, load = False, block = False):
   def sync (self, args):
     self.setup (args, args.dry_run, True)
     self.force            = args.force
-    self.limit            = args.limit
     self.list_labels      = False
 
     self.remote.get_labels ()
@@ -294,7 +293,6 @@ def push (self, args, setup = False):
       self.setup (args, args.dry_run, True)
 
       self.force            = args.force
-      self.limit            = args.limit
 
       self.remote.get_labels ()
 
@@ -314,8 +312,8 @@ def push (self, args, setup = False):
       query = notmuch.Query (db, qry)
 
       messages = list(query.search_messages ())
-      if self.limit is not None and len(messages) > self.limit:
-        messages = messages[:self.limit]
+      if self.local.config.limit is not None and len(messages) > self.local.config.limit:
+        messages = messages[:self.local.config.limit]
 
       # get gids and filter out messages outside this repository
       messages, gids = self.local.messages_to_gids (messages)
@@ -345,8 +343,8 @@ def _got_msgs (ms):
       actions = [ a for a in actions if a ]
 
       # limit
-      if self.limit is not None and len(actions) >= self.limit:
-        actions = actions[:self.limit]
+      if self.local.config.limit is not None and len(actions) >= self.local.config.limit:
+        actions = actions[:self.local.config.limit]
 
       # push changes
       if len(actions) > 0:
@@ -387,7 +385,6 @@ def pull (self, args, setup = False):
 
       self.list_labels      = args.list_labels
       self.force            = args.force
-      self.limit            = args.limit
 
       self.remote.get_labels () # to make sure label map is initialized
 
@@ -423,7 +420,7 @@ def partial_pull (self):
 
         self.bar_update (len(hist))
 
-        if self.limit is not None and len(history) >= self.limit:
+        if self.local.config.limit is not None and len(history) >= self.local.config.limit:
           break
 
     except googleapiclient.errors.HttpError as excep:
@@ -589,7 +586,7 @@ def full_pull (self):
     # simple metadata like message ids.
     message_gids = []
     last_id      = self.remote.get_current_history_id (self.local.state.last_historyId)
-
+    print(self.local.config.limit)
     for mset in self.remote.all_messages ():
       (total, gids) = mset
 
@@ -599,14 +596,12 @@ def full_pull (self):
       for m in gids:
         message_gids.append (m['id'])
 
-      if self.limit is not None and len(message_gids) >= self.limit:
+      if self.local.config.limit is not None and len(message_gids) >= self.local.config.limit:
         break
 
     self.bar_close ()
 
     if self.local.config.remove_local_messages:
-      if self.limit and self.local.state.last_historyId != 0:
-        raise AttributeError('Previous synchronization state detected, remove the --limit tag.')
 
       # removing files that have been deleted remotely
       all_remote = set (message_gids)

From 0eadd6375a7f63feca8b0356614f2c331862d4f4 Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Wed, 15 Jul 2020 23:26:22 +0530
Subject: [PATCH 4/8] remove mails over the set limit - partial_pull

---
 lieer/gmailieer.py | 21 +++++++++++----------
 lieer/local.py     | 18 ++++++++++++++++++
 2 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 627ad50..270824d 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -59,10 +59,6 @@ def main (self):
     parser_pull.add_argument ('-t', '--list-labels', action='store_true', default = False,
         help = 'list all remote labels (pull)')
 
-    parser_pull.add_argument ('--limit', type = int, default = None,
-        help = 'Maximum number of messages to pull (soft limit, GMail may return more), note that this may upset the tally of synchronized messages.')
-
-
     parser_pull.add_argument ('-d', '--dry-run', action='store_true',
         default = False, help = 'do not make any changes')
 
@@ -76,9 +72,6 @@ def main (self):
         description = 'push',
         help = 'push local tag-changes')
 
-    parser_push.add_argument ('--limit', type = int, default = None,
-        help = 'Maximum number of messages to push, note that this may upset the tally of synchronized messages.')
-
     parser_push.add_argument ('-d', '--dry-run', action='store_true',
         default = False, help = 'do not make any changes')
 
@@ -119,9 +112,6 @@ def main (self):
         description = 'sync',
         help = 'sync changes (flags have same meaning as for push and pull)')
 
-    parser_sync.add_argument ('--limit', type = int, default = None,
-        help = 'Maximum number of messages to sync, note that this may upset the tally of synchronized messages.')
-
     parser_sync.add_argument ('-d', '--dry-run', action='store_true',
         default = False, help = 'do not make any changes')
 
@@ -549,6 +539,17 @@ def remove_from_list (lst, m):
 
       changed = True
 
+    #limiting the number of messages in the database to local.config.limit parameter
+    with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
+        query = notmuch.Query(db,'')
+        query.set_sort(notmuch.Query.SORT.NEWEST_FIRST)
+        msglist = list(query.search_messages())
+        if len(msglist) > self.local.config.limit:
+            delete_list = self.local.nm_messages_to_gids(msglist[self.local.config.limit:])
+            for m in delete_list:
+                self.local.remove (m,db)
+            changed = True
+
     if len (labels_changed) > 0:
       lchanged = 0
       with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
diff --git a/lieer/local.py b/lieer/local.py
index 9bfacf0..eac8ddd 100644
--- a/lieer/local.py
+++ b/lieer/local.py
@@ -419,6 +419,24 @@ def messages_to_gids (self, msgs):
 
     return (messages, gids)
 
+  def nm_messages_to_gids (self, msgs):
+    """
+    Gets GIDs from a list of NotmuchMessages, the returned list of tuples may contain
+    the same NotmuchMessage several times for each matching file. Files outside the
+    repository are filtered out.
+    """
+    gids     = []
+
+    for m in msgs:
+      for fname in m.get_filenames ():
+        if self.contains (fname):
+          # get gmail id
+          gid = self.__filename_to_gid__ (os.path.basename (fname))
+          if gid:
+            gids.append (gid)
+
+    return gids
+
   def __filename_to_gid__ (self, fname):
     ext = ''
     if self.config.file_extension:

From cef4686b9db18620d2175fad2add386daba70ec3 Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Thu, 16 Jul 2020 13:37:08 +0530
Subject: [PATCH 5/8] Updated function partial pull

---
 lieer/gmailieer.py | 23 +++++++++++++++++++----
 lieer/local.py     |  7 +++++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 270824d..7767e73 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -184,7 +184,7 @@ def main (self):
     parser_set.add_argument ('--no-remove-local-messages', action = 'store_true', default = False,
         help = 'Do not remove messages that have been deleted on the remote')
     parser_set.add_argument ('--limit', type = int, default = None,
-        help = 'Maximum number of messages to pull (soft limit, GMail may return more), note that this may upset the tally of synchronized messages.')
+        help = 'Maximum number of messages to sync with the local database. To unset any limit, use --limit 0.')
     parser_set.set_defaults (func = self.set)
 
 
@@ -543,11 +543,26 @@ def remove_from_list (lst, m):
     with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
         query = notmuch.Query(db,'')
         query.set_sort(notmuch.Query.SORT.NEWEST_FIRST)
-        msglist = list(query.search_messages())
+        thdlist = list(query.search_threads())
+        msglist = []
+        for t in thdlist:
+            msglist += list(t.get_messages())
+        l = len(msglist)
+        i = 0
+        while l > self.local.config.limit: #number of messages to keep, avoiding incomplete threads
+            l -= thdlist[-1-i].get_total_messages()
+            i += 1
         if len(msglist) > self.local.config.limit:
-            delete_list = self.local.nm_messages_to_gids(msglist[self.local.config.limit:])
+            self.bar_create (total = len(msglist)-l, leave = True, desc = 'Removing older messages (0)')
+            delete_list = self.local.nm_messages_to_gids(msglist[l-len(msglist):])
+            deleted = 0
             for m in delete_list:
                 self.local.remove (m,db)
+                deleted += 1
+                if not self.args.quiet and self.bar:
+                    self.bar.set_description ('Removing older messages (%d)' % deleted)
+                self.bar_update (1)
+            self.bar_close ()
             changed = True
 
     if len (labels_changed) > 0:
@@ -587,7 +602,7 @@ def full_pull (self):
     # simple metadata like message ids.
     message_gids = []
     last_id      = self.remote.get_current_history_id (self.local.state.last_historyId)
-    print(self.local.config.limit)
+    
     for mset in self.remote.all_messages ():
       (total, gids) = mset
 
diff --git a/lieer/local.py b/lieer/local.py
index eac8ddd..eaafe14 100644
--- a/lieer/local.py
+++ b/lieer/local.py
@@ -177,7 +177,10 @@ def set_file_extension (self, t):
         raise
 
     def set_limit (self,l):
-        self.limit = l
+        if l != 0:
+            self.limit = l
+        else:
+            self.limit = None
         self.write()
 
 
@@ -421,7 +424,7 @@ def messages_to_gids (self, msgs):
 
   def nm_messages_to_gids (self, msgs):
     """
-    Gets GIDs from a list of NotmuchMessages, the returned list of tuples may contain
+    Gets GIDs from a list of NotmuchMessages, the returned list may contain
     the same NotmuchMessage several times for each matching file. Files outside the
     repository are filtered out.
     """

From 786b84af5218f3cab6d2657c9962798e95f5253e Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Fri, 17 Jul 2020 01:51:49 +0530
Subject: [PATCH 6/8] partial pull, removing the extra function I made

---
 lieer/gmailieer.py | 62 +++++++++++++++++++++++++++-------------------
 lieer/local.py     | 17 -------------
 2 files changed, 37 insertions(+), 42 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 7767e73..1961c70 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -539,31 +539,43 @@ def remove_from_list (lst, m):
 
       changed = True
 
-    #limiting the number of messages in the database to local.config.limit parameter
-    with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
-        query = notmuch.Query(db,'')
-        query.set_sort(notmuch.Query.SORT.NEWEST_FIRST)
-        thdlist = list(query.search_threads())
-        msglist = []
-        for t in thdlist:
-            msglist += list(t.get_messages())
-        l = len(msglist)
-        i = 0
-        while l > self.local.config.limit: #number of messages to keep, avoiding incomplete threads
-            l -= thdlist[-1-i].get_total_messages()
-            i += 1
-        if len(msglist) > self.local.config.limit:
-            self.bar_create (total = len(msglist)-l, leave = True, desc = 'Removing older messages (0)')
-            delete_list = self.local.nm_messages_to_gids(msglist[l-len(msglist):])
-            deleted = 0
-            for m in delete_list:
-                self.local.remove (m,db)
-                deleted += 1
-                if not self.args.quiet and self.bar:
-                    self.bar.set_description ('Removing older messages (%d)' % deleted)
-                self.bar_update (1)
-            self.bar_close ()
-            changed = True
+    #limiting the number of messages in the database to local.config.limit parameter if it is set
+    if self.local.config.limit is not None:
+        with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
+            query = notmuch.Query(db,'')
+            query.set_sort(notmuch.Query.SORT.NEWEST_FIRST)
+            thdlist = list(query.search_threads())
+            msglist = []
+            for t in thdlist:
+                msglist += list(t.get_messages())
+            n_keep = len(msglist) #initiating the parameter
+            i = 0
+            while n_keep > self.local.config.limit: #number of messages to keep, avoiding incomplete threads
+                n_keep -= thdlist[-1-i].get_total_messages()
+                i += 1
+
+            if len(msglist) > self.local.config.limit:
+                self.bar_create (total = len(msglist)-n_keep, leave = True, desc = 'Removing older messages (0)')
+                delete_msgs = msglist[n_keep-len(msglist):] # list of older messages to be deleted
+                
+                delete_gids=[] # getting the gids for messages to be deleted
+                for m in delete_msgs:
+                  for fname in m.get_filenames ():
+                    if self.contains (fname):
+                      # get gmail id
+                      gid = self.__filename_to_gid__ (os.path.basename (fname))
+                      if gid:
+                        delete_gids.append (gid)
+
+                deleted = 0
+                for m in delete_gids:
+                    self.local.remove (m,db)
+                    deleted += 1
+                    if not self.args.quiet and self.bar:
+                        self.bar.set_description ('Removing older messages (%d)' % deleted)
+                    self.bar_update (1)
+                self.bar_close ()
+                changed = True
 
     if len (labels_changed) > 0:
       lchanged = 0
diff --git a/lieer/local.py b/lieer/local.py
index eaafe14..792ccbe 100644
--- a/lieer/local.py
+++ b/lieer/local.py
@@ -422,23 +422,6 @@ def messages_to_gids (self, msgs):
 
     return (messages, gids)
 
-  def nm_messages_to_gids (self, msgs):
-    """
-    Gets GIDs from a list of NotmuchMessages, the returned list may contain
-    the same NotmuchMessage several times for each matching file. Files outside the
-    repository are filtered out.
-    """
-    gids     = []
-
-    for m in msgs:
-      for fname in m.get_filenames ():
-        if self.contains (fname):
-          # get gmail id
-          gid = self.__filename_to_gid__ (os.path.basename (fname))
-          if gid:
-            gids.append (gid)
-
-    return gids
 
   def __filename_to_gid__ (self, fname):
     ext = ''

From 4adfd6cb5c3f3f9e6f4293d7610abf0a9401cad7 Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Fri, 17 Jul 2020 13:33:31 +0530
Subject: [PATCH 7/8] Final commit for --limit.

---
 lieer/gmailieer.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index 1961c70..a3c73fe 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -561,9 +561,9 @@ def remove_from_list (lst, m):
                 delete_gids=[] # getting the gids for messages to be deleted
                 for m in delete_msgs:
                   for fname in m.get_filenames ():
-                    if self.contains (fname):
+                    if self.local.contains (fname):
                       # get gmail id
-                      gid = self.__filename_to_gid__ (os.path.basename (fname))
+                      gid = self.local.__filename_to_gid__ (os.path.basename (fname))
                       if gid:
                         delete_gids.append (gid)
 
@@ -606,6 +606,9 @@ def remove_from_list (lst, m):
   def full_pull (self):
     total = 1
 
+    if self.local.config.limit is not None:
+    	print("Limit parameter set, number of messages that will be fetched:",self.local.config.limit)
+    	
     self.bar_create (leave = True, total = total, desc = 'fetching messages')
 
     # NOTE:
@@ -829,7 +832,7 @@ def set (self, args):
     print ("historyId .........: %d" % self.local.state.last_historyId)
     print ("lastmod ...........: %d" % self.local.state.lastmod)
     print ("Timeout ...........: %f" % self.local.config.timeout)
-    print ("Limit .............: %d" % self.local.config.limit)
+    print ("Limit .............:",self.local.config.limit)
     print ("File extension ....: %s" % self.local.config.file_extension)
     print ("Remove local messages .....:", self.local.config.remove_local_messages)
     print ("Drop non existing labels...:", self.local.config.drop_non_existing_label)

From d3ecbd065e9c5c20adab86a5cad39804d3c6ef64 Mon Sep 17 00:00:00 2001
From: Lattitude75 <c.lattitude75@gmail.com>
Date: Sat, 18 Jul 2020 00:42:16 +0530
Subject: [PATCH 8/8] --unset-limit and added option to readme

---
 docs/index.md      | 4 ++++
 lieer/gmailieer.py | 7 +++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index b2759b5..ec49804 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -159,6 +159,10 @@ Lieer can be configured using `gmi set`. Use without any options to get a list o
 
 **`lastmod`** is the latest synced Notmuch database revision. Anything changed after this revision will be pushed on [`gmi push`](#ush).
 
+**`limit`** is the parameter which limits the number of messages to be saved in the local database. It affects all the pull and push commands and removes older messages to keep the number of messages less than the limit specified. Further, the older messages are removed in a way as to not leave any incomplete threads. This limit can be removed using `gmi set --unset-limit`. 
+
+*Important:* If the limit parameter is changed to a value which is greater than one set previously, the next pull will not be able to fetch extra mails to fill upto the limit. So, a forced pull `gmi pull -f` needs to be performed. This situation would not arise if the new limit parameter is less than the older limit, as the partial pull will limit the number of messages by itself.
+
 **`Timeout`** is the timeout in seconds used for the HTTP connection to GMail. `0` means the forever or system error/timeout, [whichever occurs first](https://github.com/gauteh/lieer/issues/83#issuecomment-396487919).
 
 **`File extension`** is an optional argument to include the specified extension in local file names (e.g., `mbox`) which can be useful for indexing them with third-party programs.  
diff --git a/lieer/gmailieer.py b/lieer/gmailieer.py
index a3c73fe..205efca 100644
--- a/lieer/gmailieer.py
+++ b/lieer/gmailieer.py
@@ -184,7 +184,8 @@ def main (self):
     parser_set.add_argument ('--no-remove-local-messages', action = 'store_true', default = False,
         help = 'Do not remove messages that have been deleted on the remote')
     parser_set.add_argument ('--limit', type = int, default = None,
-        help = 'Maximum number of messages to sync with the local database. To unset any limit, use --limit 0.')
+        help = 'Maximum number of messages to sync with the local database')
+    parser_set.add_argument ('--unset-limit',action = 'store_true',default = False, help = 'Do not limit the number of messages to be synced in the local database')
     parser_set.set_defaults (func = self.set)
 
 
@@ -621,7 +622,6 @@ def full_pull (self):
     for mset in self.remote.all_messages ():
       (total, gids) = mset
 
-      self.bar.total = total
       self.bar_update (len(gids))
 
       for m in gids:
@@ -815,6 +815,9 @@ def set (self, args):
     if args.limit is not None:
    	  self.local.config.set_limit (args.limit)
 
+    if args.unset_limit:
+        self.local.config.set_limit (0)
+
     if args.no_remove_local_messages:
       self.local.config.set_remove_local_messages (False)