rate limit prevention, proper proxy support & docs

GetOldTweets3/manager/TweetManager.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-import json, re, datetime, sys, random, http.cookiejar
+import json, re, datetime, sys, random, http.cookiejar, time, collections, string
 import urllib.request, urllib.parse, urllib.error
 from pyquery import PyQuery
 from .. import models
@@ -22,7 +22,7 @@ def __init__(self):
     ]
 
     @staticmethod
-    def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None, debug=False):
+    def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None, debug=False, rateLimitStrategy=None):
         """Get tweets that match the tweetCriteria parameter
         A static method.
 
@@ -62,7 +62,7 @@ def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None, d
 
             active = True
             while active:
-                json = TweetManager.getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, user_agent, debug=debug)
+                json = TweetManager.getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, user_agent, rateLimitStrategy, debug=debug)
                 if len(json['items_html'].strip()) == 0:
                     break
 
@@ -271,7 +271,7 @@ def parse_attributes(markup):
         return attr
 
     @staticmethod
-    def getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, useragent=None, debug=False):
+    def getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, useragent=None, rateLimitStrategy=None, debug=False):
         """Invoke an HTTP query to Twitter.
         Should not be used as an API function. A static method.
         """
@@ -329,23 +329,36 @@ def getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, useragent=No
             ('Connection', "keep-alive")
         ]
 
-        if proxy:
-            opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxy, 'https': proxy}), urllib.request.HTTPCookieProcessor(cookieJar))
-        else:
-            opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))
-        opener.addheaders = headers
-
-        if debug:
-            print(url)
-            print('\n'.join(h[0]+': '+h[1] for h in headers))
-
-        try:
-            response = opener.open(url)
-            jsonResponse = response.read()
-        except Exception as e:
-            print("An error occured during an HTTP request:", str(e))
-            print("Try to open in browser: https://twitter.com/search?q=%s&src=typd" % urllib.parse.quote(urlGetData))
-            sys.exit()
+        # Generate a unique request ID (passed to the rateLimitStrategy)
+        uniqueRequestId = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
+        retry = True
+        while retry:
+            proxyString = ""
+            if proxy:
+                proxyString = proxy
+                if isinstance(proxy, collections.Callable):
+                    proxyString = proxy()
+
+                opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxyString, 'https': proxyString}), urllib.request.HTTPCookieProcessor(cookieJar))
+            else:
+                opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))
+            opener.addheaders = headers
+
+            if debug:
+                print(url)
+                print('\n'.join(h[0]+': '+h[1] for h in headers))
+
+            retry = False
+            try:
+                response = opener.open(url)
+                jsonResponse = response.read()
+            except Exception as e:
+                print("An error occured during an HTTP request:", str(e))
+                print("Try to open in browser: https://twitter.com/search?q=%s&src=typd" % urllib.parse.quote(urlGetData))
+                if isinstance(rateLimitStrategy, collections.Callable):
+                    retry = rateLimitStrategy(request=uniqueRequestId, error=e, proxy=proxyString)
+                if retry is False:
+                    sys.exit()
 
         try:
             s_json = jsonResponse.decode()

README.md

@@ -128,3 +128,66 @@ tweetCriteria = got.manager.TweetCriteria().setUsername("barackobama")\
 tweet = got.manager.TweetManager.getTweets(tweetCriteria)[0]
 print(tweet.text)
 ```
+
+**Stream Tweets in batches of 100:**
+``` python
+tweetCriteria = got.manager.TweetCriteria().setQuerySearch('stream processing')\
+                                            .setMaxTweets(1000)
+def streamTweets(tweets):
+    # tweets is an array of size <= bufferLength
+    print(tweets[0].text)
+
+got.manager.TweetManager.getTweets(tweetCriteria, receiveBuffer=streamTweets, bufferLength=100)
+```
+
+**Use a proxy to execute the request:**
+``` python
+tweetCriteria = got.manager.TweetCriteria().setUsername("barackobama")\
+                                           .setTopTweets(True)\
+                                           .setMaxTweets(10)
+tweet = got.manager.TweetManager.getTweets(tweetCriteria, proxy="ip:port")[0]
+print(tweet.text)
+```
+
+**Dynamically get proxy to execute the request:**
+``` python
+tweetCriteria = got.manager.TweetCriteria().setUsername("barackobama")\
+                                           .setTopTweets(True)\
+                                           .setMaxTweets(10)
+def getRandomProxy():
+    return random.choice(["ip:port", "ip2:port"])
+tweet = got.manager.TweetManager.getTweets(tweetCriteria, proxy=getRandomProxy)[0]
+print(tweet.text)
+```
+
+**Custom ratelimit prevention strategy:**
+``` python
+import collections, urllib.error
+tweetCriteria = got.manager.TweetCriteria().setUsername("barackobama")\
+                                           .setTopTweets(True)\
+                                           .setMaxTweets(10)
+def sleepBetweenFailedRequests(request, error, proxy):
+    # A unique request ID and the proxy it used are passed
+    # for more advanced rate limiting preventing strategies.
+
+    # Deal with all the potential URLLib errors that may happen
+    # https://docs.python.org/3/library/urllib.error.html
+    if (isinstance(error, HTTPError) and error.status_code in [429, 503]):
+      # Sleep for 60 seconds
+      time.sleep(60)
+      return True
+
+    if (isinstance(error, URLError) and error.errno in [111]):
+      # Sleep for 60 seconds
+      time.sleep(60)
+      return True
+
+    # To stop execution of the scraper
+    # raise Exception("Rate Limiting Strategy received an error it doesn't know how to deal with")
+
+    # To stop just this single request, return False
+    return False
+
+tweet = got.manager.TweetManager.getTweets(tweetCriteria, rateLimitStrategy=sleepBetweenFailedRequests)[0]
+print(tweet.text)
+```