From f341eaa82d32be1fa002da966b1a12508c7610f8 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Tue, 3 Jan 2017 09:26:51 -0200 Subject: [PATCH 1/5] Retry and raise exceptions upon API errors. Fixes issue #43. --- wikitools/api.py | 127 ++++++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 56 deletions(-) diff --git a/wikitools/api.py b/wikitools/api.py index 85e2132..943b73a 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -47,7 +47,21 @@ class APIError(Exception): class APIDisabled(APIError): """API not enabled""" - + +class APIMaxlagError(APIError): + """API maxlag reached. + + This typically means you should retry after a while; see + https://www.mediawiki.org/wiki/Manual:Maxlag_parameter + + The retry_after attribute of this exception suggests how long to wait. + """ + + def __init__(self, retry_after): + message = 'Server lag, try again after %d seconds' % retry_after + super(APIMaxlagError, self).__init__(self, message) + self.retry_after = retry_after + class APIRequest: """A request to the site's API""" def __init__(self, wiki, data, write=False, multipart=False): @@ -141,53 +155,54 @@ def changeParam(self, param, value): self.headers['Content-Type'] = "application/x-www-form-urlencoded" self.request = urllib2.Request(self.wiki.apibase, self.encodeddata, self.headers) - def query(self, querycontinue=True): + def query(self, querycontinue=True, retries=1): """Actually do the query here and return usable stuff querycontinue - look for query-continue in the results and continue querying until there is no more data to retrieve (DEPRECATED: use queryGen as a more reliable and efficient alternative) - + + retries - How many retries to allow for in case of errors (set to 0 for no + retries). If the number of attempts is exceeeded, an exception is raised: + either wiki.UserBlocked, or a (subclass of) APIError. """ if querycontinue and self.data['action'] == 'query': warnings.warn("""The querycontinue option is deprecated and will be removed in a future release, use the new queryGen function instead for queries requring multiple requests""", FutureWarning) - data = False - while not data: - rawdata = self.__getRaw() - data = self.__parseJSON(rawdata) - if not data and type(data) is APIListResult: - break - if 'error' in data: - if self.iswrite and data['error']['code'] == 'blocked': - raise wiki.UserBlocked(data['error']['info']) - raise APIError(data['error']['code'], data['error']['info']) + attempts = 1 + retries + for attempt in range(1, attempts + 1): + try: + rawdata = self.__getRaw() + data = self.__parseJSON(rawdata) + if data or isinstance(data, APIListResult): + break + except Exception, e: + print 'Attempt %d (of %d) raised exception %r' % (attempt, attempts, e) + if attempt == attempts: + raise + if isinstance(e, APIMaxlagError): + print 'Server lag, sleeping for %d seconds' % e.retry_after + time.sleep(e.retry_after) if 'query-continue' in data and querycontinue: data = self.__longQuery(data) return data - def queryGen(self): + def queryGen(self, retries_per_request=1): """Unlike the old query-continue method that tried to stitch results together, which could work poorly for complex result sets and could use a lot of memory, this yield each set returned by the API and lets the user process the data. Loosely based on the recommended implementation on mediawiki.org + retries_per_request - How many retries to allow, per API + request. See the retries parameter in query() for the exact + behavior. """ reqcopy = copy.deepcopy(self.request) self.changeParam('continue', '') while True: - data = False - while not data: - rawdata = self.__getRaw() - data = self.__parseJSON(rawdata) - if not data and type(data) is APIListResult: - break - if 'error' in data: - if self.iswrite and data['error']['code'] == 'blocked': - raise wiki.UserBlocked(data['error']['info']) - raise APIError(data['error']['code'], data['error']['info']) + data = self.query(querycontinue=False, retries=retries_per_request) yield data if 'continue' not in data: break @@ -277,39 +292,39 @@ def __getRaw(self): return data def __parseJSON(self, data): - maxlag = True - while maxlag: - try: - maxlag = False - parsed = json.loads(data.read()) - content = None - if isinstance(parsed, dict): - content = APIResult(parsed) - content.response = self.response.items() - elif isinstance(parsed, list): - content = APIListResult(parsed) - content.response = self.response.items() - else: - content = parsed - if 'error' in content: - error = content['error']['code'] - if error == "maxlag": - lagtime = int(re.search("(\d+) seconds", content['error']['info']).group(1)) - if lagtime > self.wiki.maxwaittime: - lagtime = self.wiki.maxwaittime - print("Server lag, sleeping for "+str(lagtime)+" seconds") - maxlag = True - time.sleep(int(lagtime)+0.5) - return False - except: # Something's wrong with the data... - data.seek(0) - if "MediaWiki API is not enabled for this site. Add the following line to your LocalSettings.php
$wgEnableAPI=true;
" in data.read(): - raise APIDisabled("The API is not enabled on this site") - print "Invalid JSON, trying request again" - # FIXME: Would be nice if this didn't just go forever if its never going to work - return False + try: + content = None + data = data.read() + parsed = json.loads(data) + if isinstance(parsed, dict): + content = APIResult(parsed) + content.response = self.response.items() + elif isinstance(parsed, list): + content = APIListResult(parsed) + content.response = self.response.items() + else: + content = parsed + except: # Something's wrong with the data... + self.__exceptionFromAPIError(data, content) + if 'error' in content: + self.__exceptionFromAPIError(data, content) return content - + + def __exceptionFromAPIError(self, data, content): + if "MediaWiki API is not enabled for this site. Add the following line to your LocalSettings.php
$wgEnableAPI=true;
" in data: + raise APIDisabled("The API is not enabled on this site") + if content is None: + raise APIError(data) + error = content['error']['code'] + if error == "maxlag": + lagtime = int(re.search("(\d+) seconds", content['error']['info']).group(1)) + if lagtime > self.wiki.maxwaittime: + lagtime = self.wiki.maxwaittime + raise APIMaxlagError(lagtime) + if self.iswrite and content['error']['code'] == 'blocked': + raise wiki.UserBlocked(content['error']['info']) + raise APIError(content['error']['code'], content['error']['info']) + class APIResult(dict): response = [] From cdfd4391ff10935b4025517623a9b8e825e06611 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Tue, 3 Jan 2017 09:54:46 -0200 Subject: [PATCH 2/5] Handle floating-point lag being returned by the API. According to https://www.mediawiki.org/wiki/Manual:Maxlag_parameter, this may happen for caching layer errors. --- wikitools/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wikitools/api.py b/wikitools/api.py index 943b73a..b2e305d 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -58,7 +58,7 @@ class APIMaxlagError(APIError): """ def __init__(self, retry_after): - message = 'Server lag, try again after %d seconds' % retry_after + message = 'Server lag, try again after %.2f seconds' % retry_after super(APIMaxlagError, self).__init__(self, message) self.retry_after = retry_after @@ -182,7 +182,7 @@ def query(self, querycontinue=True, retries=1): if attempt == attempts: raise if isinstance(e, APIMaxlagError): - print 'Server lag, sleeping for %d seconds' % e.retry_after + print 'Server lag, sleeping for %.2f seconds' % e.retry_after time.sleep(e.retry_after) if 'query-continue' in data and querycontinue: data = self.__longQuery(data) @@ -317,7 +317,8 @@ def __exceptionFromAPIError(self, data, content): raise APIError(data) error = content['error']['code'] if error == "maxlag": - lagtime = int(re.search("(\d+) seconds", content['error']['info']).group(1)) + lagtime = 0.5 + float(re.search("([0-9.-]+) seconds", + content['error']['info']).group(1)) if lagtime > self.wiki.maxwaittime: lagtime = self.wiki.maxwaittime raise APIMaxlagError(lagtime) From a5bc58ca1901dbafcaa7d553370d255cf017e8f8 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Tue, 3 Jan 2017 10:12:59 -0200 Subject: [PATCH 3/5] Handle the Retry-After header, if present in the API response. Also remove some retry logic, since we now retry from query(). --- wikitools/api.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/wikitools/api.py b/wikitools/api.py index b2e305d..598c1d4 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -272,23 +272,14 @@ def __longQuery(self, initialdata): def __getRaw(self): data = False while not data: - try: - if self.sleep >= self.wiki.maxwaittime or self.iswrite: - catcherror = None - else: - catcherror = Exception - data = self.opener.open(self.request) - self.response = data.info() - if gzip: - encoding = self.response.get('Content-encoding') - if encoding in ('gzip', 'x-gzip'): - data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data.read())) - except catcherror, exc: - errname = sys.exc_info()[0].__name__ - errinfo = exc - print("%s: %s trying request again in %d seconds" % (errname, errinfo, self.sleep)) - time.sleep(self.sleep+0.5) - self.sleep+=5 + data = self.opener.open(self.request) + self.response = data.info() + if 'Retry-After' in self.response: + raise APIMaxlagError(float(self.response.get('Retry-After'))) + if gzip: + encoding = self.response.get('Content-encoding') + if encoding in ('gzip', 'x-gzip'): + data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data.read())) return data def __parseJSON(self, data): From 89c4e2712b52f0813defb9b099614765c21dc749 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Tue, 3 Jan 2017 12:49:51 -0200 Subject: [PATCH 4/5] Always bound the sleep time by the Wiki's maxwaittime. --- wikitools/api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wikitools/api.py b/wikitools/api.py index 598c1d4..73c54f4 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -182,8 +182,9 @@ def query(self, querycontinue=True, retries=1): if attempt == attempts: raise if isinstance(e, APIMaxlagError): - print 'Server lag, sleeping for %.2f seconds' % e.retry_after - time.sleep(e.retry_after) + waittime = min(e.retry_after, self.wiki.maxwaittime) + print 'Server lag, sleeping for %.2f seconds' % waittime + time.sleep(waittime) if 'query-continue' in data and querycontinue: data = self.__longQuery(data) return data @@ -310,8 +311,6 @@ def __exceptionFromAPIError(self, data, content): if error == "maxlag": lagtime = 0.5 + float(re.search("([0-9.-]+) seconds", content['error']['info']).group(1)) - if lagtime > self.wiki.maxwaittime: - lagtime = self.wiki.maxwaittime raise APIMaxlagError(lagtime) if self.iswrite and content['error']['code'] == 'blocked': raise wiki.UserBlocked(content['error']['info']) From 98010fd63aad8d088ec84962e107f3183fcb11ce Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Wed, 4 Jan 2017 08:59:33 -0200 Subject: [PATCH 5/5] Fix typo. --- wikitools/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wikitools/api.py b/wikitools/api.py index 73c54f4..17d37f1 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -163,7 +163,7 @@ def query(self, querycontinue=True, retries=1): reliable and efficient alternative) retries - How many retries to allow for in case of errors (set to 0 for no - retries). If the number of attempts is exceeeded, an exception is raised: + retries). If the number of attempts is exceeded, an exception is raised: either wiki.UserBlocked, or a (subclass of) APIError. """ if querycontinue and self.data['action'] == 'query':