From 5c1e11b02de2afb819f320703c9cf391dbb498e6 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 17 Dec 2013 12:37:54 +0700 Subject: [PATCH] new: use ``requests`` instead of ``urllib2``. This avoids much of hassle with gzip encoding, multipart, authentification, url encoding... which this module should avoid managing because requests does it better and is widely supported and tested. This allows wikitools to concentrate on its main task. --- wikitools/api.py | 156 ++++++---------------------------------------- wikitools/wiki.py | 14 ++--- 2 files changed, 25 insertions(+), 145 deletions(-) diff --git a/wikitools/api.py b/wikitools/api.py index f784eb1..0eb6e36 100644 --- a/wikitools/api.py +++ b/wikitools/api.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Copyright 2008-2013 Alex Zaddach (mrzmanwiki@gmail.com) # This file is part of wikitools. @@ -17,27 +17,20 @@ # This module is documented at http://code.google.com/p/python-wikitools/wiki/api -import urllib2 +import requests +from requests.auth import HTTPDigestAuth + import re import time import sys import wiki +import StringIO from urllib import quote_plus, _is_unicode -try: - from poster.encode import multipart_encode - canupload = True -except: - canupload = False try: import json except: import simplejson as json -try: - import gzip - import StringIO -except: - gzip = False class APIError(Exception): """Base class for errors""" @@ -47,19 +40,17 @@ class APIDisabled(APIError): class APIRequest: """A request to the site's API""" - def __init__(self, wiki, data, write=False, multipart=False): + def __init__(self, wiki, data, write=False, multipart=True): """ wiki - A Wiki object data - API parameters in the form of a dict write - set to True if doing a write query, so it won't try again on error - multipart - use multipart data transfer, required for file uploads, - requires the poster package - + multipart - obsolete and unused option, you can use file objet to send + multipart requests. + maxlag is set by default to 5 but can be changed - format is always set to json + format is always overriden to force 'json' """ - if not canupload and multipart: - raise APIError("The poster module is required for multipart support") self.sleep = 5 self.data = data.copy() self.data['format'] = "json" @@ -68,72 +59,25 @@ def __init__(self, wiki, data, write=False, multipart=False): self.data['assert'] = wiki.assertval if not 'maxlag' in self.data and not wiki.maxlag < 0: self.data['maxlag'] = wiki.maxlag - self.multipart = multipart - if self.multipart: - (datagen, self.headers) = multipart_encode(self.data) - self.encodeddata = '' - for singledata in datagen: - self.encodeddata = self.encodeddata + singledata - else: - self.encodeddata = urlencode(self.data, 1) - self.headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Content-Length": str(len(self.encodeddata)) - } + self.headers = {} self.headers["User-agent"] = wiki.useragent - if gzip: - self.headers['Accept-Encoding'] = 'gzip' + self.headers['Accept-Encoding'] = 'gzip' self.wiki = wiki self.response = False - if wiki.passman is not None: - self.opener = urllib2.build_opener(urllib2.HTTPDigestAuthHandler(wiki.passman), urllib2.HTTPCookieProcessor(wiki.cookies)) - else: - self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(wiki.cookies)) - self.request = urllib2.Request(self.wiki.apibase, self.encodeddata, self.headers) + self.authman = None if wiki.auth is None else HTTPDigest(wiki.auth) - def setMultipart(self, multipart=True): - """Enable multipart data transfer, required for file uploads.""" - if not canupload and multipart: - raise APIError("The poster package is required for multipart support") - self.multipart = multipart - if multipart: - (datagen, headers) = multipart_encode(self.data) - self.headers.pop('Content-Length') - self.headers.pop('Content-Type') - self.headers.update(headers) - self.encodeddata = '' - for singledata in datagen: - self.encodeddata = self.encodeddata + singledata - else: - self.encodeddata = urlencode(self.data, 1) - self.headers['Content-Length'] = str(len(self.encodeddata)) - self.headers['Content-Type'] = "application/x-www-form-urlencoded" - def changeParam(self, param, value): """Change or add a parameter after making the request object Simply changing self.data won't work as it needs to update other things. value can either be a normal string value, or a file-like object, - which will be uploaded, if setMultipart was called previously. + which will be uploaded. """ if param == 'format': raise APIError('You can not change the result format') self.data[param] = value - if self.multipart: - (datagen, headers) = multipart_encode(self.data) - self.headers.pop('Content-Length') - self.headers.pop('Content-Type') - self.headers.update(headers) - self.encodeddata = '' - for singledata in datagen: - self.encodeddata = self.encodeddata + singledata - else: - self.encodeddata = urlencode(self.data, 1) - self.headers['Content-Length'] = str(len(self.encodeddata)) - self.headers['Content-Type'] = "application/x-www-form-urlencoded" - self.request = urllib2.Request(self.wiki.apibase, self.encodeddata, self.headers) def query(self, querycontinue=True): """Actually do the query here and return usable stuff @@ -144,7 +88,7 @@ def query(self, querycontinue=True): """ data = False while not data: - rawdata = self.__getRaw() + rawdata = StringIO.StringIO(self.__getRaw().content) data = self.__parseJSON(rawdata) if not data and type(data) is APIListResult: break @@ -222,12 +166,8 @@ def __getRaw(self): catcherror = None else: catcherror = Exception - data = self.opener.open(self.request) - self.response = data.info() - if gzip: - encoding = self.response.get('Content-encoding') - if encoding in ('gzip', 'x-gzip'): - data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data.read())) + data = self.response = requests.get(self.wiki.apibase, params=self.data, + headers=self.headers, auth=self.authman) except catcherror, exc: errname = sys.exc_info()[0].__name__ errinfo = exc @@ -245,10 +185,10 @@ def __parseJSON(self, data): content = None if isinstance(parsed, dict): content = APIResult(parsed) - content.response = self.response.items() + content.response = self.response.headers.items() elif isinstance(parsed, list): content = APIListResult(parsed) - content.response = self.response.items() + content.response = self.response.headers.items() else: content = parsed if 'error' in content: @@ -303,61 +243,3 @@ def resultCombine(type, old, new): ret['query']['pages'][key][type] = [dict(entry) for entry in retset] return ret -def urlencode(query,doseq=0): - """ - Hack of urllib's urlencode function, which can handle - utf-8, but for unknown reasons, chooses not to by - trying to encode everything as ascii - """ - if hasattr(query,"items"): - # mapping objects - query = query.items() - else: - # it's a bother at times that strings and string-like objects are - # sequences... - try: - # non-sequence items should not work with len() - # non-empty strings will fail this - if len(query) and not isinstance(query[0], tuple): - raise TypeError - # zero-length sequences of all types will get here and succeed, - # but that's a minor nit - since the original implementation - # allowed empty dicts that type of behavior probably should be - # preserved for consistency - except TypeError: - ty,va,tb = sys.exc_info() - raise TypeError, "not a valid non-string sequence or mapping object", tb - - l = [] - if not doseq: - # preserve old behavior - for k, v in query: - k = quote_plus(str(k)) - v = quote_plus(str(v)) - l.append(k + '=' + v) - else: - for k, v in query: - k = quote_plus(str(k)) - if isinstance(v, str): - v = quote_plus(v) - l.append(k + '=' + v) - elif _is_unicode(v): - # is there a reasonable way to convert to ASCII? - # encode generates a string, but "replace" or "ignore" - # lose information and "strict" can raise UnicodeError - v = quote_plus(v.encode("utf8","replace")) - l.append(k + '=' + v) - else: - try: - # is this a sufficient test for sequence-ness? - x = len(v) - except TypeError: - # not a sequence - v = quote_plus(str(v)) - l.append(k + '=' + v) - else: - # loop over the sequence - for elt in v: - l.append(k + '=' + quote_plus(str(elt))) - return '&'.join(l) - diff --git a/wikitools/wiki.py b/wikitools/wiki.py index 0b7cf6e..60f0fa9 100644 --- a/wikitools/wiki.py +++ b/wikitools/wiki.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Copyright 2008-2013 Alex Zaddach (mrzmanwiki@gmail.com) # This file is part of wikitools. @@ -17,12 +17,11 @@ import cookielib import api -import urllib import re import time import os from urlparse import urlparse -from urllib2 import HTTPPasswordMgrWithDefaultRealm + try: import cPickle as pickle except: @@ -57,7 +56,7 @@ def __init__(self, url="https://en.wikipedia.org/w/api.php", httpuser=None, http """ url - A URL to the site's API, defaults to en.wikipedia httpuser - optional user name for HTTP Auth - httppass - password for HTTP Auth, leave out to enter interactively + httppass - password for HTTP Auth, leave out to enter interactively """ self.apibase = url @@ -68,11 +67,10 @@ def __init__(self, url="https://en.wikipedia.org/w/api.php", httpuser=None, http if httpuser is not None: if httppass is None: from getpass import getpass - self.httppass = getpass("HTTP Auth password for "+httpuser+": ") - self.passman = HTTPPasswordMgrWithDefaultRealm() - self.passman.add_password(None, self.domain, httpuser, httppass) + httppass = getpass("HTTP Auth password for "+httpuser+": ") + self.auth = (httpuser, httppass) else: - self.passman = None + self.auth = None self.maxlag = 5 self.maxwaittime = 120 self.useragent = "python-wikitools/%s" % VERSION