From 356045a6755e564c0d76f19c5a30e7d1f449bf66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Elsd=C3=B6rfer?= Date: Mon, 7 Oct 2013 17:26:07 +0200 Subject: [PATCH] Port to Python 3. Requires >=3.3, 2.x still supported. --- test_urlnorm.py | 29 +++++++++-------- urlnorm.py | 86 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 72 insertions(+), 43 deletions(-) diff --git a/test_urlnorm.py b/test_urlnorm.py index e1f7916..6f1d825 100644 --- a/test_urlnorm.py +++ b/test_urlnorm.py @@ -1,6 +1,7 @@ """ this is a py.test test file """ +from __future__ import print_function import urlnorm from urlnorm import _unicode @@ -20,28 +21,28 @@ def pytest_generate_tests(metafunc): 'http://USER:pass@www.Example.COM/foo/bar': 'http://USER:pass@www.example.com/foo/bar', 'http://www.example.com./': 'http://www.example.com/', 'http://test.example/?a=%26&b=1': 'http://test.example/?a=%26&b=1', # should not un-encode the & that is part of a parameter value - 'http://test.example/?a=%e3%82%82%26': 'http://test.example/?a=\xe3\x82\x82%26'.decode('utf8'), # should return a unicode character + 'http://test.example/?a=%e3%82%82%26': b'http://test.example/?a=\xe3\x82\x82%26'.decode('utf8'), # should return a unicode character # note: this breaks the internet for parameters that are positional (stupid nextel) and/or don't have an = sign # 'http://test.example/?a=1&b=2&a=3': 'http://test.example/?a=1&a=3&b=2', # should be in sorted/grouped order - + # 'http://s.xn--q-bga.de/': 'http://s.q\xc3\xa9.de/'.decode('utf8'), # should be in idna format 'http://test.example/?': 'http://test.example/', # no trailing ? 'http://test.example?': 'http://test.example/', # with trailing / 'http://a.COM/path/?b&a' : 'http://a.com/path/?b&a', # test utf8 and unicode - u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), - u'http://XBLA\u306eXbox.com'.encode('utf8'): 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), - u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), + u'http://XBLA\u306eXbox.com': b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), + u'http://XBLA\u306eXbox.com'.encode('utf8'): b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), + u'http://XBLA\u306eXbox.com': b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), # test idna + utf8 domain # u'http://xn--q-bga.XBLA\u306eXbox.com'.encode('utf8'): 'http://q\xc3\xa9.xbla\xe3\x81\xaexbox.com'.decode('utf8'), - 'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': 'http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3'.decode('utf8'), + 'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': b'http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3'.decode('utf8'), 'http://test.example/\xe3\x82\xad': 'http://test.example/\xe3\x82\xad', - + # check that %23 (#) is not escaped where it shouldn't be 'http://test.example/?p=%23val#test-%23-val%25': 'http://test.example/?p=%23val#test-%23-val%25', # check that %20 or %25 is not unescaped to ' ' or % 'http://test.example/%25/?p=%20val%20%25' : 'http://test.example/%25/?p=%20val%20%25', - "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : "http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n", + "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : b"http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n".decode('utf8'), # check that spaces are collated to '+' "http://test.example/path/with a%20space+/" : "http://test.example/path/with%20a%20space+/", "http://[2001:db8:1f70::999:de8:7648:6e8]/test" : "http://[2001:db8:1f70::999:de8:7648:6e8]/test", #ipv6 address @@ -55,15 +56,15 @@ def pytest_generate_tests(metafunc): } for bad, good in tests.items(): metafunc.addcall(funcargs=dict(bad=bad, good=good)) - + elif metafunc.function == test_unquote: for bad, good, unsafe in ( - ('%20', ' ', ''), - ('%3f', '%3F', '?'), # don't unquote it, but uppercase it - ('%E3%82%AD', u'\u30ad', ''), + ('%20', ' ', b''), + ('%3f', '%3F', b'?'), # don't unquote it, but uppercase it + ('%E3%82%AD', u'\u30ad', b''), ): metafunc.addcall(funcargs=dict(bad=bad, good=good, unsafe=unsafe)) - + elif metafunc.function in [test_invalid_urls]: for url in [ 'http://http://www.exemple.com/', # invalid domain @@ -104,7 +105,7 @@ def pytest_generate_tests(metafunc): def test_invalid_urls(url): try: output = urlnorm.norm(url) - print '%r' % output + print('%r' % output) except urlnorm.InvalidUrl: return assert 1 == 0, "this should have raised an InvalidUrl exception" diff --git a/urlnorm.py b/urlnorm.py index 481a1d6..e92f363 100644 --- a/urlnorm.py +++ b/urlnorm.py @@ -13,7 +13,7 @@ * unescaping any percent escape sequences (where possible) * upercase percent escape (ie: %3f => %3F) * converts spaces to %20 - * converts ip encoded as an integer to dotted quad notation + * converts ip encoded as an integer to dotted quad notation Available functions: norm - given a URL (string), returns a normalized URL @@ -36,7 +36,7 @@ 0.92 - unknown schemes now pass the port through silently 0.91 - general cleanup - changed dictionaries to lists where appropriate - - more fine-grained authority parsing and normalisation + - more fine-grained authority parsing and normalisation """ __license__ = """ @@ -65,10 +65,28 @@ # also update in setup.py __version__ = "1.1.2" -from urlparse import urlparse, urlunparse -from string import lower +import sys +try: + from urllib.parse import urlparse, urlunparse +except ImportError: + from urlparse import urlparse, urlunparse import re + +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + text_type = str + binary_type = bytes +else: + string_types = basestring, + integer_types = (int, long) + text_type = unicode + binary_type = str + + class InvalidUrl(Exception): pass @@ -102,12 +120,22 @@ class InvalidUrl(Exception): '' ] -params_unsafe_list = ' ?=+%#;' -qs_unsafe_list = ' ?&=+%#' -fragment_unsafe_list = ' +%#' -path_unsafe_list = ' /?;%+#' -_hextochr = dict(('%02x' % i, chr(i)) for i in range(256)) -_hextochr.update(('%02X' % i, chr(i)) for i in range(256)) + +params_unsafe_list = 'b ?=+%#;' +qs_unsafe_list = b' ?&=+%#' +fragment_unsafe_list = b' +%#' +path_unsafe_list = b' /?;%+#' + +if PY3: + # This is how urllib.parse in 3.3 does it + _hexdig = '0123456789ABCDEFabcdef' + _hextobyte = dict([((a + b).encode(), bytes([int(a + b, 16)])) for a in _hexdig for b in _hexdig]) + _hextobyte.update(dict([((a + b).upper().encode(), bytes([int(a + b, 16)])) for a in _hexdig for b in _hexdig])) +else: + # I'm leaving the old code in for 2.x, because bytes() is not available in 2.5 + # and works differently in 2.6/2.7 + _hextobyte = dict((b'%02x' % i, chr(i)) for i in range(256)) + _hextobyte.update((b'%02X' % i, chr(i)) for i in range(256)) def unquote_path(s): return unquote_safe(s, path_unsafe_list) @@ -125,22 +153,22 @@ def unquote_safe(s, unsafe_list): """unquote percent escaped string except for percent escape sequences that are in unsafe_list""" # note: this build utf8 raw strings ,then does a .decode('utf8') at the end. # as a result it's doing .encode('utf8') on each block of the string as it's processed. - res = _utf8(s).split('%') - for i in xrange(1, len(res)): + res = _utf8(s).split(b'%') + for i in range(1, len(res)): item = res[i] try: - raw_chr = _hextochr[item[:2]] + raw_chr = _hextobyte[item[:2]] if raw_chr in unsafe_list or ord(raw_chr) < 20: # leave it unescaped (but uppercase the percent escape) - res[i] = '%' + item[:2].upper() + item[2:] + res[i] = b'%' + item[:2].upper() + item[2:] else: res[i] = raw_chr + item[2:] except KeyError: - res[i] = '%' + item + res[i] = b'%' + item except UnicodeDecodeError: # note: i'm not sure what this does res[i] = unichr(int(item[:2], 16)) + item[2:] - o = "".join(res) + o = b"".join(res) return _unicode(o) def norm(url): @@ -152,7 +180,7 @@ def norm(url): def norm_tuple(scheme, authority, path, parameters, query, fragment): """given individual url components, return its normalized form""" - scheme = lower(scheme) + scheme = scheme.lower() if not scheme: raise InvalidUrl('missing URL scheme') authority = norm_netloc(scheme, authority) @@ -161,7 +189,7 @@ def norm_tuple(scheme, authority, path, parameters, query, fragment): path = norm_path(scheme, path) # TODO: put query in sorted order; or at least group parameters together # Note that some websites use positional parameters or the name part of a query so this would break the internet - # query = urlencode(parse_qs(query, keep_blank_values=1), doseq=1) + # query = urlencode(parse_qs(query, keep_blank_values=1), doseq=1) parameters = unquote_params(parameters) query = unquote_qs(query) fragment = unquote_fragment(fragment) @@ -180,7 +208,7 @@ def norm_path(scheme, path): return '/' return path -MAX_IP=0xffffffffL +MAX_IP=0xffffffff def int2ip(ipnum): assert isinstance(ipnum, int) if MAX_IP < ipnum or ipnum < 0: @@ -190,14 +218,14 @@ def int2ip(ipnum): ip3 = ipnum >> 8 & 0xFF ip4 = ipnum & 0xFF return "%d.%d.%d.%d" % (ip1, ip2, ip3, ip4) - + def norm_netloc(scheme, netloc): if not netloc: return netloc match = _server_authority.match(netloc) if not match: raise InvalidUrl('no host in netloc %r' % netloc) - + userinfo, host, port = match.groups() # catch a few common errors: if host.isdigit(): @@ -207,16 +235,16 @@ def norm_netloc(scheme, netloc): raise InvalidUrl('host %r does not escape to a valid ip' % host) if host[-1] == '.': host = host[:-1] - + # bracket check is for ipv6 hosts if '.' not in host and not (host[0] == '[' and host[-1] == ']'): raise InvalidUrl('host %r is not valid' % host) - - authority = lower(host) + + authority = host.lower() if 'xn--' in authority: subdomains = [_idn(subdomain) for subdomain in authority.split('.')] authority = '.'.join(subdomains) - + if userinfo: authority = "%s@%s" % (userinfo, authority) if port and port != _default_port.get(scheme, None): @@ -234,14 +262,14 @@ def _idn(subdomain): def _utf8(value): - if isinstance(value, unicode): + if isinstance(value, text_type): return value.encode("utf-8") - assert isinstance(value, str) + assert isinstance(value, binary_type) return value def _unicode(value): - if isinstance(value, str): + if isinstance(value, binary_type): return value.decode("utf-8") - assert isinstance(value, unicode) + assert isinstance(value, text_type) return value