Port to Python 3. Requires >=3.3, 2.x still supported.

test_urlnorm.py

@@ -1,6 +1,7 @@
 """
 this is a py.test test file
 """
+from __future__ import print_function
 import urlnorm
 from urlnorm import _unicode
 
@@ -20,28 +21,28 @@ def pytest_generate_tests(metafunc):
             'http://USER:pass@www.Example.COM/foo/bar': 'http://USER:pass@www.example.com/foo/bar',
             'http://www.example.com./':      'http://www.example.com/',
             'http://test.example/?a=%26&b=1': 'http://test.example/?a=%26&b=1', # should not un-encode the & that is part of a parameter value
-            'http://test.example/?a=%e3%82%82%26': 'http://test.example/?a=\xe3\x82\x82%26'.decode('utf8'), # should return a unicode character
+            'http://test.example/?a=%e3%82%82%26': b'http://test.example/?a=\xe3\x82\x82%26'.decode('utf8'), # should return a unicode character
             # note: this breaks the internet for parameters that are positional (stupid nextel) and/or don't have an = sign
             # 'http://test.example/?a=1&b=2&a=3': 'http://test.example/?a=1&a=3&b=2', # should be in sorted/grouped order
-            
+
             # 'http://s.xn--q-bga.de/':       'http://s.q\xc3\xa9.de/'.decode('utf8'), # should be in idna format
             'http://test.example/?':        'http://test.example/', # no trailing ?
             'http://test.example?':       'http://test.example/', # with trailing /
             'http://a.COM/path/?b&a' : 'http://a.com/path/?b&a',
             # test utf8 and unicode
-            u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
-            u'http://XBLA\u306eXbox.com'.encode('utf8'): 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
-            u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
+            u'http://XBLA\u306eXbox.com': b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
+            u'http://XBLA\u306eXbox.com'.encode('utf8'): b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
+            u'http://XBLA\u306eXbox.com': b'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'),
             # test idna + utf8 domain
             # u'http://xn--q-bga.XBLA\u306eXbox.com'.encode('utf8'): 'http://q\xc3\xa9.xbla\xe3\x81\xaexbox.com'.decode('utf8'),
-            'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': 'http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3'.decode('utf8'),
+            'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': b'http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3'.decode('utf8'),
             'http://test.example/\xe3\x82\xad': 'http://test.example/\xe3\x82\xad',
-            
+
             # check that %23 (#) is not escaped where it shouldn't be
             'http://test.example/?p=%23val#test-%23-val%25': 'http://test.example/?p=%23val#test-%23-val%25',
             # check that %20 or %25 is not unescaped to ' ' or %
             'http://test.example/%25/?p=%20val%20%25' : 'http://test.example/%25/?p=%20val%20%25',
-            "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : "http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n",
+            "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : b"http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n".decode('utf8'),
             # check that spaces are collated to '+'
             "http://test.example/path/with a%20space+/" : "http://test.example/path/with%20a%20space+/",
             "http://[2001:db8:1f70::999:de8:7648:6e8]/test" : "http://[2001:db8:1f70::999:de8:7648:6e8]/test", #ipv6 address
@@ -55,15 +56,15 @@ def pytest_generate_tests(metafunc):
         }
         for bad, good in tests.items():
             metafunc.addcall(funcargs=dict(bad=bad, good=good))
-    
+
     elif metafunc.function == test_unquote:
         for bad, good, unsafe in (
-            ('%20', ' ', ''),
-            ('%3f', '%3F', '?'), # don't unquote it, but uppercase it
-            ('%E3%82%AD', u'\u30ad', ''),
+            ('%20', ' ', b''),
+            ('%3f', '%3F', b'?'), # don't unquote it, but uppercase it
+            ('%E3%82%AD', u'\u30ad', b''),
             ):
             metafunc.addcall(funcargs=dict(bad=bad, good=good, unsafe=unsafe))
-    
+
     elif metafunc.function in [test_invalid_urls]:
         for url in [
             'http://http://www.exemple.com/', # invalid domain
@@ -104,7 +105,7 @@ def pytest_generate_tests(metafunc):
 def test_invalid_urls(url):
     try:
         output = urlnorm.norm(url)
-        print '%r' % output
+        print('%r' % output)
     except urlnorm.InvalidUrl:
         return
     assert 1 == 0, "this should have raised an InvalidUrl exception"

urlnorm.py

@@ -13,7 +13,7 @@
   * unescaping any percent escape sequences (where possible)
   * upercase percent escape (ie: %3f => %3F)
   * converts spaces to %20
-  * converts ip encoded as an integer to dotted quad notation 
+  * converts ip encoded as an integer to dotted quad notation
 
 Available functions:
   norm - given a URL (string), returns a normalized URL
@@ -36,7 +36,7 @@
 0.92 - unknown schemes now pass the port through silently
 0.91 - general cleanup
      - changed dictionaries to lists where appropriate
-     - more fine-grained authority parsing and normalisation    
+     - more fine-grained authority parsing and normalisation
 """
 
 __license__ = """
@@ -65,10 +65,28 @@
 # also update in setup.py
 __version__ = "1.1.2"
 
-from urlparse import urlparse, urlunparse
-from string import lower
+import sys
+try:
+    from urllib.parse import urlparse, urlunparse
+except ImportError:
+    from urlparse import urlparse, urlunparse
 import re
 
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    text_type = str
+    binary_type = bytes
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    text_type = unicode
+    binary_type = str
+
+
 class InvalidUrl(Exception):
     pass
 
@@ -102,12 +120,22 @@ class InvalidUrl(Exception):
                         ''
                     ]
 
-params_unsafe_list = ' ?=+%#;'
-qs_unsafe_list = ' ?&=+%#'
-fragment_unsafe_list = ' +%#'
-path_unsafe_list = ' /?;%+#'
-_hextochr = dict(('%02x' % i, chr(i)) for i in range(256))
-_hextochr.update(('%02X' % i, chr(i)) for i in range(256))
+
+params_unsafe_list = 'b ?=+%#;'
+qs_unsafe_list = b' ?&=+%#'
+fragment_unsafe_list = b' +%#'
+path_unsafe_list = b' /?;%+#'
+
+if PY3:
+    # This is how urllib.parse in 3.3 does it
+    _hexdig = '0123456789ABCDEFabcdef'
+    _hextobyte = dict([((a + b).encode(), bytes([int(a + b, 16)])) for a in _hexdig for b in _hexdig])
+    _hextobyte.update(dict([((a + b).upper().encode(), bytes([int(a + b, 16)])) for a in _hexdig for b in _hexdig]))
+else:
+    # I'm leaving the old code in for 2.x, because bytes() is not available in 2.5
+    # and works differently in 2.6/2.7
+    _hextobyte = dict((b'%02x' % i, chr(i)) for i in range(256))
+    _hextobyte.update((b'%02X' % i, chr(i)) for i in range(256))
 
 def unquote_path(s):
     return unquote_safe(s, path_unsafe_list)
@@ -125,22 +153,22 @@ def unquote_safe(s, unsafe_list):
     """unquote percent escaped string except for percent escape sequences that are in unsafe_list"""
     # note: this build utf8 raw strings ,then does a .decode('utf8') at the end.
     # as a result it's doing .encode('utf8') on each block of the string as it's processed.
-    res = _utf8(s).split('%')
-    for i in xrange(1, len(res)):
+    res = _utf8(s).split(b'%')
+    for i in range(1, len(res)):
         item = res[i]
         try:
-            raw_chr = _hextochr[item[:2]]
+            raw_chr = _hextobyte[item[:2]]
             if raw_chr in unsafe_list or ord(raw_chr) < 20:
                 # leave it unescaped (but uppercase the percent escape)
-                res[i] = '%' + item[:2].upper() + item[2:]
+                res[i] = b'%' + item[:2].upper() + item[2:]
             else:
                 res[i] = raw_chr + item[2:]
         except KeyError:
-            res[i] = '%' + item
+            res[i] = b'%' + item
         except UnicodeDecodeError:
             # note: i'm not sure what this does
             res[i] = unichr(int(item[:2], 16)) + item[2:]
-    o = "".join(res)
+    o = b"".join(res)
     return _unicode(o)
 
 def norm(url):
@@ -152,7 +180,7 @@ def norm(url):
 
 def norm_tuple(scheme, authority, path, parameters, query, fragment):
     """given individual url components, return its normalized form"""
-    scheme = lower(scheme)
+    scheme = scheme.lower()
     if not scheme:
         raise InvalidUrl('missing URL scheme')
     authority = norm_netloc(scheme, authority)
@@ -161,7 +189,7 @@ def norm_tuple(scheme, authority, path, parameters, query, fragment):
     path = norm_path(scheme, path)
     # TODO: put query in sorted order; or at least group parameters together
     # Note that some websites use positional parameters or the name part of a query so this would break the internet
-    # query = urlencode(parse_qs(query, keep_blank_values=1), doseq=1) 
+    # query = urlencode(parse_qs(query, keep_blank_values=1), doseq=1)
     parameters = unquote_params(parameters)
     query = unquote_qs(query)
     fragment = unquote_fragment(fragment)
@@ -180,7 +208,7 @@ def norm_path(scheme, path):
         return '/'
     return path
 
-MAX_IP=0xffffffffL
+MAX_IP=0xffffffff
 def int2ip(ipnum):
     assert isinstance(ipnum, int)
     if MAX_IP < ipnum or ipnum < 0:
@@ -190,14 +218,14 @@ def int2ip(ipnum):
     ip3 = ipnum >> 8 & 0xFF
     ip4 = ipnum & 0xFF
     return "%d.%d.%d.%d" % (ip1, ip2, ip3, ip4)
-    
+
 def norm_netloc(scheme, netloc):
     if not netloc:
         return netloc
     match = _server_authority.match(netloc)
     if not match:
         raise InvalidUrl('no host in netloc %r' % netloc)
-    
+
     userinfo, host, port = match.groups()
     # catch a few common errors:
     if host.isdigit():
@@ -207,16 +235,16 @@ def norm_netloc(scheme, netloc):
             raise InvalidUrl('host %r does not escape to a valid ip' % host)
     if host[-1] == '.':
         host = host[:-1]
-    
+
     # bracket check is for ipv6 hosts
     if '.' not in host and not (host[0] == '[' and host[-1] == ']'):
         raise InvalidUrl('host %r is not valid' % host)
-    
-    authority = lower(host)
+
+    authority = host.lower()
     if 'xn--' in authority:
         subdomains = [_idn(subdomain) for subdomain in authority.split('.')]
         authority = '.'.join(subdomains)
-        
+
     if userinfo:
         authority = "%s@%s" % (userinfo, authority)
     if port and port != _default_port.get(scheme, None):
@@ -234,14 +262,14 @@ def _idn(subdomain):
 
 
 def _utf8(value):
-    if isinstance(value, unicode):
+    if isinstance(value, text_type):
         return value.encode("utf-8")
-    assert isinstance(value, str)
+    assert isinstance(value, binary_type)
     return value
 
 
 def _unicode(value):
-    if isinstance(value, str):
+    if isinstance(value, binary_type):
         return value.decode("utf-8")
-    assert isinstance(value, unicode)
+    assert isinstance(value, text_type)
     return value