From 9225a5132a384212c29dd960ceb61f5d1be47912 Mon Sep 17 00:00:00 2001 From: Thomas Bird Date: Fri, 15 Jan 2021 16:58:17 +0000 Subject: [PATCH 1/4] update brisish post code regex to find more post codes --- pyap/source_GB/data.py | 57 +++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/pyap/source_GB/data.py b/pyap/source_GB/data.py index bbbc9f2..394a88a 100644 --- a/pyap/source_GB/data.py +++ b/pyap/source_GB/data.py @@ -353,32 +353,43 @@ postal_code = r""" (?P - (?: - (?:[gG][iI][rR] {0,}0[aA]{2})| + # Girobank postcode + (?:[gG][iI][rR] {0,}0[aA]{2})| + (?: # British Overseas Territories in usual format (?: - (?: - [aA][sS][cC][nN]| - [sS][tT][hH][lL]| - [tT][dD][cC][uU]| - [bB][bB][nN][dD]| - [bB][iI][qQ][qQ]| - [fF][iI][qQ][qQ]| - [pP][cC][rR][nN]| - [sS][iI][qQ][qQ]| - [iT][kK][cC][aA] - ) - \ {0,}1[zZ]{2} - )| + [aA][sS][cC][nN]| + [sS][tT][hH][lL]| + [tT][dD][cC][uU]| + [bB][bB][nN][dD]| + [bB][iI][qQ][qQ]| + [fF][iI][qQ][qQ]| + [pP][cC][rR][nN]| + [sS][iI][qQ][qQ]| + [iT][kK][cC][aA] + ) + \ {0,}1[zZ]{2} + )| + (?: # British Overseas Territories in zip-code format + (KY[0-9]|MSR|VG|AI)[ -]{0,}[0-9]{4} + )| + # (?: # Bermuda including this causes too many false positives, so excluded for now + # [a-zA-Z]{2}\ {0,}[0-9]{2} + # )| + (?: # British Forces Post Office + [Bb][Ff][Pp][Oo]\ {0,}[0-9]{1,4} + )| + (?: # Mainland British postcodes (?: - (?: - (?:[a-pr-uwyzA-PR-UWYZ][a-hk-yxA-HK-XY]?[0-9][0-9]?)| - (?: - (?:[a-pr-uwyzA-PR-UWYZ][0-9][a-hjkstuwA-HJKSTUW])| - (?:[a-pr-uwyzA-PR-UWYZ][a-hk-yA-HK-Y][0-9][abehmnprv-yABEHMNPRV-Y]) - ) - ) - \ {0,}[0-9][abd-hjlnp-uw-zABD-HJLNP-UW-Z]{2} + (?:[Ww][Cc][0-9][abehmnprvwxyABEHMNPRVWXY])| + (?:[Ee][Cc][1-4][abehmnprvwxyABEHMNPRVWXY])| + (?:[Nn][Ww]1[Ww])| + (?:[Ss][Ee]1[Pp])| + (?:[Ss][Ww]1[abehmnprvwxyABEHMNPRVWXY])| + (?:[EeNnWw]1[a-hjkpstuwA-HJKPSTUW])| + (?:[BbEeGgLlMmNnSsWw][0-9][0-9]?)| + (?:[a-pr-uwyzA-PR-UWYZ][a-hk-yxA-HK-XY][0-9][0-9]?) ) + \ {0,}[0-9][abd-hjlnp-uw-zABD-HJLNP-UW-Z]{2} ) ) # end postal_code """ From 4f9445147c553b6ee0030602d5176a2c7ceaf137 Mon Sep 17 00:00:00 2001 From: Thomas Bird Date: Fri, 15 Jan 2021 17:06:19 +0000 Subject: [PATCH 2/4] add a test against a list of all post codes to ensure we catch them all --- test_parser_gb.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test_parser_gb.py b/test_parser_gb.py index 6612894..07dd648 100644 --- a/test_parser_gb.py +++ b/test_parser_gb.py @@ -4,6 +4,10 @@ import re import pytest +import pandas as pd +import pathlib +import zipfile +import requests import itertools import pyap import pyap.parser @@ -392,6 +396,31 @@ def test_postal_code(input, expected): execute_matching_test(input, expected, data_gb.postal_code) +def test_postal_code_extensive(): + """Test post code regex against a list of all post codes.""" + zip_location = pathlib.Path(__file__).parent / 'code_point_uk_post_codes.zip' + + # Download an extensive list of all postcodes + if not zip_location.exists(): + url = 'https://api.os.uk/downloads/v1/products/CodePointOpen/downloads?area=GB&format=CSV&redirect' + r = requests.get(url, allow_redirects=True) + with open(zip_location.name, 'wb') as f: + f.write(r.content) + + # Run the detector against this list to ensure we pickup all post codes + with zipfile.ZipFile(zip_location.name) as zip: + data_file_names = [ + name for name in zip.namelist() + if name.lower().endswith('.csv') and name.startswith('Data/CSV') + ] + for data_file_name in data_file_names: + with zip.open(data_file_name) as data_file: + df = pd.read_csv(data_file, header=None) + post_codes = df.loc[:, 0].values.tolist() + for post_code in post_codes: + execute_matching_test(post_code, True, data_gb.postal_code) + + @pytest.mark.parametrize("input,expected", [ # positive assertions ("Montana", True), From a20ecd1ceb4e1502388e473356d1b18c6a42c94d Mon Sep 17 00:00:00 2001 From: Thomas Bird Date: Fri, 15 Jan 2021 17:06:48 +0000 Subject: [PATCH 3/4] fix an invalid postcode in the tests --- test_parser_gb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_parser_gb.py b/test_parser_gb.py index 07dd648..0446cdb 100644 --- a/test_parser_gb.py +++ b/test_parser_gb.py @@ -461,7 +461,7 @@ def test_country(input, expected): # positive assertions ("11-59 High Road, East Finchley London, N2 8AW", True), ("88 White parkway, Stanleyton, L2 3DB", True), - ("Studio 96D, Graham roads, Westtown, L1A 3GP, Great Britain", True), + ("Studio 96D, Graham roads, Westtown, L1 3GP, Great Britain", True), ("01 Brett mall, Lake Donna, W02 3JQ", True), ("Flat 05, Byrne shores, Howardshire, GL6 8EA, UK", True), ("12 Henry route, Clementsborough, W2 5DQ", True), From b1198075c63d1d8fc48e097538943394879aa8cf Mon Sep 17 00:00:00 2001 From: Thomas Bird Date: Fri, 15 Jan 2021 17:08:34 +0000 Subject: [PATCH 4/4] add a couple of new deps for the test of all GB postcodes --- tox.ini | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index f81fa40..1458c7a 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,13 @@ envlist = py27, py38 [testenv] -commands = py.test \ - test_parser.py \ - test_parser_ca.py \ - test_parser_us.py \ - test_parser_gb.py +commands = + py.test \ + test_parser.py \ + test_parser_ca.py \ + test_parser_us.py \ + test_parser_gb.py deps = pytest + pandas + requests