From 4ed72fb3a9cb0e429a35bd582ba38495dc5dc1b4 Mon Sep 17 00:00:00 2001 From: Samuel Progin Date: Sat, 1 Sep 2018 00:00:20 +0200 Subject: [PATCH 1/2] backporting UnicodeDecodeError protection Signed-off-by: Samuel Progin --- noisy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/noisy.py b/noisy.py index 341389c..ca4aed0 100644 --- a/noisy.py +++ b/noisy.py @@ -101,7 +101,10 @@ def _is_blacklisted(self, url): :param url: full URL :return: boolean indicating whether a URL is blacklisted or not """ - return any(blacklisted_url in url for blacklisted_url in self._config["blacklisted_urls"]) + try: + return any(blacklisted_url in url for blacklisted_url in self._config["blacklisted_urls"]) + except UnicodeDecodeError: + return True def _should_accept_url(self, url): """ From f2caaa25541520ca70d1e3702235f8706b9f2aaa Mon Sep 17 00:00:00 2001 From: Samuel Progin Date: Wed, 27 Feb 2019 21:15:00 +0100 Subject: [PATCH 2/2] Protection from another potentially leaking exception from requests Signed-off-by: Samuel Progin --- noisy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noisy.py b/noisy.py index ca4aed0..5b56096 100644 --- a/noisy.py +++ b/noisy.py @@ -174,7 +174,7 @@ def _browse_from_links(self, depth=0): # remove the dead-end link from our list self._remove_and_blacklist(random_link) - except requests.exceptions.RequestException: + except (requests.exceptions.RequestException, UnicodeDecodeError): logging.debug("Exception on URL: %s, removing from list and trying again!" % random_link) self._remove_and_blacklist(random_link)