From 04a229ff94ce95db1233da76b27e6c09091c7f06 Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Sat, 15 Nov 2025 11:10:08 -0500 Subject: [PATCH 1/4] fix: fallback to GET from HEAD in redirect check on 500 Some sites do not implement HEAD, giving a 500, but, are otherwise valid sites. Fallback to GET in this case. Resolves #126 --- courlan/network.py | 7 +++++++ tests/unit_tests.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/courlan/network.py b/courlan/network.py index b637ff5e..990fca32 100644 --- a/courlan/network.py +++ b/courlan/network.py @@ -64,6 +64,13 @@ def redirection_test(url: str) -> str: LOGGER.exception("unknown error: %s %s", url, err) else: # response + if rhead.status == 500: + # Some sites don't implement HEAD, fallback to GET + try: + rhead = HTTP_POOL.request("GET", url) # type:ignore[no-untyped-call] + except Exception as err: + LOGGER.exception("unknown error: %s %s", url, err) + raise ValueError(f"cannot reach URL: ${url}") if rhead.status in ACCEPTABLE_CODES: LOGGER.debug("result found: %s %s", rhead.geturl(), rhead.status) return rhead.geturl() # type: ignore diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 39d18ca1..feec4925 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -758,6 +758,38 @@ def test_urlcheck_redirects(): assert check_url("https://www.ht.or", with_redirects=True) is None +def test_redirection_fallback_from_head_to_get(): + """Test that redirection_test falls back to GET when HEAD returns 500.""" + from unittest.mock import Mock, patch + from courlan.network import redirection_test + + test_url = "https://example.org/test" + + # Create mock responses + mock_head_response = Mock() + mock_head_response.status = 500 + mock_head_response.geturl = Mock(return_value=test_url) + + mock_get_response = Mock() + mock_get_response.status = 200 + mock_get_response.geturl = Mock(return_value=test_url) + + # Patch HTTP_POOL.request to return 500 for HEAD, then 200 for GET + with patch('courlan.network.HTTP_POOL.request') as mock_request: + # Set up side_effect to return different responses for HEAD and GET + mock_request.side_effect = [mock_head_response, mock_get_response] + + result = redirection_test(test_url) + + # Verify the function returned the correct URL + assert result == test_url + + # Verify that both HEAD and GET were called + assert mock_request.call_count == 2 + assert mock_request.call_args_list[0][0] == ("HEAD", test_url) + assert mock_request.call_args_list[1][0] == ("GET", test_url) + + def test_urlutils(): """Test URL manipulation tools""" # domain extraction From ea0383a27524e19f488be8f5bb2c53ba765bd909 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Fri, 19 Dec 2025 10:50:00 +0100 Subject: [PATCH 2/4] fix: code consistency --- courlan/network.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/courlan/network.py b/courlan/network.py index 990fca32..4120e588 100644 --- a/courlan/network.py +++ b/courlan/network.py @@ -18,7 +18,6 @@ status_forcelist=[ 429, 499, - 500, 502, 503, 504, @@ -61,18 +60,16 @@ def redirection_test(url: str) -> str: try: rhead = HTTP_POOL.request("HEAD", url) # type:ignore[no-untyped-call] except Exception as err: - LOGGER.exception("unknown error: %s %s", url, err) + LOGGER.exception("unknown HEAD error: %s %s", url, err) else: - # response - if rhead.status == 500: - # Some sites don't implement HEAD, fallback to GET - try: - rhead = HTTP_POOL.request("GET", url) # type:ignore[no-untyped-call] - except Exception as err: - LOGGER.exception("unknown error: %s %s", url, err) - raise ValueError(f"cannot reach URL: ${url}") if rhead.status in ACCEPTABLE_CODES: LOGGER.debug("result found: %s %s", rhead.geturl(), rhead.status) return rhead.geturl() # type: ignore + # Some sites don't implement HEAD, fallback to GET + elif rhead.status == 500: + try: + rhead = HTTP_POOL.request("GET", url) # type:ignore[no-untyped-call] + except Exception as err: + LOGGER.exception("unknown GET error: %s %s", url, err) # else: raise ValueError(f"cannot reach URL: ${url}") From bf3d486d9a24c6d3d7a8e4daea0c271c46eb24a4 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Fri, 19 Dec 2025 11:03:04 +0100 Subject: [PATCH 3/4] fix: return value for redirection test --- courlan/network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/courlan/network.py b/courlan/network.py index 4120e588..9503c51c 100644 --- a/courlan/network.py +++ b/courlan/network.py @@ -69,6 +69,9 @@ def redirection_test(url: str) -> str: elif rhead.status == 500: try: rhead = HTTP_POOL.request("GET", url) # type:ignore[no-untyped-call] + if rhead.status in ACCEPTABLE_CODES: + LOGGER.debug("result found with GET: %s %s", rhead.geturl(), rhead.status) + return rhead.geturl() # type: ignore except Exception as err: LOGGER.exception("unknown GET error: %s %s", url, err) # else: From 94e8dc2df3d8b14ba9c717c6ec688e87babb3528 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 13 Jan 2026 16:38:17 +0100 Subject: [PATCH 4/4] fix: redirection test with httpbin --- tests/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 9454cb5c..d09659d1 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -750,7 +750,7 @@ def test_domain_filter(): def test_urlcheck_redirects(): "Test redirection checks." - assert check_url("https://httpbun.org/redirect-to?url=http%3A%2F%2Fexample.org", with_redirects=True) == ( + assert check_url("https://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.org", with_redirects=True) == ( "http://example.org", "example.org", )