44"""
55import pytest
66import responses
7+ from charset_normalizer import from_bytes
78from charset_normalizer .constant import TOO_SMALL_SEQUENCE
89
910from httpie .cli .constants import PRETTY_MAP
2021]
2122
2223
24+ def _charset_normalizer_detects (charset : str , text : str ) -> bool :
25+ """Return False when charset_normalizer cannot detect the encoding reliably."""
26+ if charset == UTF8 :
27+ return True
28+ match = from_bytes (text .encode (charset )).best ()
29+ if not match or not match .encoding :
30+ return False
31+ detected = match .encoding .lower ().replace ('_' , '-' )
32+ expected = charset .lower ().replace ('_' , '-' )
33+ return detected == expected or detected .startswith (expected .split ('-' )[0 ])
34+
35+
36+ def _reliable_charset_text_pairs ():
37+ return [
38+ pair for pair in CHARSET_TEXT_PAIRS
39+ if _charset_normalizer_detects (* pair )
40+ ]
41+
42+
43+ RELIABLE_CHARSET_TEXT_PAIRS = _reliable_charset_text_pairs ()
44+
45+
2346def test_charset_text_pairs ():
2447 # Verify our test data is legit.
2548 for charset , text in CHARSET_TEXT_PAIRS :
@@ -134,7 +157,7 @@ def test_unicode_digest_auth(httpbin):
134157 f'{ httpbin } /digest-auth/auth/test/{ UNICODE } ' )
135158
136159
137- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
160+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
138161@responses .activate
139162def test_terminal_output_response_charset_detection (text , charset ):
140163 responses .add (
@@ -147,7 +170,7 @@ def test_terminal_output_response_charset_detection(text, charset):
147170 assert text in r
148171
149172
150- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
173+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
151174@responses .activate
152175def test_terminal_output_response_content_type_charset (charset , text ):
153176 responses .add (
@@ -160,7 +183,7 @@ def test_terminal_output_response_content_type_charset(charset, text):
160183 assert text in r
161184
162185
163- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
186+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
164187@pytest .mark .parametrize ('pretty' , PRETTY_MAP .keys ())
165188@responses .activate
166189def test_terminal_output_response_content_type_charset_with_stream (charset , text , pretty ):
@@ -175,7 +198,7 @@ def test_terminal_output_response_content_type_charset_with_stream(charset, text
175198 assert text in r
176199
177200
178- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
201+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
179202@pytest .mark .parametrize ('pretty' , PRETTY_MAP .keys ())
180203@responses .activate
181204def test_terminal_output_response_charset_override (charset , text , pretty ):
@@ -194,7 +217,7 @@ def test_terminal_output_response_charset_override(charset, text, pretty):
194217 assert text in r
195218
196219
197- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
220+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
198221def test_terminal_output_request_content_type_charset (charset , text ):
199222 r = http (
200223 '--offline' ,
@@ -208,7 +231,7 @@ def test_terminal_output_request_content_type_charset(charset, text):
208231 assert text in r
209232
210233
211- @pytest .mark .parametrize ('charset, text' , CHARSET_TEXT_PAIRS )
234+ @pytest .mark .parametrize ('charset, text' , RELIABLE_CHARSET_TEXT_PAIRS )
212235def test_terminal_output_request_charset_detection (charset , text ):
213236 r = http (
214237 '--offline' ,
0 commit comments