fix: tolerate non-UTF-8 svn output instead of crashing (#1383)

SVN servers can emit output in the system code page (e.g. CP1252 on
Windows) rather than UTF-8, for example when a file path contains
accented characters. Every decode of subprocess output in dfetch.vcs.svn
now goes through util.cmdline.decode_subprocess_output, which tries
UTF-8, falls back to CP1252, and finally replaces any byte it still
can't decode, so commands like `dfetch update` no longer crash with
UnicodeDecodeError on such output.

Author: claude

CHANGELOG.rst

@@ -1,3 +1,8 @@
+Release 0.14.4 (unreleased)
+====================================
+
+* Fix SVN commands raising ``UnicodeDecodeError`` on non-UTF-8 output (#1383)
+
 Release 0.14.3 (released 2026-06-25)
 ====================================
 

dfetch/util/cmdline.py

@@ -36,6 +36,26 @@ def message(self) -> str:
         return self._message
 
 
+def decode_subprocess_output(data: bytes) -> str:
+    """Decode bytes from a subprocess, tolerating non-UTF-8 output.
+
+    Command line tools such as ``svn`` can emit output in the system's
+    native code page (e.g. CP1252 on Windows) rather than UTF-8, for
+    example when a file path contains accented characters. UTF-8 is tried
+    first since it is the common case, then CP1252, a common source of
+    non-UTF-8 output. As a last resort, undecodable bytes are replaced so
+    that unexpected output never crashes the command that produced it.
+    """
+    try:
+        return data.decode()
+    except UnicodeDecodeError:
+        pass
+    try:
+        return data.decode(encoding="cp1252")
+    except UnicodeDecodeError:
+        return data.decode(errors="replace")
+
+
 def run_on_cmdline(
     logger: logging.Logger,
     cmd: list[str],
@@ -52,8 +72,8 @@ def run_on_cmdline(
     except subprocess.CalledProcessError as exc:
         raise SubprocessCommandError(
             exc.cmd,
-            exc.output.decode(errors="replace").strip(),
-            exc.stderr.decode(errors="replace").strip(),
+            decode_subprocess_output(exc.output).strip(),
+            decode_subprocess_output(exc.stderr).strip(),
             exc.returncode,
         ) from exc
     except FileNotFoundError as exc:
@@ -66,8 +86,8 @@ def run_on_cmdline(
     if proc.returncode:
         raise SubprocessCommandError(
             cmd,
-            stdout.decode(errors="replace"),
-            stderr.decode(errors="replace").strip(),
+            decode_subprocess_output(stdout),
+            decode_subprocess_output(stderr).strip(),
             proc.returncode,
         )
 
@@ -83,9 +103,5 @@ def _log_output(proc: subprocess.CompletedProcess, logger: logging.Logger) -> No
 
 def _log_output_stream(name: str, stream: Any, logger: logging.Logger) -> None:
     logger.debug(f"{name}:")
-    try:
-        for line in stream.decode().split("\n\n"):
-            logger.debug(line)
-    except UnicodeDecodeError:
-        for line in stream.decode(encoding="cp1252").split("\n\n"):
-            logger.debug(line)
+    for line in decode_subprocess_output(stream).split("\n\n"):
+        logger.debug(line)

dfetch/vcs/svn.py

@@ -14,7 +14,11 @@
 from urllib.parse import urlparse
 
 from dfetch.log import get_logger
-from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline
+from dfetch.util.cmdline import (
+    SubprocessCommandError,
+    decode_subprocess_output,
+    run_on_cmdline,
+)
 from dfetch.util.util import in_directory
 from dfetch.vcs.patch import Patch, PatchType
 
@@ -85,7 +89,7 @@ def _run_svn_raw(args: list[str], *, url: str = "") -> bytes:
 
 def _run_svn(args: list[str], *, url: str = "") -> str:
     """Run an svn subcommand and return decoded stdout (see _run_svn_raw)."""
-    return _run_svn_raw(args, url=url).decode()
+    return decode_subprocess_output(_run_svn_raw(args, url=url))
 
 
 def get_svn_version() -> tuple[str, str]:
@@ -303,7 +307,7 @@ def _inherited_auto_props(self, directory: str) -> str:
                     raise
                 except (SubprocessCommandError, RuntimeError):
                     continue
-                return result.decode()
+                return decode_subprocess_output(result)
         return ""
 
     def externals(self) -> list[External]:
@@ -461,7 +465,9 @@ def get_last_changed_revision(target: str | Path) -> str:
         target_str = str(target).strip()
         if os.path.isdir(target_str):
             last_digits = re.compile(r"(?P<digits>\d+)(?!.*\d)")
-            version = run_on_cmdline(logger, ["svnversion", target_str]).stdout.decode()
+            version = decode_subprocess_output(
+                run_on_cmdline(logger, ["svnversion", target_str]).stdout
+            )
 
             parsed_version = last_digits.search(version)
             if parsed_version:

tests/test_cmdline.py

@@ -10,7 +10,11 @@
 
 import pytest
 
-from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline
+from dfetch.util.cmdline import (
+    SubprocessCommandError,
+    decode_subprocess_output,
+    run_on_cmdline,
+)
 
 LS_CMD = "ls ."
 LS_OK_RESULT = CompletedProcess(
@@ -44,3 +48,15 @@ def test_run_on_cmdline(name, cmd, cmd_result, expectation):
         else:
             with pytest.raises(expectation):
                 run_on_cmdline(logger_mock, cmd)
+
+
+@pytest.mark.parametrize(
+    "name, data, expected",
+    [
+        ("utf-8", "café".encode(), "café"),
+        ("cp1252 fallback", "café".encode("cp1252"), "café"),
+        ("undefined in both codecs is replaced", b"\x81", "�"),
+    ],
+)
+def test_decode_subprocess_output(name, data, expected):
+    assert decode_subprocess_output(data) == expected, name

tests/test_svn_vcs.py

@@ -115,35 +115,33 @@ def test_eol_style_for_without_svn_returns_none(tmp_path):
         assert SvnRepo(tmp_path).eol_style_for("ext/mylib/_") is None
 
 
-def test_export_raises_unicodedecodeerror_on_non_utf8_output():
-    """Reproduces #1383: export() must not crash on non-UTF-8 svn stdout.
+def test_export_tolerates_non_utf8_output():
+    """Regression test for #1383: export() must not crash on non-UTF-8 svn stdout.
 
     Real-world SVN servers (e.g. accessed over HTTP on Windows) can emit
     output in the system code page (such as CP1252) rather than UTF-8, for
     example when printing exported file paths containing accented
-    characters. ``_run_svn`` currently hardcodes ``.decode()`` (UTF-8),
-    so a byte sequence that is invalid UTF-8 but valid CP1252 (like 0xe9,
-    "e" with an acute accent) crashes the whole update instead of being
-    handled gracefully.
+    characters. ``_run_svn`` used to hardcode ``.decode()`` (UTF-8), so a
+    byte sequence that is invalid UTF-8 but valid CP1252 (like 0xe9, "e"
+    with an acute accent) crashed the whole update instead of the export
+    completing.
     """
     with patch("dfetch.vcs.svn.run_on_cmdline") as mock_run:
         mock_run.return_value.stdout = b"A    caf\xe9.txt\n"
-        with pytest.raises(UnicodeDecodeError):
-            SvnRepo.export("svn://example.com/repo", dst="/tmp/out")
+        SvnRepo.export("svn://example.com/repo", dst="/tmp/out")
 
 
-def test_run_svn_raises_unicodedecodeerror_on_non_utf8_output():
-    """Reproduces #1383 at the lower level: any _run_svn caller can crash.
+def test_run_svn_tolerates_non_utf8_output():
+    """Regression test for #1383 at the lower level: any _run_svn caller must not crash.
 
     ``_run_svn`` is used by several ``SvnRepo`` methods (info, externals,
-    files_in_path, ...). None of them can currently tolerate non-UTF-8
-    bytes in svn's stdout, since the decode happens unconditionally before
-    the caller ever sees the output.
+    files_in_path, ...). Non-UTF-8 bytes in svn's stdout must be decoded
+    with a fallback (CP1252) instead of raising, and CP1252-decodable
+    bytes must round-trip to the original text.
     """
     with patch("dfetch.vcs.svn.run_on_cmdline") as mock_run:
-        mock_run.return_value.stdout = b"Path: caf\xe9\n"
-        with pytest.raises(UnicodeDecodeError):
-            SvnRepo.files_in_path("svn://example.com/repo/caf\xe9")
+        mock_run.return_value.stdout = "Path: café\n".encode("cp1252")
+        assert SvnRepo.files_in_path("svn://example.com/repo/café") == ["Path: café"]
 
 
 def test_export_rejects_non_digit_revision():