fix(console): centralize UTF-8 stdio so emoji logs never fail translations

benchmark/cli.py

@@ -18,15 +18,9 @@
 from typing import Optional
 
 # Force UTF-8 stdio so emoji prints (e.g. 💬, ❌, ⚠️) don't crash on Windows
-# consoles using cp1252 (charmap codec). errors='replace' keeps the process
-# alive even on terminals that can't render some glyphs.
-for _stream_name in ("stdout", "stderr"):
-    _stream = getattr(sys, _stream_name, None)
-    if _stream is not None and hasattr(_stream, "reconfigure"):
-        try:
-            _stream.reconfigure(encoding="utf-8", errors="replace")
-        except Exception:
-            pass
+# cp1252 consoles. See issue #184.
+from src.utils.console import ensure_utf8_stdio
+ensure_utf8_stdio()
 
 from benchmark.aggregator import CLOUD_PROVIDERS, BenchmarkAggregator
 from benchmark.config import BenchmarkConfig, DEFAULT_EVALUATOR_MODEL, DEFAULT_EVALUATOR_PROVIDER, DEFAULT_POE_EVALUATOR_MODEL

src/__version__.py

@@ -1 +1 @@
-__version__ = "1.4.4"
+__version__ = "1.4.5"

src/utils/console.py

@@ -0,0 +1,51 @@
+"""
+Console output helpers.
+
+Centralizes the one piece of setup every entrypoint needs: forcing stdout and
+stderr to UTF-8 so emoji log lines (e.g. 💬, ✅, ❌, ⚠️) never crash on Windows
+consoles using the cp1252 'charmap' codec.
+
+Previously each entrypoint reimplemented this (with two divergent patterns), and
+the main CLI forgot it entirely, so an emoji print inside a provider's request
+loop raised UnicodeEncodeError, was caught by the broad `except`, and failed the
+translation of that unit. See issue #184.
+
+Call ensure_utf8_stdio() once at the top of every entrypoint.
+"""
+
+import sys
+
+
+def _is_utf8(stream) -> bool:
+    enc = (getattr(stream, "encoding", "") or "").lower().replace("-", "")
+    return enc == "utf8"
+
+
+def ensure_utf8_stdio() -> None:
+    """Force sys.stdout / sys.stderr to UTF-8 with errors='replace'.
+
+    Idempotent and defensive: it skips streams already in UTF-8, prefers the
+    modern TextIOWrapper.reconfigure(), falls back to wrapping the raw buffer
+    for streams that lack it, and never raises (a failure here must not take
+    down the process it is meant to protect).
+    """
+    for name in ("stdout", "stderr"):
+        stream = getattr(sys, name, None)
+        if stream is None or _is_utf8(stream):
+            continue
+
+        reconfigure = getattr(stream, "reconfigure", None)
+        if callable(reconfigure):
+            try:
+                reconfigure(encoding="utf-8", errors="replace")
+                continue
+            except Exception:
+                pass  # Fall through to the buffer-wrapping fallback.
+
+        buffer = getattr(stream, "buffer", None)
+        if buffer is not None:
+            try:
+                import codecs
+                setattr(sys, name, codecs.getwriter("utf-8")(buffer, "replace"))
+            except Exception:
+                pass

tests/unit/test_console_utf8.py

@@ -0,0 +1,92 @@
+"""
+Unit tests for src.utils.console.ensure_utf8_stdio (issue #184).
+
+These exercise the decision logic against fake streams so they never mutate the
+real test-runner stdout/stderr.
+"""
+
+import io
+import sys
+
+from src.utils.console import ensure_utf8_stdio
+
+
+class ReconfigurableStream:
+    """Stand-in for a TextIOWrapper that supports reconfigure()."""
+
+    def __init__(self, encoding, raises=False, buffer=None):
+        self.encoding = encoding
+        self.calls = []
+        self._raises = raises
+        self.buffer = buffer
+
+    def reconfigure(self, encoding=None, errors=None):
+        self.calls.append((encoding, errors))
+        if self._raises:
+            raise OSError("reconfigure not supported")
+        self.encoding = encoding
+
+
+class LegacyStream:
+    """Stand-in for a stream without reconfigure() but with a raw buffer."""
+
+    def __init__(self, encoding, buffer):
+        self.encoding = encoding
+        self.buffer = buffer
+
+
+def test_reconfigures_non_utf8_stream(monkeypatch):
+    out = ReconfigurableStream("cp1252")
+    err = ReconfigurableStream("cp1252")
+    monkeypatch.setattr(sys, "stdout", out)
+    monkeypatch.setattr(sys, "stderr", err)
+
+    ensure_utf8_stdio()
+
+    assert out.calls == [("utf-8", "replace")]
+    assert err.calls == [("utf-8", "replace")]
+
+
+def test_idempotent_on_utf8_stream(monkeypatch):
+    for enc in ("utf-8", "UTF-8", "utf8"):
+        out = ReconfigurableStream(enc)
+        monkeypatch.setattr(sys, "stdout", out)
+        monkeypatch.setattr(sys, "stderr", ReconfigurableStream(enc))
+        ensure_utf8_stdio()
+        assert out.calls == [], f"reconfigure should be skipped for {enc}"
+
+
+def test_falls_back_to_buffer_wrapping(monkeypatch):
+    raw = io.BytesIO()
+    legacy = LegacyStream("cp1252", buffer=raw)
+    monkeypatch.setattr(sys, "stdout", legacy)
+    monkeypatch.setattr(sys, "stderr", LegacyStream("cp1252", buffer=io.BytesIO()))
+
+    ensure_utf8_stdio()
+
+    # stdout was swapped for a UTF-8 writer over the same buffer; an emoji must
+    # now encode without raising (the original cp1252 crash, '\U0001f4ac').
+    assert sys.stdout is not legacy
+    sys.stdout.write("\U0001f4ac ok")
+    sys.stdout.flush()
+    assert "\U0001f4ac ok".encode("utf-8") in raw.getvalue()
+
+
+def test_falls_back_when_reconfigure_raises(monkeypatch):
+    raw = io.BytesIO()
+    stream = ReconfigurableStream("cp1252", raises=True, buffer=raw)
+    monkeypatch.setattr(sys, "stdout", stream)
+    monkeypatch.setattr(sys, "stderr", ReconfigurableStream("cp1252", raises=True, buffer=io.BytesIO()))
+
+    ensure_utf8_stdio()
+
+    # reconfigure was attempted, then we fell back to wrapping the buffer.
+    assert stream.calls == [("utf-8", "replace")]
+    assert sys.stdout is not stream
+
+
+def test_never_raises_on_missing_streams(monkeypatch):
+    monkeypatch.setattr(sys, "stdout", None)
+    monkeypatch.setattr(sys, "stderr", None)
+    # Must not raise even when there is nothing to fix.
+    ensure_utf8_stdio()

translate.py

@@ -6,6 +6,11 @@
 import asyncio
 import logging
 
+# Force UTF-8 stdio before anything prints, so emoji log lines (💬, ✅, ❌, ...)
+# don't crash on Windows cp1252 consoles. See issue #184.
+from src.utils.console import ensure_utf8_stdio
+ensure_utf8_stdio()
+
 # Reduce verbosity of httpx (avoid showing 400 errors during model detection)
 logging.getLogger('httpx').setLevel(logging.WARNING)
 

translation_api.py

@@ -39,11 +39,9 @@
 # Reduce verbosity of httpx (avoid showing 400 errors during model detection)
 logging.getLogger('httpx').setLevel(logging.WARNING)
 
-# Fix Windows console encoding for emojis
-if sys.platform == 'win32':
-    import codecs
-    sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'replace')
-    sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'replace')
+# Force UTF-8 stdio so emoji log lines don't crash on Windows cp1252 consoles.
+from src.utils.console import ensure_utf8_stdio
+ensure_utf8_stdio()
 
 from src.config import (
     API_ENDPOINT as DEFAULT_OLLAMA_API_ENDPOINT,