From 10fe04f2023a9459b57f8d23d6e340ef5396f736 Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 19 Jun 2026 09:21:53 +1000
Subject: [PATCH 1/2] Default low_memory=False when reading CSVs with the C
 parser

pandas' default low_memory=True reads a CSV in chunks and infers each
column's dtype per chunk.
For mostly-null or mixed-type metadata columns this is non-deterministic
across runs and emits a DtypeWarning ("Columns (N) have mixed types"),
so the same file can load as object on one machine and float64 on another.

Default low_memory=False so dtype inference is a single, deterministic,
quiet pass.
The option is only valid for the C parser, so it is left untouched for the
python and pyarrow engines (which reject it), and an explicit caller value
still wins.
---
 src/scmdata/run.py     |  7 ++++++
 tests/unit/test_run.py | 54 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/src/scmdata/run.py b/src/scmdata/run.py
index 62be9471..7736d181 100644
--- a/src/scmdata/run.py
+++ b/src/scmdata/run.py
@@ -149,6 +149,13 @@ def _read_pandas(
 
     else:
         _logger.debug("Reading with pandas read_csv")
+        # pandas' default ``low_memory=True`` reads the file in chunks and infers
+        # each column's dtype per chunk. For mostly-null or mixed-type metadata
+        # columns this is non-deterministic across runs and emits a ``DtypeWarning``.
+        # Reading the whole column at once makes inference deterministic and quiet.
+        # ``low_memory`` is only accepted by the C parser, so leave other engines be.
+        if (kwargs.get("engine") or "c") == "c":
+            kwargs.setdefault("low_memory", False)
         dateframe = pd.read_csv(fname, *args, **kwargs)
 
     def _to_lower(c):
diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py
index 6d19fdd6..cc349f77 100644
--- a/tests/unit/test_run.py
+++ b/tests/unit/test_run.py
@@ -3367,6 +3367,60 @@ def test_read_from_disk(test_file, test_kwargs, test_data_path, use_pathlib):
     )
 
 
+def _write_two_timeseries_csv(tmp_path):
+    df = pd.DataFrame(
+        {
+            "model": "idealised",
+            "scenario": "idealised",
+            "region": ["World", "Europe"],
+            "variable": "Emissions|CO2",
+            "unit": "GtC / yr",
+            "2020": [1.0, 2.0],
+        }
+    )
+    fname = tmp_path / "two_timeseries.csv"
+    df.to_csv(fname, index=False)
+    return fname
+
+
+def _capture_read_csv_kwargs(call):
+    """Run ``call`` while spying on ``pd.read_csv``; return the kwargs it received."""
+    captured = {}
+    real_read_csv = pd.read_csv
+
+    def _spy(*args, **kwargs):
+        captured.update(kwargs)
+        return real_read_csv(*args, **kwargs)
+
+    with patch.object(pd, "read_csv", _spy):
+        call()
+    return captured
+
+
+def test_read_csv_defaults_to_low_memory_false(tmp_path):
+    # pandas' default low_memory=True infers each column's dtype per-chunk, which
+    # is non-deterministic for mostly-null / mixed-type metadata columns and emits
+    # a DtypeWarning. ScmRun should read in a single deterministic, quiet pass.
+    fname = _write_two_timeseries_csv(tmp_path)
+    captured = _capture_read_csv_kwargs(lambda: ScmRun(fname))
+    assert captured["low_memory"] is False
+
+
+def test_read_csv_respects_explicit_low_memory(tmp_path):
+    # An explicit low_memory must win over the injected default.
+    fname = _write_two_timeseries_csv(tmp_path)
+    captured = _capture_read_csv_kwargs(lambda: ScmRun(fname, low_memory=True))
+    assert captured["low_memory"] is True
+
+
+def test_read_csv_python_engine_omits_low_memory(tmp_path):
+    # low_memory is rejected by the python parser engine, so it must not be
+    # injected when the caller selects it -- otherwise read_csv raises ValueError.
+    fname = _write_two_timeseries_csv(tmp_path)
+    captured = _capture_read_csv_kwargs(lambda: ScmRun(fname, engine="python"))
+    assert "low_memory" not in captured
+
+
 def test_read_from_disk_different_number_of_digits_years(test_data_path):
     loaded = ScmRun(
         os.path.join(test_data_path, "different_number_of_digits_years.csv")

From 258fd3f0b714d1d4b8b24419564a5e3ce60ea0c1 Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 19 Jun 2026 09:23:37 +1000
Subject: [PATCH 2/2] Add changelog fragment for PR #322

---
 changelog/322.fix.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 changelog/322.fix.md

diff --git a/changelog/322.fix.md b/changelog/322.fix.md
new file mode 100644
index 00000000..803ffb9e
--- /dev/null
+++ b/changelog/322.fix.md
@@ -0,0 +1,3 @@
+Reading timeseries from a CSV now defaults to ``low_memory=False`` for the C parser, so column dtype inference is a single deterministic pass.
+This stops a mostly-null or mixed-type metadata column from loading as ``object`` on one run and ``float64`` on another, and silences the accompanying ``DtypeWarning``.
+An explicitly supplied ``low_memory`` is still respected, and the option is left untouched for the ``python`` and ``pyarrow`` engines.