From 4eeed73666cde15c0a06761999f2f670f1e9f5ed Mon Sep 17 00:00:00 2001 From: ErykKul Date: Fri, 3 Jul 2026 14:38:45 +0200 Subject: [PATCH 1/2] cdi_generator: raise csv field limit; emit keywords in CatalogDetails Two fixes surfaced by profiling real deposited datasets at scale: - csv.field_size_limit: deposited CSVs carry free-text fields (open survey answers, JSON blobs) beyond the csv module's 128 KiB default, which made stream_profile_csv fail with 'field larger than field limit (131072)'. Raise the limit so profiling survives such files. - Keywords: extract_rich_metadata already extracts the citation block's keyword values, but _build_catalog_details never used them, so they were silently dropped from the output. DDI-CDI 1.0 CatalogDetails has no dedicated keyword property, so fold them into typeOfResource together with the subjects (the placement the existing comment already intended). Adds regression tests for both. --- image/cdi_generator_jsonld.py | 14 ++++++++--- image/test_cdi_generator_jsonld.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/image/cdi_generator_jsonld.py b/image/cdi_generator_jsonld.py index 725d080..2da574d 100644 --- a/image/cdi_generator_jsonld.py +++ b/image/cdi_generator_jsonld.py @@ -69,6 +69,11 @@ from datasketch import HyperLogLog from dateutil import parser as dateparser +# Real deposited CSVs carry free-text fields (open survey answers, JSON blobs) +# far beyond the csv module's 128 KiB default field limit; profiling must not +# die on them. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + # ---- Official DDI-CDI 1.0 JSON-LD Context URL ---- DDI_CDI_CONTEXT = "https://ddi-cdi.github.io/m2t-ng/DDI-CDI_1-0/encoding/json-ld/ddi-cdi.jsonld" @@ -1154,11 +1159,14 @@ def _build_catalog_details( if related: catalog["relatedResource"] = related - # Type of resource (subjects/keywords as ControlledVocabularyEntry) - if rich_metadata.get("subjects"): + # Type of resource (subjects/keywords as ControlledVocabularyEntry). + # CatalogDetails has no dedicated keyword property in DDI-CDI 1.0, so + # keywords ride with the subjects here rather than being dropped. + topics = list(rich_metadata.get("subjects", [])) + list(rich_metadata.get("keywords", [])) + if topics: catalog["typeOfResource"] = { "@type": "ControlledVocabularyEntry", - "entryValue": "; ".join(rich_metadata["subjects"]), + "entryValue": "; ".join(topics), } return catalog diff --git a/image/test_cdi_generator_jsonld.py b/image/test_cdi_generator_jsonld.py index 6040b5a..f09f13c 100644 --- a/image/test_cdi_generator_jsonld.py +++ b/image/test_cdi_generator_jsonld.py @@ -172,6 +172,20 @@ def test_simple_csv(self): self.assertEqual(stats[1].xsd_datatype_name(), "string") # name self.assertEqual(stats[2].xsd_datatype_name(), "integer") # age + def test_csv_with_oversized_field(self): + """A free-text field beyond the csv module's 128 KiB default limit + must profile, not crash (real deposited data carries such fields).""" + csv_file = self.test_path / "bigfield.csv" + big = "x" * (256 * 1024) + csv_file.write_text(f"id,text\n1,{big}\n2,small\n") + + cols, stats, info, _, _ = cdi_gen.stream_profile_csv( + csv_file, header=True, compute_md5=False + ) + + self.assertEqual(cols, ["id", "text"]) + self.assertEqual(info["rows_read"], 2) + def test_csv_with_missing_values(self): """Test CSV with missing/null values""" csv_file = self.test_path / "missing.csv" @@ -313,6 +327,29 @@ def test_extract_dataset_description(self): description = cdi_gen.extract_dataset_description(metadata) self.assertEqual(description, "A test description") + def test_catalog_details_emits_keywords(self): + """Keywords extracted from the citation block must reach the + CatalogDetails node (folded into typeOfResource with the subjects, + since DDI-CDI 1.0 CatalogDetails has no keyword property).""" + catalog = cdi_gen._build_catalog_details( + { + "title": "Test Dataset", + "subjects": ["Social Sciences"], + "keywords": ["elections", "turnout"], + } + ) + entry = catalog["typeOfResource"]["entryValue"] + self.assertIn("Social Sciences", entry) + self.assertIn("elections", entry) + self.assertIn("turnout", entry) + + def test_catalog_details_keywords_only(self): + """typeOfResource must be emitted even when only keywords exist.""" + catalog = cdi_gen._build_catalog_details( + {"title": "Test Dataset", "keywords": ["elections"]} + ) + self.assertEqual(catalog["typeOfResource"]["entryValue"], "elections") + class TestDDIMetadata(unittest.TestCase): """Test DDI metadata loading and parsing""" From 78641d48cfd771ec33b34341626c21822490a856 Mon Sep 17 00:00:00 2001 From: ErykKul Date: Fri, 3 Jul 2026 14:46:01 +0200 Subject: [PATCH 2/2] docs: fix MD036 (emphasis used as heading) in README and ddi-cdi.md Pre-existing markdownlint failures that turned the governance check red for every PR; converts the three bold-text pseudo-headings into real headings at the correct level. --- README.md | 2 +- ddi-cdi.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 25fb038..26b694e 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ export BACKEND_CONFIG_FILE=../conf/backend_config.json Globus requires additional frontend plugin, OAuth secret, Dataverse setting, and storage configuration. See [GLOBUS_INTEGRATION.md](GLOBUS_INTEGRATION.md#configuration). -**Dataverse File System Drivers** +#### Dataverse File System Drivers The application can directly upload files to the Dataverse file system. Two drivers are supported: diff --git a/ddi-cdi.md b/ddi-cdi.md index 916e23b..ae9bb1c 100644 --- a/ddi-cdi.md +++ b/ddi-cdi.md @@ -383,12 +383,14 @@ When generation completes (either by waiting or returning later): After generation, you have two options: -**Option A: Download** +##### Option A: Download + - Click the **"Download"** button to save the JSON-LD file locally - File is named `ddi-cdi-[timestamp].jsonld` - Useful for archiving or sharing outside of Dataverse -**Option B: Open in Viewer (Recommended)** +##### Option B: Open in Viewer (Recommended) + - Click the **"Open in Viewer"** button to open the **cdi-viewer** in a new browser window - The system first adds your DDI-CDI file to the dataset with the correct MIME type - The viewer opens and loads the file directly from Dataverse