cdi_generator: raise csv field limit; emit keywords in CatalogDetails

README.md

@@ -199,7 +199,7 @@ export BACKEND_CONFIG_FILE=../conf/backend_config.json
 Globus requires additional frontend plugin, OAuth secret, Dataverse setting,
 and storage configuration. See [GLOBUS_INTEGRATION.md](GLOBUS_INTEGRATION.md#configuration).
 
-**Dataverse File System Drivers**
+#### Dataverse File System Drivers
 
 The application can directly upload files to the Dataverse file system. Two drivers are supported:
 

ddi-cdi.md

@@ -383,12 +383,14 @@ When generation completes (either by waiting or returning later):
 
 After generation, you have two options:
 
-**Option A: Download**
+##### Option A: Download
+
 - Click the **"Download"** button to save the JSON-LD file locally
 - File is named `ddi-cdi-[timestamp].jsonld`
 - Useful for archiving or sharing outside of Dataverse
 
-**Option B: Open in Viewer (Recommended)**
+##### Option B: Open in Viewer (Recommended)
+
 - Click the **"Open in Viewer"** button to open the **cdi-viewer** in a new browser window
 - The system first adds your DDI-CDI file to the dataset with the correct MIME type
 - The viewer opens and loads the file directly from Dataverse

image/cdi_generator_jsonld.py

@@ -69,6 +69,11 @@
 from datasketch import HyperLogLog
 from dateutil import parser as dateparser
 
+# Real deposited CSVs carry free-text fields (open survey answers, JSON blobs)
+# far beyond the csv module's 128 KiB default field limit; profiling must not
+# die on them.
+csv.field_size_limit(min(sys.maxsize, 2**31 - 1))
+
 
 # ---- Official DDI-CDI 1.0 JSON-LD Context URL ----
 DDI_CDI_CONTEXT = "https://ddi-cdi.github.io/m2t-ng/DDI-CDI_1-0/encoding/json-ld/ddi-cdi.jsonld"
@@ -1154,11 +1159,14 @@ def _build_catalog_details(
         if related:
             catalog["relatedResource"] = related
 
-    # Type of resource (subjects/keywords as ControlledVocabularyEntry)
-    if rich_metadata.get("subjects"):
+    # Type of resource (subjects/keywords as ControlledVocabularyEntry).
+    # CatalogDetails has no dedicated keyword property in DDI-CDI 1.0, so
+    # keywords ride with the subjects here rather than being dropped.
+    topics = list(rich_metadata.get("subjects", [])) + list(rich_metadata.get("keywords", []))
+    if topics:
         catalog["typeOfResource"] = {
             "@type": "ControlledVocabularyEntry",
-            "entryValue": "; ".join(rich_metadata["subjects"]),
+            "entryValue": "; ".join(topics),
         }
 
     return catalog

image/test_cdi_generator_jsonld.py

@@ -172,6 +172,20 @@ def test_simple_csv(self):
         self.assertEqual(stats[1].xsd_datatype_name(), "string")   # name
         self.assertEqual(stats[2].xsd_datatype_name(), "integer")  # age
 
+    def test_csv_with_oversized_field(self):
+        """A free-text field beyond the csv module's 128 KiB default limit
+        must profile, not crash (real deposited data carries such fields)."""
+        csv_file = self.test_path / "bigfield.csv"
+        big = "x" * (256 * 1024)
+        csv_file.write_text(f"id,text\n1,{big}\n2,small\n")
+
+        cols, stats, info, _, _ = cdi_gen.stream_profile_csv(
+            csv_file, header=True, compute_md5=False
+        )
+
+        self.assertEqual(cols, ["id", "text"])
+        self.assertEqual(info["rows_read"], 2)
+
     def test_csv_with_missing_values(self):
         """Test CSV with missing/null values"""
         csv_file = self.test_path / "missing.csv"
@@ -313,6 +327,29 @@ def test_extract_dataset_description(self):
         description = cdi_gen.extract_dataset_description(metadata)
         self.assertEqual(description, "A test description")
 
+    def test_catalog_details_emits_keywords(self):
+        """Keywords extracted from the citation block must reach the
+        CatalogDetails node (folded into typeOfResource with the subjects,
+        since DDI-CDI 1.0 CatalogDetails has no keyword property)."""
+        catalog = cdi_gen._build_catalog_details(
+            {
+                "title": "Test Dataset",
+                "subjects": ["Social Sciences"],
+                "keywords": ["elections", "turnout"],
+            }
+        )
+        entry = catalog["typeOfResource"]["entryValue"]
+        self.assertIn("Social Sciences", entry)
+        self.assertIn("elections", entry)
+        self.assertIn("turnout", entry)
+
+    def test_catalog_details_keywords_only(self):
+        """typeOfResource must be emitted even when only keywords exist."""
+        catalog = cdi_gen._build_catalog_details(
+            {"title": "Test Dataset", "keywords": ["elections"]}
+        )
+        self.assertEqual(catalog["typeOfResource"]["entryValue"], "elections")
+
 
 class TestDDIMetadata(unittest.TestCase):
     """Test DDI metadata loading and parsing"""