Use the default namespace in XML export

README.md

@@ -10,7 +10,7 @@ A Python script to automate the preparation of oceanographic datasets for submis
 2. **[ICOS Carbon Portal](https://www.icos-cp.eu/)** – data repository providing persistent identifiers (PIDs/DOIs)
 3. **OADS XML template** – a dataset-specific metadata file pre-filled by the user (see [XML template](#xml-template))
 
-The script downloads a dataset from QuinCE, enriches it with metadata from the Carbon Portal, populates a SOCAT-compliant OADS XML metadata file, prepends the required SOCAT header to the TSV data file, and repackages everything into a zip archive ready for SOCAT upload.
+The script downloads a dataset from QuinCE, enriches it with metadata from the Carbon Portal, populates a SOCAT-compliant OADS XML metadata file, prepends the required SOCAT header to the TSV data file, and copies the resulting `.tsv` and `.xml` files to an output folder ready for SOCAT upload. The script runs on both Linux and Windows.
 
 ## Workflow
 
@@ -30,7 +30,7 @@ QuinCE API ──► Download dataset (zip)
               Write SOCAT header to .tsv
                      │
                      ▼
-              Repack ──► output zip (SOCAT-ready)
+         Move .tsv + .xml ──► output folder (SOCAT-ready)
                      │
                      ▼
               Clean up temporary files
@@ -41,7 +41,7 @@ QuinCE API ──► Download dataset (zip)
 - Python 3.x
 - [`icoscp_core`](https://pypi.org/project/icoscp-core/) – ICOS Carbon Portal client
 - [`requests`](https://pypi.org/project/requests/) – HTTP library for the QuinCE API
-- `os`, `sys`, `json`, `zipfile`, `glob`, `subprocess`, `xml.etree.ElementTree`, `datetime` – Python standard library (included with Python)
+- `os`, `sys`, `json`, `zipfile`, `tempfile`, `shutil`, `xml.etree.ElementTree`, `datetime` – Python standard library (included with Python)
 
 Install third-party dependencies:
 
@@ -103,8 +103,8 @@ python prep_SOCAT.py -n <dataset_name> -S <template_name> [options]
 
 | Flag | Default | Description |
 |---|---|---|
-| `-t`, `--tmp` | `/tmp/` | Temporary working directory |
-| `-o`, `--output` | Current directory | Output folder for the final zip |
+| `-t`, `--tmp` | System temp folder | Temporary working directory |
+| `-o`, `--output` | Dataset name | Output folder for the final `.tsv` and `.xml` files |
 | `-d`, `--data` | `Data/` | Folder containing `credentials.json` and the XML template |
 | `-v`, `--version` | — | Print version and exit |
 
@@ -119,7 +119,7 @@ This will:
 2. Read `Data/1199_template.xml` and fill in the `TK` fields
 3. Query the Carbon Portal for the corresponding PID/URI
 4. Prepend the SOCAT header to the TSV file
-5. Write `output/119920230901.zip` ready for SOCAT upload
+5. Write `output/119920230901.tsv` and `output/119920230901.xml` ready for SOCAT upload
 
 ## Exit codes
 
@@ -134,7 +134,12 @@ This will:
 
 ## Version
 
-Current version: **1.0**
+Current version: **1.1**
+
+### Changelog
+
+- **1.1** – Output the SOCAT-ready `.tsv` and `.xml` files directly to the output folder instead of repacking into a zip archive. Added Windows compatibility (system-independent temp folder and path handling). Various bug fixes.
+- **1.0** – Initial release.
 
 ## Author
 

prep_SOCAT.py

@@ -9,7 +9,7 @@
 # Major version number.
 __major__ = '1'
 # Minor version number.
-__minor__ = '0'
+__minor__ = '1'
 # Script version.
 __version__ = __major__ + '.' + __minor__
 
@@ -20,30 +20,17 @@
 import xml.etree.ElementTree as ET
 import json
 import zipfile
-import glob
-import subprocess
+import tempfile
 from datetime import datetime as dt
 from icoscp_core.icos import meta
 import shutil
 
-# Check if the folder exists and create it if not
-#
-def make_folder(folder):
-    if (folder != '') & (folder != '.'):
-        if not os.path.isdir(folder):
-            try:
-                os.makedirs(folder)
-            except OSError as e:
-                print("Error: could not create the folder:", e)
-                sys.exit(3)
-
 # Run a QuinCE API call to download the dataset corresponding to the given filename
 #
 def QuinCe_API(dataset_name, data_file):
     import requests
     from requests.auth import HTTPBasicAuth
-    import json
-
+
     # Load configuration for the QuinCe instance
     try:
         with open(os.path.join(base_path, 'credentials.json')) as f:
@@ -71,7 +58,7 @@ def QuinCe_API(dataset_name, data_file):
                 if fsize >= 1024:
                     if fsize >= 1024*1024:
                         size_str = 'MB'
-                        fsize = fsize/1024*1024
+                        fsize = fsize/(1024*1024)
                     else:
                         size_str = 'kB'
                         fsize = fsize/1024
@@ -114,16 +101,14 @@ def unzip_data(data_file):
     except OSError as e:
         print('Warning: the zip file could not be removed: ', e)
 
-# Load the metadata contained in the manifest.json file as well as the data in
-# the tsv file.
-# The original data zipfile is unpacked into the temp folder in order to load
-# the .tsv file.
+# Unpack the archive exported from QuinCe into the temp folder (which also makes
+# the .tsv data file available) and load the metadata from the manifest.json file.
 #
 def import_metadata(tmp_folder, data_file):
     unzip_data(data_file)
-    metadata_fname = os.path.join(tmp_folder, 'manifest.json')
+    manifest_file = os.path.join(tmp_folder, 'manifest.json')
     try:
-        with open(metadata_fname, 'r') as file:
+        with open(manifest_file, 'r') as file:
             metadata = json.load(file)
     except (OSError, json.JSONDecodeError) as e:
         print('Error: could not load "manifest.json" metadata: ', e)
@@ -216,18 +201,29 @@ def populate_xml(xml_data, metadata):
         child = populate_xml(child, metadata)
     return xml_data
 
+# Get the default namespace from the imported XML
+def get_namespace_uri(el):
+    # ChatGPT gave me this. I hope it works - Steve
+    if el.tag.startswith("{"):
+        return el.tag.split("}", 1)[0][1:]
+    return None
 
-# Save the completed xml file into the data folder
+# Save the completed xml file into the temporary folder
 #
 def save_xml(xml_data, tmp_folder):
-    fname = os.path.join(tmp_folder, tmp_folder.split('/')[-1] + '.xml')
+    fname = os.path.join(tmp_folder, os.path.basename(tmp_folder) + '.xml')
+
+    # We want to use the default (empty) namespace on export
+    ET.register_namespace("", get_namespace_uri(xml_data))
+
     xml_str = ET.tostring(xml_data).decode()
     try:
         with open(fname, 'w') as xml_file:
             xml_file.write(xml_str)
     except OSError as e:
         print('Error: could not save the metadata to the xml file: ', e)
         sys.exit(5)
+    return fname
 
 # Find the value corresponding to a keys tree in the given xml etree.
 # If several values are found, a semicolon separated string is returned
@@ -248,7 +244,7 @@ def find_leaf(xml_data, keys, multiple_entries=False):
 
 # Add the "SOCAT" header to the datafile
 #
-def write_header(data_file, xml_data):
+def write_header(tsv_file, xml_data):
     # Prepare the header from the xml metadata
     expocode = find_leaf(xml_data, 'expocodes/expocode')
     vessel = find_leaf(xml_data, 'platforms/platform/name')
@@ -258,22 +254,29 @@ def write_header(data_file, xml_data):
          f"Vessel type: {vtype} \n")
     try:
         # Read the content of the existing file
-        with open(data_fname, "r") as f:
+        with open(tsv_file, "r") as f:
             content = f.read()
         # Write header + existing content
-        with open(data_fname, "w") as f:
+        with open(tsv_file, "w") as f:
             f.write(h + content)
     except OSError as e:
         print('Error: could not save the data file header: ', e)
         sys.exit(5)
-
-
-# Remove extra files unnecessary for SOCAT import.
-#
-
 
-
-
+# Check if the folder exists and create it if not
+#
+<<<<<<< HEAD
+def make_folder(folder):
+    if (folder != '') & (folder != '.'):
+        if not os.path.isdir(folder):
+            try:
+                os.makedirs(folder)
+            except OSError as e:
+                print("Error: could not create the folder:", e)
+                sys.exit(3)
+
+# Move the SOCAT tsv and xml files to the output folder.
+=======
 def repack_preclean(tmp_folder):
     try:
         # Standardize paths to prevent slash mismatches
@@ -304,33 +307,25 @@ def repack_preclean(tmp_folder):
         sys.exit(5)
 
 # Recompress the data into a zip ready for upload.
+>>>>>>> origin/master
 #
-def repack_zip(tmp_folder, out_folder, zip_fname):
-    print('Removing unnecessary files', end='...')
-    repack_preclean(tmp_folder)
-    print('Done')
-    print('Compressing SOCAT files', end='...')
+def move_to_output(tsv_file, xml_file, out_folder):
+    # Ensure the output folder exists before moving files into it
+    make_folder(out_folder)
+    print('Copying SOCAT files to output folder', end='...')
     try:
-        with zipfile.ZipFile(os.path.join(out_folder, zip_fname), 'w') as zipf:
-            for root, dirs, files in os.walk(tmp_folder):
-                for file in files:
-                    zipf.write(os.path.join(root, file), 
-                               os.path.relpath(os.path.join(root, file), 
-                                               os.path.join(tmp_folder, '..')))
+        shutil.move(tsv_file, out_folder)
+        shutil.move(xml_file, out_folder)
     except OSError as e:
-        print('Error: could not create the output zip file: ', e)
+        print('Error: could not copy files to the output folder: ', e)
         sys.exit(5)
     print('Done')
 
 # Clean up the temporary files to avoid unnecessary fill up of hard-drive
+#
 def clean_tmp(tmp_folder):
     try:
-        for root, dirs, files in os.walk(tmp_folder, topdown=False):
-            for name in files:
-                os.remove(os.path.join(root, name))
-            for name in dirs:
-                os.rmdir(os.path.join(root, name))
-        os.rmdir(tmp_folder)
+        shutil.rmtree(tmp_folder)
     except OSError as e:
         print('Warning: could not clean up the temporary files: ', e)
 
@@ -351,13 +346,13 @@ def clean_tmp(tmp_folder):
                                )
     parser.add_argument('-t', '--tmp', '--temp', type = str,
                         help = "(optional) Temp folder path.\n"
-                               "Defines the folder used to temporarily store all downloaded data before repacking it into a zip file."
-                               "By default this value is set to '/tmp/'"
+                               "Defines the folder used to temporarily store all downloaded data before processing."
+                               "By default this value is set to the system temporary folder."
                                )
     parser.add_argument('-o', '--output', type = str,
                         help = "(optional) Output folder path.\n"
-                               "Defines the folder where the final zip file containing the data ready for SOCAT import will be stored."
-                               "By default this value is set to the current working folder"
+                               "Defines the folder where the final data and xml files containing the data ready for SOCAT import will be stored."
+                               "By default this value is set to the dataset name"
                                )
     parser.add_argument('-d', '--data', type = str,
                         help = "(optional) Data folder path.\n"
@@ -388,7 +383,6 @@ def clean_tmp(tmp_folder):
             dataset_name = dataset_name[:-4]
         else:
             zip_fname = dataset_name + '.zip'
-    # xml_fname = '1199_template.xml'
         xml_fname = args.SOCAT
         if '.xml' not in xml_fname:
             xml_fname = xml_fname + '.xml'
@@ -397,11 +391,11 @@ def clean_tmp(tmp_folder):
     if args.tmp != None:
         tmp_path = args.tmp
     else:
-        tmp_path = '/tmp/'
+        tmp_path = tempfile.gettempdir()
     if args.output != None:
         out_folder = args.output
     else:
-        out_folder = ''
+        out_folder = dataset_name
     if args.data != None:
         base_path = args.data
     else:
@@ -412,12 +406,12 @@ def clean_tmp(tmp_folder):
     tmp_folder = os.path.join(tmp_path, dataset_name)
     make_folder(tmp_folder)
     data_file = os.path.join(tmp_folder, zip_fname)
-    make_folder(tmp_folder)
-    data_fname = os.path.join(
+    tsv_fname = dataset_name + '.tsv'
+    tsv_file = os.path.join(
         tmp_folder,
         'dataset',
         'SOCAT',
-        dataset_name + '.tsv'
+        tsv_fname
         )
 
     # Retrieve the dataset from QuinCe
@@ -435,10 +429,10 @@ def clean_tmp(tmp_folder):
     # Fill up the missing metadata in the xml file
     xml_data = populate_xml(xml_data, metadata)
     # write the xml file to disk
-    save_xml(xml_data, tmp_folder)
+    out_xml_file = save_xml(xml_data, tmp_folder)
     # write the header to the tsv file
-    write_header(data_file, xml_data)
-    # Recompress the data with updated metadata into a file for import into SOCAT
-    repack_zip(tmp_folder, out_folder, zip_fname)
+    write_header(tsv_file, xml_data)
+    # Move the tsv and xml files to the output folder for import into SOCAT
+    move_to_output(tsv_file, out_xml_file, out_folder)
     # Clean up after yourself :)
     clean_tmp(tmp_folder)