diff --git a/README.md b/README.md index 5d46a4e..efe8af2 100755 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A Python script to automate the preparation of oceanographic datasets for submis 2. **[ICOS Carbon Portal](https://www.icos-cp.eu/)** – data repository providing persistent identifiers (PIDs/DOIs) 3. **OADS XML template** – a dataset-specific metadata file pre-filled by the user (see [XML template](#xml-template)) -The script downloads a dataset from QuinCE, enriches it with metadata from the Carbon Portal, populates a SOCAT-compliant OADS XML metadata file, prepends the required SOCAT header to the TSV data file, and repackages everything into a zip archive ready for SOCAT upload. +The script downloads a dataset from QuinCE, enriches it with metadata from the Carbon Portal, populates a SOCAT-compliant OADS XML metadata file, prepends the required SOCAT header to the TSV data file, and copies the resulting `.tsv` and `.xml` files to an output folder ready for SOCAT upload. The script runs on both Linux and Windows. ## Workflow @@ -30,7 +30,7 @@ QuinCE API ──► Download dataset (zip) Write SOCAT header to .tsv │ ▼ - Repack ──► output zip (SOCAT-ready) + Move .tsv + .xml ──► output folder (SOCAT-ready) │ ▼ Clean up temporary files @@ -41,7 +41,7 @@ QuinCE API ──► Download dataset (zip) - Python 3.x - [`icoscp_core`](https://pypi.org/project/icoscp-core/) – ICOS Carbon Portal client - [`requests`](https://pypi.org/project/requests/) – HTTP library for the QuinCE API -- `os`, `sys`, `json`, `zipfile`, `glob`, `subprocess`, `xml.etree.ElementTree`, `datetime` – Python standard library (included with Python) +- `os`, `sys`, `json`, `zipfile`, `tempfile`, `shutil`, `xml.etree.ElementTree`, `datetime` – Python standard library (included with Python) Install third-party dependencies: @@ -103,8 +103,8 @@ python prep_SOCAT.py -n -S [options] | Flag | Default | Description | |---|---|---| -| `-t`, `--tmp` | `/tmp/` | Temporary working directory | -| `-o`, `--output` | Current directory | Output folder for the final zip | +| `-t`, `--tmp` | System temp folder | Temporary working directory | +| `-o`, `--output` | Dataset name | Output folder for the final `.tsv` and `.xml` files | | `-d`, `--data` | `Data/` | Folder containing `credentials.json` and the XML template | | `-v`, `--version` | — | Print version and exit | @@ -119,7 +119,7 @@ This will: 2. Read `Data/1199_template.xml` and fill in the `TK` fields 3. Query the Carbon Portal for the corresponding PID/URI 4. Prepend the SOCAT header to the TSV file -5. Write `output/119920230901.zip` ready for SOCAT upload +5. Write `output/119920230901.tsv` and `output/119920230901.xml` ready for SOCAT upload ## Exit codes @@ -134,7 +134,12 @@ This will: ## Version -Current version: **1.0** +Current version: **1.1** + +### Changelog + +- **1.1** – Output the SOCAT-ready `.tsv` and `.xml` files directly to the output folder instead of repacking into a zip archive. Added Windows compatibility (system-independent temp folder and path handling). Various bug fixes. +- **1.0** – Initial release. ## Author diff --git a/prep_SOCAT.py b/prep_SOCAT.py index 9333940..ca6da8b 100644 --- a/prep_SOCAT.py +++ b/prep_SOCAT.py @@ -9,7 +9,7 @@ # Major version number. __major__ = '1' # Minor version number. -__minor__ = '0' +__minor__ = '1' # Script version. __version__ = __major__ + '.' + __minor__ @@ -20,30 +20,17 @@ import xml.etree.ElementTree as ET import json import zipfile -import glob -import subprocess +import tempfile from datetime import datetime as dt from icoscp_core.icos import meta import shutil -# Check if the folder exists and create it if not -# -def make_folder(folder): - if (folder != '') & (folder != '.'): - if not os.path.isdir(folder): - try: - os.makedirs(folder) - except OSError as e: - print("Error: could not create the folder:", e) - sys.exit(3) - # Run a QuinCE API call to download the dataset corresponding to the given filename # def QuinCe_API(dataset_name, data_file): import requests from requests.auth import HTTPBasicAuth - import json - + # Load configuration for the QuinCe instance try: with open(os.path.join(base_path, 'credentials.json')) as f: @@ -71,7 +58,7 @@ def QuinCe_API(dataset_name, data_file): if fsize >= 1024: if fsize >= 1024*1024: size_str = 'MB' - fsize = fsize/1024*1024 + fsize = fsize/(1024*1024) else: size_str = 'kB' fsize = fsize/1024 @@ -114,16 +101,14 @@ def unzip_data(data_file): except OSError as e: print('Warning: the zip file could not be removed: ', e) -# Load the metadata contained in the manifest.json file as well as the data in -# the tsv file. -# The original data zipfile is unpacked into the temp folder in order to load -# the .tsv file. +# Unpack the archive exported from QuinCe into the temp folder (which also makes +# the .tsv data file available) and load the metadata from the manifest.json file. # def import_metadata(tmp_folder, data_file): unzip_data(data_file) - metadata_fname = os.path.join(tmp_folder, 'manifest.json') + manifest_file = os.path.join(tmp_folder, 'manifest.json') try: - with open(metadata_fname, 'r') as file: + with open(manifest_file, 'r') as file: metadata = json.load(file) except (OSError, json.JSONDecodeError) as e: print('Error: could not load "manifest.json" metadata: ', e) @@ -216,11 +201,21 @@ def populate_xml(xml_data, metadata): child = populate_xml(child, metadata) return xml_data +# Get the default namespace from the imported XML +def get_namespace_uri(el): + # ChatGPT gave me this. I hope it works - Steve + if el.tag.startswith("{"): + return el.tag.split("}", 1)[0][1:] + return None -# Save the completed xml file into the data folder +# Save the completed xml file into the temporary folder # def save_xml(xml_data, tmp_folder): - fname = os.path.join(tmp_folder, tmp_folder.split('/')[-1] + '.xml') + fname = os.path.join(tmp_folder, os.path.basename(tmp_folder) + '.xml') + + # We want to use the default (empty) namespace on export + ET.register_namespace("", get_namespace_uri(xml_data)) + xml_str = ET.tostring(xml_data).decode() try: with open(fname, 'w') as xml_file: @@ -228,6 +223,7 @@ def save_xml(xml_data, tmp_folder): except OSError as e: print('Error: could not save the metadata to the xml file: ', e) sys.exit(5) + return fname # Find the value corresponding to a keys tree in the given xml etree. # If several values are found, a semicolon separated string is returned @@ -248,7 +244,7 @@ def find_leaf(xml_data, keys, multiple_entries=False): # Add the "SOCAT" header to the datafile # -def write_header(data_file, xml_data): +def write_header(tsv_file, xml_data): # Prepare the header from the xml metadata expocode = find_leaf(xml_data, 'expocodes/expocode') vessel = find_leaf(xml_data, 'platforms/platform/name') @@ -258,22 +254,29 @@ def write_header(data_file, xml_data): f"Vessel type: {vtype} \n") try: # Read the content of the existing file - with open(data_fname, "r") as f: + with open(tsv_file, "r") as f: content = f.read() # Write header + existing content - with open(data_fname, "w") as f: + with open(tsv_file, "w") as f: f.write(h + content) except OSError as e: print('Error: could not save the data file header: ', e) sys.exit(5) - - -# Remove extra files unnecessary for SOCAT import. -# - - - +# Check if the folder exists and create it if not +# +<<<<<<< HEAD +def make_folder(folder): + if (folder != '') & (folder != '.'): + if not os.path.isdir(folder): + try: + os.makedirs(folder) + except OSError as e: + print("Error: could not create the folder:", e) + sys.exit(3) + +# Move the SOCAT tsv and xml files to the output folder. +======= def repack_preclean(tmp_folder): try: # Standardize paths to prevent slash mismatches @@ -304,33 +307,25 @@ def repack_preclean(tmp_folder): sys.exit(5) # Recompress the data into a zip ready for upload. +>>>>>>> origin/master # -def repack_zip(tmp_folder, out_folder, zip_fname): - print('Removing unnecessary files', end='...') - repack_preclean(tmp_folder) - print('Done') - print('Compressing SOCAT files', end='...') +def move_to_output(tsv_file, xml_file, out_folder): + # Ensure the output folder exists before moving files into it + make_folder(out_folder) + print('Copying SOCAT files to output folder', end='...') try: - with zipfile.ZipFile(os.path.join(out_folder, zip_fname), 'w') as zipf: - for root, dirs, files in os.walk(tmp_folder): - for file in files: - zipf.write(os.path.join(root, file), - os.path.relpath(os.path.join(root, file), - os.path.join(tmp_folder, '..'))) + shutil.move(tsv_file, out_folder) + shutil.move(xml_file, out_folder) except OSError as e: - print('Error: could not create the output zip file: ', e) + print('Error: could not copy files to the output folder: ', e) sys.exit(5) print('Done') # Clean up the temporary files to avoid unnecessary fill up of hard-drive +# def clean_tmp(tmp_folder): try: - for root, dirs, files in os.walk(tmp_folder, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(tmp_folder) + shutil.rmtree(tmp_folder) except OSError as e: print('Warning: could not clean up the temporary files: ', e) @@ -351,13 +346,13 @@ def clean_tmp(tmp_folder): ) parser.add_argument('-t', '--tmp', '--temp', type = str, help = "(optional) Temp folder path.\n" - "Defines the folder used to temporarily store all downloaded data before repacking it into a zip file." - "By default this value is set to '/tmp/'" + "Defines the folder used to temporarily store all downloaded data before processing." + "By default this value is set to the system temporary folder." ) parser.add_argument('-o', '--output', type = str, help = "(optional) Output folder path.\n" - "Defines the folder where the final zip file containing the data ready for SOCAT import will be stored." - "By default this value is set to the current working folder" + "Defines the folder where the final data and xml files containing the data ready for SOCAT import will be stored." + "By default this value is set to the dataset name" ) parser.add_argument('-d', '--data', type = str, help = "(optional) Data folder path.\n" @@ -388,7 +383,6 @@ def clean_tmp(tmp_folder): dataset_name = dataset_name[:-4] else: zip_fname = dataset_name + '.zip' - # xml_fname = '1199_template.xml' xml_fname = args.SOCAT if '.xml' not in xml_fname: xml_fname = xml_fname + '.xml' @@ -397,11 +391,11 @@ def clean_tmp(tmp_folder): if args.tmp != None: tmp_path = args.tmp else: - tmp_path = '/tmp/' + tmp_path = tempfile.gettempdir() if args.output != None: out_folder = args.output else: - out_folder = '' + out_folder = dataset_name if args.data != None: base_path = args.data else: @@ -412,12 +406,12 @@ def clean_tmp(tmp_folder): tmp_folder = os.path.join(tmp_path, dataset_name) make_folder(tmp_folder) data_file = os.path.join(tmp_folder, zip_fname) - make_folder(tmp_folder) - data_fname = os.path.join( + tsv_fname = dataset_name + '.tsv' + tsv_file = os.path.join( tmp_folder, 'dataset', 'SOCAT', - dataset_name + '.tsv' + tsv_fname ) # Retrieve the dataset from QuinCe @@ -435,10 +429,10 @@ def clean_tmp(tmp_folder): # Fill up the missing metadata in the xml file xml_data = populate_xml(xml_data, metadata) # write the xml file to disk - save_xml(xml_data, tmp_folder) + out_xml_file = save_xml(xml_data, tmp_folder) # write the header to the tsv file - write_header(data_file, xml_data) - # Recompress the data with updated metadata into a file for import into SOCAT - repack_zip(tmp_folder, out_folder, zip_fname) + write_header(tsv_file, xml_data) + # Move the tsv and xml files to the output folder for import into SOCAT + move_to_output(tsv_file, out_xml_file, out_folder) # Clean up after yourself :) clean_tmp(tmp_folder)