From 98e463f5de063b24aac2d533b5f748e241576bdd Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Thu, 27 Aug 2026 14:58:31 -0500 Subject: [PATCH 1/4] ENH/ADD: Revamped ICARTT class, plus write_icartt --- README.rst | 1 - act/io/__init__.py | 2 +- act/io/icartt.py | 926 ++++++++++++++++++++++++++++++++++------ docs/source/index.rst | 1 - tests/io/test_icartt.py | 306 ++++++++++++- 5 files changed, 1104 insertions(+), 132 deletions(-) diff --git a/README.rst b/README.rst index fffa71f740..b73bc32205 100644 --- a/README.rst +++ b/README.rst @@ -91,7 +91,6 @@ Optional Dependencies * `Cartopy `_ Mapping and geoplots * `Py-ART `_ Reading radar files, plotting and corrections * `scikit-posthocs `_ Using interquartile range or generalized Extreme Studentized Deviate quality control tests -* `icartt `_ icartt is an ICARTT file format reader and writer for Python * `PySP2 `_ PySP2 is a python package for reading and processing Single Particle Soot Photometer (SP2) datasets. * `MoviePy `_ MoviePy is a python package for creating movies from images diff --git a/act/io/__init__.py b/act/io/__init__.py index 2889b7edad..6f4305d13c 100644 --- a/act/io/__init__.py +++ b/act/io/__init__.py @@ -30,7 +30,7 @@ ], 'ameriflux': ['convert_to_ameriflux', 'read_ameriflux'], 'text': ['read_csv'], - 'icartt': ['read_icartt'], + 'icartt': ['Icartt', 'read_icartt', 'write_icartt'], 'mpl': ['proc_sigma_mplv5_read', 'read_sigma_mplv5'], 'neon': ['read_neon_csv'], 'noaagml': [ diff --git a/act/io/icartt.py b/act/io/icartt.py index d3f33020da..9e5d316b2d 100644 --- a/act/io/icartt.py +++ b/act/io/icartt.py @@ -2,25 +2,784 @@ Modules for Reading/Writing the International Consortium for Atmospheric Research on Transport and Transformation (ICARTT) file format standards V2.0 +This module implements the ICARTT FFI 1001 format directly, so no third-party +ICARTT library is required. + References: ICARTT V2.0 Standards/Conventions: - https://www.earthdata.nasa.gov/s3fs-public/imported/ESDS-RFC-029v2.pdf """ +import ast +import re +import warnings +from collections import OrderedDict +from datetime import datetime +from pathlib import Path + +import numpy as np import xarray as xr -try: - import icartt +# Deprecated. ICARTT support is built in now, so this is always True. Retained so +# existing ``skipif`` guards and downstream references keep working. +_ICARTT_AVAILABLE = True + +#: Field delimiter for the ICARTT format (ESDS-RFC-029v2 section 2.3.2). +DEFAULT_FIELD_DELIM = ',' + +#: Numeric format used when writing data records. +DEFAULT_NUM_FORMAT = '%.10g' + +#: Scale factor and missing value assumed for the independent variable, which +#: carries neither in the header (ESDS-RFC-029v2 section 2.3.2.12). +DEFAULT_SCALE_FACTOR = 1.0 +DEFAULT_MISSING_VALUE = -9999.0 + +#: Required normal-comment keywords, in the order given by ESDS-RFC-029v2 Table 1. +REQUIRED_KEYWORDS = ( + 'PI_CONTACT_INFO', + 'PLATFORM', + 'LOCATION', + 'ASSOCIATED_DATA', + 'INSTRUMENT_INFO', + 'DATA_INFO', + 'UNCERTAINTY', + 'ULOD_FLAG', + 'ULOD_VALUE', + 'LLOD_FLAG', + 'LLOD_VALUE', + 'DM_CONTACT_INFO', + 'PROJECT_INFO', + 'STIPULATIONS_ON_USE', + 'OTHER_COMMENTS', + 'REVISION', +) - _ICARTT_AVAILABLE = True - _format = icartt.Formats.FFI1001 -except ImportError: - _ICARTT_AVAILABLE = False - _format = None +# Revision keywords are the current and all previous revision identifiers, e.g. +# "R0", "RA", "R12" (ESDS-RFC-029v2 Table 1, row 17). +_REVISION_RE = re.compile(r'^R[A-Za-z0-9]{1,2}$') -def read_icartt(filename, format=_format, return_None=False, **kwargs): +class IcarttVariable: + """ + A single ICARTT variable description. + + Parameters + ---------- + shortname : str + Variable short name, used as the data column header. + units : str + Variable units, or 'none' if unitless. + standardname : str, optional + Variable standard name from the controlled list. + longname : str, optional + Free-form descriptive name. + scale : str or float, optional + Scale factor for the variable. + miss : str or float, optional + Missing data flag for the variable. + + """ + + __slots__ = ('shortname', 'units', 'standardname', 'longname', 'scale', 'miss') + + def __init__( + self, + shortname, + units, + standardname=None, + longname=None, + scale=DEFAULT_SCALE_FACTOR, + miss=DEFAULT_MISSING_VALUE, + ): + self.shortname = shortname + self.units = units + self.standardname = standardname + self.longname = longname + self.scale = scale + self.miss = miss + + @classmethod + def from_desc(cls, parts, **kwargs): + """ + Build a variable from a split header description line. + + Per ESDS-RFC-029v2 section 2.3.2.13 the line is + ``shortname, units, standardname, [longname]``. The long name may itself + contain commas, so any trailing fields are rejoined into it. + + """ + parts = [p.strip() for p in parts] + shortname = parts[0] if parts else '' + units = parts[1] if len(parts) > 1 else '' + standardname = parts[2] if len(parts) > 2 else None + longname = DEFAULT_FIELD_DELIM.join(parts[3:]) if len(parts) > 3 else None + return cls(shortname, units, standardname, longname, **kwargs) + + def desc(self, delimiter=DEFAULT_FIELD_DELIM): + """Variable description string as it appears in an ICARTT file.""" + out = [str(self.shortname), str(self.units)] + if self.standardname is not None: + out.append(str(self.standardname)) + if self.longname is not None: + out.append(str(self.longname)) + return delimiter.join(out) + + def __str__(self): + return self.desc() + + def __repr__(self): + return f'IcarttVariable({self.shortname!r}, {self.units!r})' + + +class Icartt: + """ + + Container for an ICARTT FFI 1001 file: the full header model plus the data + records. Reads and writes the format described by ESDS-RFC-029v2. + + Attributes are named after the fields in the standard, so the header can be + inspected and edited directly before writing. + + Examples + -------- + .. code-block :: python + + from act.io.icartt import Icartt + + ict = Icartt.from_file('AAFNAV_COR_20181104_R0.ict') + print(ict.NV, ict.keywords['PLATFORM']) + ds = ict.to_xarray() + + """ + + def __init__(self): + # Line 1 - file format information. + self.FFI = 1001 + self.version = None + # Number of header lines declared by the file, kept for validation only. + # The authoritative value is the computed ``NLHEAD`` property. + self.declared_nlhead = None + + # Lines 2-5 - originator, affiliation, data source, mission. + self.ONAME = '' + self.ORG = '' + self.SNAME = '' + self.MNAME = '' + + # Line 6 - file volume number, total number of file volumes. + self.IVOL = 1 + self.VVOL = 1 + + # Line 7 - collection and revision dates as (yyyy, mm, dd) tuples. + self.DATE = (1970, 1, 1) + self.RDATE = (1970, 1, 1) + + # Line 8 - data interval code(s). + self.DX = [1.0] + + # Line 9 - independent variable definition. + self.XNAME = None + + # Lines 10 to 12+NV - dependent variable definitions. + self.VNAME = [] + + # Special comments. + self.SCOM = [] + + # Normal comments, split into the three parts of section 2.3.2.17. + self.freeform = [] + self.keywords = OrderedDict((k, '') for k in REQUIRED_KEYWORDS) + self.shortnames = [] + + # Data records, keyed by variable short name. + self.data = {} + + # Source or destination path. + self.name = '' + + # ------------------------------------------------------------------ + # Derived header fields + # ------------------------------------------------------------------ + + @property + def NV(self): + """Number of dependent variables (header line 10).""" + return len(self.VNAME) + + @property + def VSCAL(self): + """Scale factors, one per dependent variable (header line 11).""" + return [v.scale for v in self.VNAME] + + @property + def VMISS(self): + """Missing data flags, one per dependent variable (header line 12).""" + return [v.miss for v in self.VNAME] + + @property + def NSCOML(self): + """Number of special comment lines.""" + return len(self.SCOM) + + @property + def NCOM(self): + """ + Normal comment lines, rebuilt from the parsed parts. + + Ordered as free-form text, then the keyword block, then the variable + short name list, which must always be the last line. + + """ + lines = list(self.freeform) + for key, value in self.keywords.items(): + body = value if value else 'N/A' + lines.extend(f'{key}: {body}'.split('\n')) + lines.append(DEFAULT_FIELD_DELIM.join(self.shortnames)) + return lines + + @property + def NNCOML(self): + """Number of normal comment lines.""" + return len(self.NCOM) + + @property + def NLHEAD(self): + """ + Number of header lines. + + Computed rather than stored, per ESDS-RFC-029v2 section 2.3.2.1: 14 + fixed lines plus one line per dependent variable, special comment and + normal comment. + + """ + return 14 + self.NV + self.NSCOML + self.NNCOML + + @property + def variables(self): + """All variables, independent first, keyed by short name.""" + out = OrderedDict() + if self.XNAME is not None: + out[self.XNAME.shortname] = self.XNAME + for var in self.VNAME: + out[var.shortname] = var + return out + + @property + def times(self): + """ + Time steps of the data as a ``numpy.datetime64[ns]`` array. + + The independent variable is seconds since UTC midnight of the collection + date (ESDS-RFC-029v2 section 2.3.2.9). + + """ + ref = np.datetime64(datetime(*self.DATE), 'ns') + values = np.asarray(self.data[self.XNAME.shortname], dtype=np.float64) + return ref + (values * 10**9).astype('timedelta64[ns]') + + # ------------------------------------------------------------------ + # Reading + # ------------------------------------------------------------------ + + @classmethod + def from_file(cls, filename, delimiter=DEFAULT_FIELD_DELIM): + """ + Read an ICARTT FFI 1001 file. + + Parameters + ---------- + filename : str or pathlib.Path + Path to the file to read. + delimiter : str, optional + Field delimiter. The standard mandates a comma. + + Returns + ------- + ict : Icartt + + """ + obj = cls() + obj.name = str(filename) + with open(filename, encoding='utf-8', errors='replace') as fh: + obj._read_header(fh, delimiter) + obj._read_data(fh, delimiter) + return obj + + def _read_header(self, fh, delimiter): + """Read the header, following the line order of section 2.3.2.""" + + def readline(split=True): + line = fh.readline() + if line == '': + raise ValueError( + f'Unexpected end of file while reading the ICARTT header of {self.name}' + ) + line = line.rstrip('\r\n') + if split: + return [part.strip() for part in line.split(delimiter)] + return line + + # Line 1 - number of header lines, file format index, optional version. + first = readline() + try: + self.declared_nlhead = int(first[0]) + self.FFI = int(first[1]) + except (IndexError, ValueError) as err: + raise ValueError( + f'Could not parse the ICARTT file format line of {self.name}: {first!r}' + ) from err + if len(first) > 2 and first[2]: + self.version = first[2] + + if self.FFI != 1001: + raise NotImplementedError( + f'ACT supports the ICARTT FFI 1001 format only, this file declares {self.FFI}' + ) + + # Lines 2-5. + self.ONAME = readline(False) + self.ORG = readline(False) + self.SNAME = readline(False) + self.MNAME = readline(False) + + # Line 6 - file volume number, total number of file volumes. + volumes = readline() + self.IVOL = int(volumes[0]) + self.VVOL = int(volumes[1]) + + # Line 7 - collection date, revision date. + dates = readline() + if len(dates) < 6: + raise ValueError( + f'ICARTT date line of {self.name} needs six fields, found {len(dates)}' + ) + self.DATE = tuple(int(x) for x in dates[:3]) + self.RDATE = tuple(int(x) for x in dates[3:6]) + + # Line 8 - data interval code. + self.DX = [float(x) for x in readline()] + + # Line 9 - independent variable definition. + self.XNAME = IcarttVariable.from_desc(readline()) + + # Line 10 - number of dependent variables. + nvar = int(readline()[0]) + + # Lines 11-12 - scale factors and missing value flags. + vscal = readline() + vmiss = readline() + for label, values in (('scale factor', vscal), ('missing value', vmiss)): + if len(values) != nvar: + raise ValueError( + f'ICARTT {label} line of {self.name} has {len(values)} entries ' + f'but the file declares {nvar} dependent variables' + ) + + # Lines 13 to 12+NV - dependent variable definitions. + self.VNAME = [ + IcarttVariable.from_desc(readline(), scale=vscal[idx], miss=vmiss[idx]) + for idx in range(nvar) + ] + + # Special comments. + nscoml = int(readline()[0]) + self.SCOM = [readline(False) for _ in range(nscoml)] + + # Normal comments. + nncoml = int(readline()[0]) + self._ingest_normal_comments([readline(False) for _ in range(nncoml)]) + + # Validate against the counts the file itself declared. The NLHEAD + # property is the canonical count for writing, which can legitimately + # differ here when a required keyword was absent and gets restored. + parsed_nlhead = 14 + nvar + nscoml + nncoml + if self.declared_nlhead != parsed_nlhead: + warnings.warn( + f'ICARTT file {self.name} declares {self.declared_nlhead} header lines ' + f'but {parsed_nlhead} were parsed', + stacklevel=2, + ) + + def _ingest_normal_comments(self, raw): + """ + Split the normal comments into free-form text, keywords and short names. + + Follows ESDS-RFC-029v2 section 2.3.2.17: free-form text runs until the + first required keyword, keyword values continue until the next keyword + line, and the final line is always the variable short name list. + + """ + raw = list(raw) + if not raw: + raise ValueError( + f'ICARTT file {self.name} has an empty normal comments section, but the ' + 'variable short name line is required' + ) + + # The last line is always the comma separated list of short names. + self.shortnames = [name.strip() for name in raw.pop().split(DEFAULT_FIELD_DELIM)] + + buffers = OrderedDict((key, []) for key in REQUIRED_KEYWORDS) + self.freeform = [] + current = None + + for line in raw: + keyword = None + # Keywords start the line with no leading whitespace and are followed + # by a colon. The space before the colon seen in some revision lines + # is tolerated. + if ':' in line and not line[:1].isspace(): + head = line.split(':', 1)[0].rstrip() + if head in buffers or _REVISION_RE.match(head): + keyword = head + + if keyword is not None: + current = keyword + buffers.setdefault(current, []) + buffers[current].append(line.split(':', 1)[1].strip()) + elif current is None: + self.freeform.append(line) + else: + # Continuation of the previous keyword's value. + buffers[current].append(line.strip()) + + missing = [key for key in REQUIRED_KEYWORDS if not buffers[key]] + if missing: + warnings.warn( + f'ICARTT file {self.name} is missing required normal comment ' + f"keywords: {', '.join(missing)}", + stacklevel=3, + ) + + self.keywords = OrderedDict((key, '\n'.join(val)) for key, val in buffers.items()) + + def _read_data(self, fh, delimiter): + """Read the data records into ``self.data``, missing values as NaN.""" + names = list(self.variables) + missing = {name: var.miss for name, var in self.variables.items()} + + with warnings.catch_warnings(): + # genfromtxt warns on an empty file; an empty dataset is legal here. + warnings.simplefilter('ignore') + records = np.genfromtxt( + fh, + names=names, + dtype=[(name, np.float64) for name in names], + missing_values=missing, + usemask=True, + delimiter=delimiter, + deletechars='', + ).filled(fill_value=np.nan) + + self.data = {name: np.atleast_1d(records[name]) for name in names} + + # ------------------------------------------------------------------ + # Conversion + # ------------------------------------------------------------------ + + def _keyword(self, key): + """Keyword value, with the standard's 'N/A' stand-in for an empty one.""" + value = self.keywords.get(key, '').strip() + return value if value else 'N/A' + + def _revision_comments(self): + """Comments for the revision named by the REVISION keyword.""" + revision = self.keywords.get('REVISION', '').strip() + if revision in self.keywords: + return self.keywords[revision].strip() + for key, value in self.keywords.items(): + if key not in REQUIRED_KEYWORDS and _REVISION_RE.match(key): + return value.strip() + return 'N/A' + + def _per_variable_values(self, key): + """ + Map a keyword holding one entry per dependent variable onto short names. + + Sized against NV, not the total variable count: the independent variable + has no uncertainty or limit of detection entry (sections 2.3.2.12, + 2.1.4.3). Returns an empty mapping when the counts do not line up, which + means the file did not supply per-variable values. + + """ + raw = self.keywords.get(key, '').strip() + if not raw: + return {} + parts = [part.strip() for part in raw.split(DEFAULT_FIELD_DELIM)] + if len(parts) != self.NV: + return {} + return {var.shortname: value for var, value in zip(self.VNAME, parts)} + + def _per_variable_flags(self, key): + """ + Map a limit of detection flag keyword onto short names. + + Section 2.1.4.3 allows either a single flag for the whole file or one per + dependent variable. + + """ + raw = self.keywords.get(key, '').strip() + if not raw: + return {} + parts = [part.strip() for part in raw.split(DEFAULT_FIELD_DELIM)] + if len(parts) == self.NV and self.NV != 1: + return {var.shortname: value for var, value in zip(self.VNAME, parts)} + return {name: raw for name in self.variables} + + def to_xarray(self): + """ + Convert to an `xarray.Dataset` with a ``time`` coordinate. + + Returns + ------- + ds : xarray.Dataset + + """ + times = self.times + + uncertainty = self._per_variable_values('UNCERTAINTY') + ulod_value = self._per_variable_values('ULOD_VALUE') + llod_value = self._per_variable_values('LLOD_VALUE') + ulod_flag = self._per_variable_flags('ULOD_FLAG') + llod_flag = self._per_variable_flags('LLOD_FLAG') + + ds = xr.Dataset() + for name, var in self.variables.items(): + # Short name for a quality flag is standardised on read. + out_name = 'quality_flag' if name == 'qc_flag' else name + da = xr.DataArray( + self.data[name], + coords=dict(time=times), + name=out_name, + dims=['time'], + ) + da.attrs['uncertainty'] = uncertainty.get(name, 'N/A') + da.attrs['ULOD_Value'] = ulod_value.get(name, 'N/A') + da.attrs['LLOD_Value'] = llod_value.get(name, 'N/A') + da.attrs['units'] = var.units + da.attrs['mvc'] = var.miss + da.attrs['scale_factor'] = var.scale + da.attrs['ULOD_Flag'] = ulod_flag.get(name, 'N/A') + da.attrs['LLOD_Flag'] = llod_flag.get(name, 'N/A') + ds[out_name] = da + + ds.attrs['PI'] = self.ONAME + ds.attrs['PI_Affiliation'] = self.ORG + ds.attrs['Platform'] = self._keyword('PLATFORM') + ds.attrs['Mission'] = self.MNAME + ds.attrs['DateOfCollection'] = str(self.DATE) + ds.attrs['DateOfRevision'] = str(self.RDATE) + ds.attrs['Data_Interval'] = str(self.DX) + ds.attrs['Independent_Var'] = str(self.XNAME) + ds.attrs['Dependent_Var_Num'] = self.NV + ds.attrs['PI_Contact'] = self._keyword('PI_CONTACT_INFO') + ds.attrs['Location'] = self._keyword('LOCATION') + ds.attrs['Associated_Data'] = self._keyword('ASSOCIATED_DATA') + ds.attrs['Instrument_Info'] = self._keyword('INSTRUMENT_INFO') + ds.attrs['Data_Info'] = self._keyword('DATA_INFO') + ds.attrs['DM_Contact'] = self._keyword('DM_CONTACT_INFO') + ds.attrs['Project_Info'] = self._keyword('PROJECT_INFO') + ds.attrs['Stipulations'] = self._keyword('STIPULATIONS_ON_USE') + ds.attrs['Comments'] = self._keyword('OTHER_COMMENTS') + ds.attrs['Revision'] = self._keyword('REVISION') + ds.attrs['Revision_Comments'] = self._revision_comments() + + # Additional ARM metadata. + ds.attrs['_datastream'] = Path(self.name).name.split('_')[0] + + return ds + + @classmethod + def from_xarray(cls, ds, filename=''): + """ + Build an `Icartt` from an `xarray.Dataset` produced by :func:`read_icartt`. + + Reverses the mapping applied by `to_xarray`, including the + ``qc_flag`` to ``quality_flag`` rename. + + Parameters + ---------- + ds : xarray.Dataset + Dataset to convert. + filename : str or pathlib.Path, optional + Name to record on the object. + + Returns + ------- + ict : Icartt + + """ + + def tuple_attr(key, default): + try: + return tuple(ast.literal_eval(str(ds.attrs[key]))) + except (KeyError, ValueError, SyntaxError, TypeError): + return default + + obj = cls() + obj.name = str(filename) + obj.ONAME = str(ds.attrs.get('PI', 'N/A')) + obj.ORG = str(ds.attrs.get('PI_Affiliation', 'N/A')) + obj.SNAME = str(ds.attrs.get('Platform', 'N/A')) + obj.MNAME = str(ds.attrs.get('Mission', 'N/A')) + obj.DATE = tuple_attr('DateOfCollection', (1970, 1, 1)) + obj.RDATE = tuple_attr('DateOfRevision', (1970, 1, 1)) + + try: + obj.DX = [float(x) for x in ast.literal_eval(str(ds.attrs['Data_Interval']))] + except (KeyError, ValueError, SyntaxError, TypeError): + obj.DX = [1.0] + + independent = str(ds.attrs.get('Independent_Var', 'Start_UTC,seconds')) + obj.XNAME = IcarttVariable.from_desc(independent.split(DEFAULT_FIELD_DELIM)) + ivar = obj.XNAME.shortname + + for out_name in ds.data_vars: + name = 'qc_flag' if out_name == 'quality_flag' else str(out_name) + attrs = ds[out_name].attrs + values = np.asarray(ds[out_name].values, dtype=np.float64) + if name == ivar: + obj.XNAME.units = str(attrs.get('units', obj.XNAME.units)) + obj.data[ivar] = values + continue + obj.VNAME.append( + IcarttVariable( + name, + str(attrs.get('units', 'none')), + scale=attrs.get('scale_factor', DEFAULT_SCALE_FACTOR), + miss=attrs.get('mvc', DEFAULT_MISSING_VALUE), + ) + ) + obj.data[name] = values + + if ivar not in obj.data: + # The independent variable was dropped from the Dataset, so rebuild it + # as seconds since UTC midnight of the collection date. + ref = np.datetime64(datetime(*obj.DATE), 'ns') + delta = ds['time'].values.astype('datetime64[ns]') - ref + obj.data[ivar] = delta.astype('timedelta64[ns]').astype(np.float64) / 1e9 + + obj.shortnames = [ivar] + [var.shortname for var in obj.VNAME] + + keyword_attrs = ( + ('PI_CONTACT_INFO', 'PI_Contact'), + ('PLATFORM', 'Platform'), + ('LOCATION', 'Location'), + ('ASSOCIATED_DATA', 'Associated_Data'), + ('INSTRUMENT_INFO', 'Instrument_Info'), + ('DATA_INFO', 'Data_Info'), + ('DM_CONTACT_INFO', 'DM_Contact'), + ('PROJECT_INFO', 'Project_Info'), + ('STIPULATIONS_ON_USE', 'Stipulations'), + ('OTHER_COMMENTS', 'Comments'), + ('REVISION', 'Revision'), + ) + for keyword, attr in keyword_attrs: + obj.keywords[keyword] = str(ds.attrs.get(attr, 'N/A')) + + # Per-variable metadata is reconstructed from the variable attributes when + # every dependent variable carries the same value, matching how the reader + # broadcasts a single file-wide entry. + for keyword, attr in ( + ('UNCERTAINTY', 'uncertainty'), + ('ULOD_FLAG', 'ULOD_Flag'), + ('ULOD_VALUE', 'ULOD_Value'), + ('LLOD_FLAG', 'LLOD_Flag'), + ('LLOD_VALUE', 'LLOD_Value'), + ): + values = [] + for var in obj.VNAME: + out_name = 'quality_flag' if var.shortname == 'qc_flag' else var.shortname + values.append(str(ds[out_name].attrs.get(attr, 'N/A'))) + + if not values: + obj.keywords[keyword] = 'N/A' + elif len(set(values)) == 1: + obj.keywords[keyword] = values[0] + else: + obj.keywords[keyword] = DEFAULT_FIELD_DELIM.join(values) + + revision = obj.keywords['REVISION'].strip() + if _REVISION_RE.match(revision): + obj.keywords[revision] = str(ds.attrs.get('Revision_Comments', 'N/A')) + + return obj + + # ------------------------------------------------------------------ + # Writing + # ------------------------------------------------------------------ + + def write(self, filename=None, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM): + """ + Write the object to an ICARTT FFI 1001 file. + + ``NLHEAD`` is recomputed from the content, so the header count is always + consistent with what is written. + + Parameters + ---------- + filename : str or pathlib.Path, optional + Destination path. Defaults to the object's ``name`` attribute. + fmt : str, optional + Numeric format for the data records. + delimiter : str, optional + Field delimiter. The standard mandates a comma. + + """ + if filename is None: + filename = self.name + if not filename: + raise ValueError('No filename given and the Icartt object has no name set') + if self.XNAME is None: + raise ValueError('Cannot write an Icartt object with no independent variable') + + ivar = self.XNAME.shortname + names = [ivar] + [var.shortname for var in self.VNAME] + for name in names: + if name not in self.data: + raise ValueError(f'No data present for the variable {name!r}') + + # Missing values go back out as the file's own flag rather than NaN. + columns = [np.asarray(self.data[ivar], dtype=np.float64)] + for var in self.VNAME: + column = np.array(self.data[var.shortname], dtype=np.float64, copy=True) + try: + column[np.isnan(column)] = float(var.miss) + except (TypeError, ValueError): + column[np.isnan(column)] = DEFAULT_MISSING_VALUE + columns.append(column) + + header = [f'{self.NLHEAD}{delimiter} {self.FFI}'] + if self.version: + header[0] += f'{delimiter} {self.version}' + header.append(self.ONAME) + header.append(self.ORG) + header.append(self.SNAME) + header.append(self.MNAME) + header.append(f'{self.IVOL}{delimiter} {self.VVOL}') + header.append(delimiter.join(f'{part:d}' for part in (*self.DATE, *self.RDATE))) + header.append(delimiter.join(str(x) for x in self.DX)) + header.append(self.XNAME.desc(delimiter + ' ')) + header.append(str(self.NV)) + header.append(delimiter.join(str(x) for x in self.VSCAL)) + header.append(delimiter.join(str(x) for x in self.VMISS)) + header.extend(var.desc(delimiter + ' ') for var in self.VNAME) + header.append(str(self.NSCOML)) + header.extend(self.SCOM) + header.append(str(self.NNCOML)) + header.extend(self.NCOM) + + with open(filename, 'w', encoding='utf-8', newline='\n') as fh: + fh.write('\n'.join(header)) + fh.write('\n') + np.savetxt(fh, np.column_stack(columns), fmt=fmt, delimiter=delimiter) + + self.name = str(filename) + + +def read_icartt(filename, format=1001, return_None=False, **kwargs): """ Returns `xarray.Dataset` with stored data and metadata from a user-defined @@ -31,139 +790,52 @@ def read_icartt(filename, format=_format, return_None=False, **kwargs): ---------- filename : str Name of file to read. - format : str - ICARTT Format to Read: FFI1001 or FFI2110. + format : int or str + ICARTT format to read. Only FFI 1001 is supported. return_None : bool, optional Catch IOError exception when file not found and return None. Default is False. **kwargs : keywords - keywords to pass on through to icartt.Dataset. + keywords to pass on through to Icartt.from_file. Returns ------- ds : xarray.Dataset (or None) ACT Xarray dataset (or None if no data file(s) found). - - Examples - -------- - This example will load the example sounding data used for unit testing. - - .. code-block :: python - - import act - ds = act.io.icartt.read_icartt(act.tests.sample_files.AAF_SAMPLE_FILE) - print(ds.attrs['_datastream']) - """ - if not _ICARTT_AVAILABLE: - raise ImportError("ICARTT is required to use to read ICARTT files but is not installed") - - ds = None - - # Create an exception tuple to use with try statements. Doing it this way - # so we can add the FileNotFoundError if requested. Can add more error - # handling in the future. - except_tuple = (ValueError,) - if return_None: - except_tuple = except_tuple + (FileNotFoundError, OSError) + if str(format) not in ('1001', 'FFI1001', 'Formats.FFI1001'): + raise NotImplementedError(f'ACT supports the ICARTT FFI 1001 format only, got {format!r}') try: - # Read data file with ICARTT dataset. - ict = icartt.Dataset(filename, format=format, **kwargs) - - except except_tuple as exception: - # If requested return None for File not found error - if type(exception).__name__ == 'FileNotFoundError': + ict = Icartt.from_file(filename, **kwargs) + except (FileNotFoundError, OSError) as exception: + if not return_None: + raise + if isinstance(exception, FileNotFoundError): return None - - # If requested return None for File not found error - if type(exception).__name__ == 'OSError' and exception.args[0] == 'no files to open': + if exception.args and exception.args[0] == 'no files to open': return None + raise - # Define the Uncertainty for each variable. Note it may not be calculated. - # If not calculated, assign 'N/A' to the attribute - uncertainty = ict.normalComments[6].split(':')[1].split(',') - - # Define the Upper and Lower Limit of Detection Flags - ulod_flag = ict.normalComments[7].split(':')[1] - ulod_value = ict.normalComments[8].split(':')[1].split(',') - llod_flag = ict.normalComments[9].split(':')[1] - llod_value = ict.normalComments[10].split(':')[1].split(',') - - # Convert ICARTT Object to Xarray Dataset - ds_container = [] - # Counter for uncertainty/LOD values - counter = 0 - - # Loop over ICART variables, convert to Xarray DataArray, Append. - for key in ict.variables: - # Note time is the only independent variable within ICARTT - # Short name for time must be "Start_UTC" for ICARTT files. - if key != 'Start_UTC': - if key == 'qc_flag': - key2 = 'quality_flag' - else: - key2 = key - da = xr.DataArray(ict.data[key], coords=dict(time=ict.times), name=key2, dims=['time']) - # Assume if Uncertainity does not match the number of variables, - # values were not set within the file. Needs to be string! - if len(uncertainty) != len(ict.variables): - da.attrs['uncertainty'] = 'N/A' - else: - da.attrs['uncertainty'] = uncertainty[counter] + return ict.to_xarray() - # Assume if ULOD does not match the number of variables within the - # the file, ULOD values were not set. - if len(ulod_value) != len(ict.variables): - da.attrs['ULOD_Value'] = 'N/A' - else: - da.attrs['ULOD_Value'] = ulod_value[counter] - # Assume if LLOD does not match the number of variables within the - # the file, LLOD values were not set. - if len(llod_value) != len(ict.variables): - da.attrs['LLOD_Value'] = 'N/A' - else: - da.attrs['LLOD_Value'] = llod_value[counter] - # Define the meta data: - da.attrs['units'] = ict.variables[key].units - da.attrs['mvc'] = ict.variables[key].miss - da.attrs['scale_factor'] = ict.variables[key].scale - da.attrs['ULOD_Flag'] = ulod_flag - da.attrs['LLOD_Flag'] = llod_flag - # Append to ds container - ds_container.append(da.to_dataset(name=key2)) - # up the counter - counter += 1 - - # Concatenate each of the Xarray DataArrays into a single Xarray DataSet - ds = xr.merge(ds_container) - - # Assign ICARTT Meta data to Xarray DataSet - ds.attrs['PI'] = ict.PIName - ds.attrs['PI_Affiliation'] = ict.PIAffiliation - ds.attrs['Platform'] = ict.dataSourceDescription - ds.attrs['Mission'] = ict.missionName - ds.attrs['DateOfCollection'] = ict.dateOfCollection - ds.attrs['DateOfRevision'] = ict.dateOfRevision - ds.attrs['Data_Interval'] = ict.dataIntervalCode - ds.attrs['Independent_Var'] = str(ict.independentVariable) - ds.attrs['Dependent_Var_Num'] = len(ict.dependentVariables) - ds.attrs['PI_Contact'] = ict.normalComments[0].split('\n')[0].split(':')[-1] - ds.attrs['Platform'] = ict.normalComments[1].split(':')[-1] - ds.attrs['Location'] = ict.normalComments[2].split(':')[-1] - ds.attrs['Associated_Data'] = ict.normalComments[3].split(':')[-1] - ds.attrs['Instrument_Info'] = ict.normalComments[4].split(':')[-1] - ds.attrs['Data_Info'] = ict.normalComments[5][11:] - ds.attrs['DM_Contact'] = ict.normalComments[11].split(':')[-1] - ds.attrs['Project_Info'] = ict.normalComments[12].split(':')[-1] - ds.attrs['Stipulations'] = ict.normalComments[13].split(':')[-1] - ds.attrs['Comments'] = ict.normalComments[14].split(':')[-1] - ds.attrs['Revision'] = ict.normalComments[15].split(':')[-1] - ds.attrs['Revision_Comments'] = ict.normalComments[15 + 1].split(':')[-1] - - # Assign Additional ARM meta data to Xarray DatatSet - ds.attrs['_datastream'] = filename.split('/')[-1].split('_')[0] - - # Return Xarray Dataset - return ds +def write_icartt(ds, filename, **kwargs): + """ + + Write an `xarray.Dataset` to an ICARTT FFI 1001 file. + + Intended as the inverse of :func:`read_icartt`, so a Dataset produced by it + round-trips back to a valid ICARTT file. Header metadata is taken from the + Dataset attributes, and anything absent falls back to 'N/A'. + + Parameters + ---------- + ds : xarray.Dataset + Dataset to write. Must have a ``time`` coordinate. + filename : str or pathlib.Path + Destination path. + **kwargs : keywords + keywords to pass on through to Icartt.write, such as ``fmt``. + """ + Icartt.from_xarray(ds, filename=filename).write(filename, **kwargs) diff --git a/docs/source/index.rst b/docs/source/index.rst index adae74905f..8429e90788 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -82,7 +82,6 @@ Optional Dependencies | `Cartopy `_ Mapping and geoplots | `Py-ART `_ Reading radar files, plotting and corrections | `scikit-posthocs `_ Using interquartile range or generalized Extreme Studentized Deviate quality control tests -| `icartt `_ icartt is an ICARTT file format reader and writer for Python Contributing diff --git a/tests/io/test_icartt.py b/tests/io/test_icartt.py index 253dfd0cb8..a8ada357f3 100644 --- a/tests/io/test_icartt.py +++ b/tests/io/test_icartt.py @@ -1,14 +1,316 @@ import numpy as np import pytest +import xarray as xr import act +from act.io.icartt import Icartt, read_icartt, write_icartt + +# A minimal but spec-legal FFI 1001 normal comments section. Includes free-form +# text ahead of the first keyword, a keyword value spanning several lines, one +# entry per dependent variable for the uncertainty and limit of detection +# keywords, and two revisions. Every one of those is allowed by ESDS-RFC-029v2 +# and each broke the previous index-based reader. +SAMPLE_NCOM = [ + 'Free-form note about this file.', + 'A second free-form line.', + 'PI_CONTACT_INFO: Address: Somewhere ; email: jane@example.org', + 'PLATFORM: Test Platform', + 'LOCATION: Somewhere', + 'ASSOCIATED_DATA: N/A', + 'INSTRUMENT_INFO: A thermometer', + 'DATA_INFO: reported at STP', + 'UNCERTAINTY: 0.5, 1.0', + 'ULOD_FLAG: -7777', + 'ULOD_VALUE: 100, 2000', + 'LLOD_FLAG: -8888', + 'LLOD_VALUE: -50, 0', + 'DM_CONTACT_INFO: dm@example.org', + 'PROJECT_INFO: Test project', + 'STIPULATIONS_ON_USE: None', + 'OTHER_COMMENTS: first line of comments', + 'continued without leading space', + ' and a space-indented continuation', + 'REVISION: R1', + 'R1: second revision', + 'R0 : first revision', + 'Start_UTC,temperature,pressure', +] + +SAMPLE_DATA = ['0,20.5,1013.2', '1,-9999,1012.8', '2,21.0,-8888'] + + +def build_ict(tmp_path, name='TEST_20240315_R1.ict', ncom=None, data=None, nlhead=None): + """Write a synthetic FFI 1001 file and return its path.""" + ncom = list(SAMPLE_NCOM if ncom is None else ncom) + data = list(SAMPLE_DATA if data is None else data) + nv, nscoml = 2, 0 + if nlhead is None: + nlhead = 14 + nv + nscoml + len(ncom) + header = [ + f'{nlhead}, 1001', + 'Doe, Jane', + 'Test Org', + 'Test Instrument', + 'TESTMISSION', + '1, 1', + '2024,03,15,2024,03,16', + '1.0', + 'Start_UTC, seconds', + str(nv), + '1, 1', + '-9999, -8888', + 'temperature, degC', + 'pressure, hPa', + str(nscoml), + str(len(ncom)), + ] + path = tmp_path / name + path.write_text('\n'.join(header + ncom + data) + '\n') + return str(path) -@pytest.mark.skipif(not act.io.icartt._ICARTT_AVAILABLE, reason='ICARTT is not installed.') def test_read_icartt(): - result = act.io.icartt.read_icartt(act.tests.EXAMPLE_AAF_ICARTT) + result = read_icartt(act.tests.EXAMPLE_AAF_ICARTT) assert 'pitch' in result assert len(result['time'].values) == 14087 assert result['true_airspeed'].units == 'm/s' assert 'Revision' in result.attrs np.testing.assert_almost_equal(result['static_pressure'].mean(), 708.75, decimal=2) + + +def test_read_icartt_lazy_loader(): + # The public act.io namespace exposes the reader, writer and container. + assert act.io.read_icartt is read_icartt + assert act.io.write_icartt is write_icartt + assert act.io.Icartt is Icartt + + +def test_read_icartt_structure(): + ds = read_icartt(act.tests.EXAMPLE_AAF_ICARTT) + # The independent variable is retained alongside the time coordinate. + assert 'start_time' in ds.data_vars + assert len(ds.data_vars) == 39 + assert ds['time'].dtype == np.dtype('datetime64[ns]') + assert str(ds['time'].values[0]) == '2018-11-04T13:04:36.000000000' + assert str(ds['time'].values[-1]) == '2018-11-04T16:59:22.000000000' + # qc_flag is renamed on read. + assert 'quality_flag' in ds.data_vars + assert 'qc_flag' not in ds.data_vars + # Missing values become NaN rather than the file's -9999 flag. + assert np.isnan(ds['drift'].values).sum() == 1181 + assert np.isnan(ds['vert_wind_speed'].values).sum() == 7778 + + +def test_read_icartt_global_attributes(): + attrs = read_icartt(act.tests.EXAMPLE_AAF_ICARTT).attrs + assert attrs['PI'] == 'ARM Aerial Facility Team' + assert attrs['PI_Affiliation'] == 'ARM PNNL' + assert attrs['Mission'] == 'N/A' + assert attrs['DateOfCollection'] == '(2018, 11, 4)' + assert attrs['DateOfRevision'] == '(2018, 11, 4)' + assert attrs['Data_Interval'] == '[1.0]' + assert attrs['Independent_Var'] == 'start_time,seconds' + assert attrs['Dependent_Var_Num'] == 38 + assert attrs['_datastream'] == 'AAFNAV' + # Keyword values keep everything after the keyword, colons included, and are + # not split on every colon as the previous implementation did. + assert attrs['PI_Contact'] == 'Address: PNNL ; email: armaaf@arm.gov' + assert attrs['Platform'] == 'Department of Energy ARM Aerial Facility Gulfstream' + assert attrs['Revision'] == 'R0' + assert attrs['Comments'].startswith('command_line:aafnaviwg_ingest') + assert attrs['Revision_Comments'].startswith('created by user dsmgr') + # Keywords with no value get the standard's N/A stand-in. + assert attrs['Associated_Data'] == 'N/A' + assert attrs['Instrument_Info'] == 'N/A' + assert attrs['Project_Info'] == 'N/A' + + +def test_read_icartt_variable_attributes(): + ds = read_icartt(act.tests.EXAMPLE_AAF_ICARTT) + attrs = ds['static_pressure'].attrs + assert attrs['units'] == 'hPa' + assert attrs['mvc'] == '-9999' + assert attrs['scale_factor'] == '1' + assert attrs['ULOD_Flag'] == '-7777' + assert attrs['LLOD_Flag'] == '-8888' + # This file supplies no per-variable uncertainty or LOD values. + assert attrs['uncertainty'] == 'N/A' + assert attrs['ULOD_Value'] == 'N/A' + assert attrs['LLOD_Value'] == 'N/A' + + +def test_read_icartt_freeform_comments(tmp_path): + # Free-form text shifts every normal comment line, which silently corrupted + # every attribute under positional lookup. + ds = read_icartt(build_ict(tmp_path)) + ict = Icartt.from_file(build_ict(tmp_path)) + assert ict.freeform == ['Free-form note about this file.', 'A second free-form line.'] + assert ds.attrs['PI_Contact'] == 'Address: Somewhere ; email: jane@example.org' + assert ds.attrs['Location'] == 'Somewhere' + assert ds.attrs['DM_Contact'] == 'dm@example.org' + assert ds.attrs['Stipulations'] == 'None' + + +def test_read_icartt_multiline_keyword(tmp_path): + ds = read_icartt(build_ict(tmp_path)) + assert ds.attrs['Comments'] == ( + 'first line of comments\n' + 'continued without leading space\n' + 'and a space-indented continuation' + ) + + +def test_read_icartt_per_variable_values(tmp_path): + # One entry per dependent variable, matched by name rather than by a counter + # that used to start on the independent variable. + ds = read_icartt(build_ict(tmp_path)) + assert ds['temperature'].attrs['uncertainty'] == '0.5' + assert ds['pressure'].attrs['uncertainty'] == '1.0' + assert ds['temperature'].attrs['ULOD_Value'] == '100' + assert ds['pressure'].attrs['ULOD_Value'] == '2000' + assert ds['temperature'].attrs['LLOD_Value'] == '-50' + assert ds['pressure'].attrs['LLOD_Value'] == '0' + # A single file-wide flag still applies to everything. + assert ds['temperature'].attrs['ULOD_Flag'] == '-7777' + assert ds['pressure'].attrs['LLOD_Flag'] == '-8888' + # The independent variable has no uncertainty or LOD entry. + assert ds['Start_UTC'].attrs['uncertainty'] == 'N/A' + + +def test_read_icartt_missing_values(tmp_path): + # Each dependent variable uses its own missing value flag. + ds = read_icartt(build_ict(tmp_path)) + np.testing.assert_array_equal(ds['temperature'].values, [20.5, np.nan, 21.0]) + np.testing.assert_array_equal(ds['pressure'].values, [1013.2, 1012.8, np.nan]) + + +def test_read_icartt_revisions(tmp_path): + # Revision comments follow the REVISION keyword, not a fixed line offset. + ds = read_icartt(build_ict(tmp_path)) + assert ds.attrs['Revision'] == 'R1' + assert ds.attrs['Revision_Comments'] == 'second revision' + + +def test_read_icartt_times(tmp_path): + ds = read_icartt(build_ict(tmp_path)) + expected = np.array( + ['2024-03-15T00:00:00', '2024-03-15T00:00:01', '2024-03-15T00:00:02'], + dtype='datetime64[ns]', + ) + np.testing.assert_array_equal(ds['time'].values, expected) + + +def test_icartt_nlhead(tmp_path): + ict = Icartt.from_file(build_ict(tmp_path)) + assert ict.NLHEAD == 14 + ict.NV + ict.NSCOML + ict.NNCOML + assert ict.NLHEAD == ict.declared_nlhead + assert ict.NV == 2 + assert ict.NNCOML == len(SAMPLE_NCOM) + + +def test_icartt_nlhead_mismatch_warns(tmp_path): + path = build_ict(tmp_path, nlhead=99) + with pytest.warns(UserWarning, match='declares 99 header lines'): + Icartt.from_file(path) + + +def test_icartt_missing_keyword_warns(tmp_path): + ncom = [line for line in SAMPLE_NCOM if not line.startswith('PROJECT_INFO')] + with pytest.warns(UserWarning, match='PROJECT_INFO'): + read_icartt(build_ict(tmp_path, ncom=ncom)) + + +def test_write_icartt_roundtrip(tmp_path): + ds1 = read_icartt(act.tests.EXAMPLE_AAF_ICARTT) + out = tmp_path / 'AAFNAV_COR_20181104_R0.ict' + write_icartt(ds1, out) + + ds2 = read_icartt(str(out)) + assert list(ds1.data_vars) == list(ds2.data_vars) + np.testing.assert_array_equal(ds1['time'].values, ds2['time'].values) + xr.testing.assert_allclose(ds1, ds2, rtol=1e-9) + assert ds1.attrs == ds2.attrs + for name in ds1.data_vars: + assert ds1[name].attrs == ds2[name].attrs + assert np.isnan(ds1[name].values).sum() == np.isnan(ds2[name].values).sum() + + +def test_write_icartt_nlhead(tmp_path): + # The written header count is recomputed from the content. + ds = read_icartt(build_ict(tmp_path)) + out = tmp_path / 'OUT_20240315_R1.ict' + write_icartt(ds, out) + + declared = int(out.read_text().splitlines()[0].split(',')[0]) + ict = Icartt.from_file(str(out)) + assert declared == 14 + ict.NV + ict.NSCOML + ict.NNCOML + assert declared == ict.NLHEAD + + +def test_write_icartt_roundtrip_synthetic(tmp_path): + ds1 = read_icartt(build_ict(tmp_path)) + out = tmp_path / 'OUT_20240315_R1.ict' + write_icartt(ds1, out) + ds2 = read_icartt(str(out)) + + xr.testing.assert_allclose(ds1, ds2, rtol=1e-9) + assert ds2.attrs['Revision'] == 'R1' + assert ds2.attrs['Revision_Comments'] == 'second revision' + assert ds2['temperature'].attrs['uncertainty'] == '0.5' + assert ds2['pressure'].attrs['uncertainty'] == '1.0' + np.testing.assert_array_equal(ds2['temperature'].values, [20.5, np.nan, 21.0]) + + +def test_read_icartt_missing_file(tmp_path): + missing = str(tmp_path / 'does_not_exist.ict') + assert read_icartt(missing, return_None=True) is None + with pytest.raises(FileNotFoundError): + read_icartt(missing) + + +def test_read_icartt_truncated_header(tmp_path): + path = tmp_path / 'TRUNC_20240315_R0.ict' + path.write_text('12, 1001\nDoe, Jane\n') + with pytest.raises(ValueError, match='Unexpected end of file'): + read_icartt(str(path)) + + +def test_read_icartt_bad_format_line(tmp_path): + path = tmp_path / 'BAD_20240315_R0.ict' + path.write_text('not a header\nDoe, Jane\n') + with pytest.raises(ValueError, match='file format line'): + read_icartt(str(path)) + + +def test_read_icartt_unsupported_ffi(tmp_path): + path = tmp_path / 'FFI_20240315_R0.ict' + path.write_text('12, 2110\nDoe, Jane\n') + with pytest.raises(NotImplementedError, match='FFI 1001'): + read_icartt(str(path)) + with pytest.raises(NotImplementedError, match='FFI 1001'): + read_icartt(act.tests.EXAMPLE_AAF_ICARTT, format=2110) + + +def test_read_icartt_variable_count_mismatch(tmp_path): + path = tmp_path / 'MISMATCH_20240315_R0.ict' + path.write_text( + '\n'.join( + [ + '20, 1001', + 'Doe, Jane', + 'Test Org', + 'Test Instrument', + 'TESTMISSION', + '1, 1', + '2024,03,15,2024,03,16', + '1.0', + 'Start_UTC, seconds', + '2', + '1', + '-9999, -9999', + ] + ) + + '\n' + ) + with pytest.raises(ValueError, match='scale factor line'): + read_icartt(str(path)) From 0555113a28c0b17f0e77f8e8b60b3fa7b1cbcd14 Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Thu, 27 Aug 2026 14:59:43 -0500 Subject: [PATCH 2/4] DEL: removal of icartt dependency --- continuous_integration/environment_actions.yml | 1 - docs/environment_docs.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/continuous_integration/environment_actions.yml b/continuous_integration/environment_actions.yml index 0fdf55e3cf..82bc66c57e 100644 --- a/continuous_integration/environment_actions.yml +++ b/continuous_integration/environment_actions.yml @@ -35,5 +35,4 @@ dependencies: - moviepy - mpl2nc - pysp2 - - icartt - git+https://github.com/ARM-DOE/arm-test-data.git diff --git a/docs/environment_docs.yml b/docs/environment_docs.yml index 956bc8de0e..a7e17a76af 100644 --- a/docs/environment_docs.yml +++ b/docs/environment_docs.yml @@ -34,5 +34,4 @@ dependencies: - lazy_loader - ablog - pooch - - icartt - git+https://github.com/ARM-DOE/arm-test-data.git From 900140d4497280db788532f68af4b9f7fef4ac41 Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Thu, 27 Aug 2026 22:30:20 -0500 Subject: [PATCH 3/4] ENH: Requested corrections --- act/io/icartt.py | 67 ++++++++++++++++++++++++++--------------- tests/io/test_icartt.py | 6 ++-- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/act/io/icartt.py b/act/io/icartt.py index 9e5d316b2d..af69b90d7d 100644 --- a/act/io/icartt.py +++ b/act/io/icartt.py @@ -21,10 +21,6 @@ import numpy as np import xarray as xr -# Deprecated. ICARTT support is built in now, so this is always True. Retained so -# existing ``skipif`` guards and downstream references keep working. -_ICARTT_AVAILABLE = True - #: Field delimiter for the ICARTT format (ESDS-RFC-029v2 section 2.3.2). DEFAULT_FIELD_DELIM = ',' @@ -61,6 +57,22 @@ _REVISION_RE = re.compile(r'^R[A-Za-z0-9]{1,2}$') +def _as_number(value, label): + """ + Coerce a header scale factor or missing data flag to a float. + + Both are numeric per ESDS-RFC-029v2: scale factors may be fractional or in + exponential notation such as ``1.0e9`` (sections 2.1.4, 2.3.2.11), and + missing data flags are negative numbers such as -9999 (sections 2.1.4.2, + 2.3.2.12). + + """ + try: + return float(value) + except (TypeError, ValueError) as err: + raise ValueError(f'ICARTT {label} must be numeric, got {value!r}') from err + + class IcarttVariable: """ A single ICARTT variable description. @@ -75,10 +87,10 @@ class IcarttVariable: Variable standard name from the controlled list. longname : str, optional Free-form descriptive name. - scale : str or float, optional - Scale factor for the variable. - miss : str or float, optional - Missing data flag for the variable. + scale : float, optional + Scale factor for the variable. Coerced to float. + miss : float, optional + Missing data flag for the variable. Coerced to float. """ @@ -97,8 +109,8 @@ def __init__( self.units = units self.standardname = standardname self.longname = longname - self.scale = scale - self.miss = miss + self.scale = _as_number(scale, 'scale factor') + self.miss = _as_number(miss, 'missing data flag') @classmethod def from_desc(cls, parts, **kwargs): @@ -367,15 +379,21 @@ def readline(split=True): # Line 10 - number of dependent variables. nvar = int(readline()[0]) - # Lines 11-12 - scale factors and missing value flags. - vscal = readline() - vmiss = readline() - for label, values in (('scale factor', vscal), ('missing value', vmiss)): + # Lines 11-12 - scale factors and missing data flags, both numeric. + parsed = [] + for label, values in (('scale factor', readline()), ('missing value', readline())): if len(values) != nvar: raise ValueError( f'ICARTT {label} line of {self.name} has {len(values)} entries ' f'but the file declares {nvar} dependent variables' ) + try: + parsed.append([float(x) for x in values]) + except ValueError as err: + raise ValueError( + f'ICARTT {label} line of {self.name} has a non-numeric entry: {values!r}' + ) from err + vscal, vmiss = parsed # Lines 13 to 12+NV - dependent variable definitions. self.VNAME = [ @@ -745,10 +763,7 @@ def write(self, filename=None, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_D columns = [np.asarray(self.data[ivar], dtype=np.float64)] for var in self.VNAME: column = np.array(self.data[var.shortname], dtype=np.float64, copy=True) - try: - column[np.isnan(column)] = float(var.miss) - except (TypeError, ValueError): - column[np.isnan(column)] = DEFAULT_MISSING_VALUE + column[np.isnan(column)] = var.miss columns.append(column) header = [f'{self.NLHEAD}{delimiter} {self.FFI}'] @@ -763,8 +778,8 @@ def write(self, filename=None, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_D header.append(delimiter.join(str(x) for x in self.DX)) header.append(self.XNAME.desc(delimiter + ' ')) header.append(str(self.NV)) - header.append(delimiter.join(str(x) for x in self.VSCAL)) - header.append(delimiter.join(str(x) for x in self.VMISS)) + header.append(delimiter.join(DEFAULT_NUM_FORMAT % x for x in self.VSCAL)) + header.append(delimiter.join(DEFAULT_NUM_FORMAT % x for x in self.VMISS)) header.extend(var.desc(delimiter + ' ') for var in self.VNAME) header.append(str(self.NSCOML)) header.extend(self.SCOM) @@ -779,18 +794,18 @@ def write(self, filename=None, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_D self.name = str(filename) -def read_icartt(filename, format=1001, return_None=False, **kwargs): +def read_icartt(filename, ict_format=1001, return_None=False, **kwargs): """ Returns `xarray.Dataset` with stored data and metadata from a user-defined query of ICARTT from a single datastream. Has some procedures to ensure - time is correctly fomatted in returned Dataset. + time is correctly formatted in returned Dataset. Parameters ---------- filename : str Name of file to read. - format : int or str + ict_format : int or str ICARTT format to read. Only FFI 1001 is supported. return_None : bool, optional Catch IOError exception when file not found and return None. @@ -803,8 +818,10 @@ def read_icartt(filename, format=1001, return_None=False, **kwargs): ds : xarray.Dataset (or None) ACT Xarray dataset (or None if no data file(s) found). """ - if str(format) not in ('1001', 'FFI1001', 'Formats.FFI1001'): - raise NotImplementedError(f'ACT supports the ICARTT FFI 1001 format only, got {format!r}') + if str(ict_format) not in ('1001', 'FFI1001', 'Formats.FFI1001'): + raise NotImplementedError( + f'ACT supports the ICARTT FFI 1001 format only, got {ict_format!r}' + ) try: ict = Icartt.from_file(filename, **kwargs) diff --git a/tests/io/test_icartt.py b/tests/io/test_icartt.py index a8ada357f3..f9877ac643 100644 --- a/tests/io/test_icartt.py +++ b/tests/io/test_icartt.py @@ -129,8 +129,8 @@ def test_read_icartt_variable_attributes(): ds = read_icartt(act.tests.EXAMPLE_AAF_ICARTT) attrs = ds['static_pressure'].attrs assert attrs['units'] == 'hPa' - assert attrs['mvc'] == '-9999' - assert attrs['scale_factor'] == '1' + assert attrs['mvc'] == -9999.0 + assert attrs['scale_factor'] == 1.0 assert attrs['ULOD_Flag'] == '-7777' assert attrs['LLOD_Flag'] == '-8888' # This file supplies no per-variable uncertainty or LOD values. @@ -288,7 +288,7 @@ def test_read_icartt_unsupported_ffi(tmp_path): with pytest.raises(NotImplementedError, match='FFI 1001'): read_icartt(str(path)) with pytest.raises(NotImplementedError, match='FFI 1001'): - read_icartt(act.tests.EXAMPLE_AAF_ICARTT, format=2110) + read_icartt(act.tests.EXAMPLE_AAF_ICARTT, ict_format=2110) def test_read_icartt_variable_count_mismatch(tmp_path): From e604230a1ff14fbcdb4a8f03627f99f7669b2290 Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Fri, 28 Aug 2026 10:49:17 -0500 Subject: [PATCH 4/4] ADD/ENH: added apply scale factor function, removed ict_format --- act/io/icartt.py | 176 +++++++++++++++++++++++++++++++++++++--- tests/io/test_icartt.py | 141 +++++++++++++++++++++++++++++--- 2 files changed, 294 insertions(+), 23 deletions(-) diff --git a/act/io/icartt.py b/act/io/icartt.py index af69b90d7d..37b1c2bed6 100644 --- a/act/io/icartt.py +++ b/act/io/icartt.py @@ -21,6 +21,11 @@ import numpy as np import xarray as xr +#: The only file format index this module implements. ICARTT defines several +#: FFIs; 1001 is the one-dimensional time series described by ESDS-RFC-029v2 +#: section 2.3, and the format is fixed rather than selectable. +SUPPORTED_FFI = 1001 + #: Field delimiter for the ICARTT format (ESDS-RFC-029v2 section 2.3.2). DEFAULT_FIELD_DELIM = ',' @@ -73,6 +78,41 @@ def _as_number(value, label): raise ValueError(f'ICARTT {label} must be numeric, got {value!r}') from err +def _lod_mask(values, attrs): + """ + Mark the points that a scale factor must not be applied to. + + Limit of detection flags are not metadata, they sit in the data column as + literal values: -7777 above the ULOD and -8888 below the LLOD + (ESDS-RFC-029v2 section 2.1.4.3). Scaling them would turn a flag into a + number that no longer reads as a flag, so they are held out of the + arithmetic in both directions. Missing values need no mask, they are + already NaN by the time the data reaches here. + + Parameters + ---------- + values : numpy.ndarray + Data column to inspect. + attrs : dict + Variable attributes, read for ``ULOD_Flag`` and ``LLOD_Flag``. The + standard's 'N/A' stand-in, and anything else non-numeric, is ignored. + + Returns + ------- + mask : numpy.ndarray + Boolean array, True where the value is a limit of detection flag. + + """ + mask = np.zeros(np.shape(values), dtype=bool) + for key in ('ULOD_Flag', 'LLOD_Flag'): + try: + flag = float(attrs.get(key)) + except (TypeError, ValueError): + continue + mask |= values == flag + return mask + + class IcarttVariable: """ A single ICARTT variable description. @@ -168,7 +208,7 @@ class Icartt: def __init__(self): # Line 1 - file format information. - self.FFI = 1001 + self.FFI = SUPPORTED_FFI self.version = None # Number of header lines declared by the file, kept for validation only. # The authoritative value is the computed ``NLHEAD`` property. @@ -345,9 +385,10 @@ def readline(split=True): if len(first) > 2 and first[2]: self.version = first[2] - if self.FFI != 1001: + if self.FFI != SUPPORTED_FFI: raise NotImplementedError( - f'ACT supports the ICARTT FFI 1001 format only, this file declares {self.FFI}' + f'ACT supports the ICARTT FFI {SUPPORTED_FFI} format only, ' + f'this file declares {self.FFI}' ) # Lines 2-5. @@ -662,11 +703,23 @@ def tuple_attr(key, default): obj.XNAME.units = str(attrs.get('units', obj.XNAME.units)) obj.data[ivar] = values continue + # A read that applied the scale factor left the spent factor under + # 'scale_factor_applied', so undo it and restore the file's own + # header. Without that record there is nothing to reverse and the + # data is written as it stands against a scale factor of 1. + scale = attrs.get('scale_factor', DEFAULT_SCALE_FACTOR) + applied = attrs.get('scale_factor_applied', DEFAULT_SCALE_FACTOR) + if applied != DEFAULT_SCALE_FACTOR: + scale = applied + values = np.array(values, dtype=np.float64, copy=True) + keep = ~_lod_mask(values, attrs) + values[keep] /= applied + obj.VNAME.append( IcarttVariable( name, str(attrs.get('units', 'none')), - scale=attrs.get('scale_factor', DEFAULT_SCALE_FACTOR), + scale=scale, miss=attrs.get('mvc', DEFAULT_MISSING_VALUE), ) ) @@ -794,22 +847,111 @@ def write(self, filename=None, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_D self.name = str(filename) -def read_icartt(filename, ict_format=1001, return_None=False, **kwargs): +def _apply_scale_factors(ds): + """ + Apply the ICARTT scale factors, in place. See :func:`apply_scale_factors`. + + Kept private and separate from the public wrapper because the + ``apply_scale_factors`` keyword of :func:`read_icartt` shadows the public + function's name inside that function's body. + + """ + for name in ds.data_vars: + attrs = ds[name].attrs + scale = attrs.get('scale_factor', DEFAULT_SCALE_FACTOR) + if scale == DEFAULT_SCALE_FACTOR: + continue + + values = np.array(ds[name].values, dtype=np.float64, copy=True) + keep = ~_lod_mask(values, attrs) + values[keep] *= scale + ds[name].values = values + + # The factor has been spent. Leaving it in place would invite a second + # application, by another call to this function or by any CF decoder, + # since 'scale_factor' is a reserved CF attribute. The original is kept + # under a name CF does not act on so the write path can reverse this. + attrs['scale_factor'] = DEFAULT_SCALE_FACTOR + attrs['scale_factor_applied'] = scale + + return ds + + +def apply_scale_factors(ds): + """ + + Apply the ICARTT scale factors to the data variables of a Dataset. + + Header line 11 gives one scale factor per dependent variable, and the value + in the file is the reported value divided by it, so reading multiplies + (ESDS-RFC-029v2 sections 2.1.4 and 2.3.2.11). Factors should be 1, but the + standard permits fractional and exponential values and its own examples use + them. + + Limit of detection flags are left alone. They sit in the data column as + literal -7777 and -8888 values rather than as metadata (section 2.1.4.3), + and scaling them would destroy them. + + Scale factors are applied to the data columns only. The ``ULOD_Value``, + ``LLOD_Value`` and ``uncertainty`` attributes are carried through exactly as + the file states them, unscaled, so a numeric limit of detection is not + directly comparable to the scaled values in the array. Section 2.1.4.3 also + allows those keywords to hold 'N/A' or the short name of another dependent + variable, so they are treated as verbatim file metadata. + + On each scaled variable ``scale_factor`` is reset to 1.0 and the original + factor recorded as ``scale_factor_applied``, which makes the call + idempotent, stops any CF decoder applying the factor a second time, and lets + :func:`write_icartt` restore the original header. + + Parameters + ---------- + ds : xarray.Dataset + Dataset from :func:`read_icartt`. Modified in place. + + Returns + ------- + ds : xarray.Dataset + The same Dataset, with scale factors applied. + + Examples + -------- + .. code-block:: python + + from act.io.icartt import apply_scale_factors, read_icartt + + ds = read_icartt(filename, apply_scale_factors=False) + ds = apply_scale_factors(ds) + + """ + return _apply_scale_factors(ds) + + +def read_icartt(filename, return_None=False, apply_scale_factors=True, **kwargs): """ Returns `xarray.Dataset` with stored data and metadata from a user-defined query of ICARTT from a single datastream. Has some procedures to ensure time is correctly formatted in returned Dataset. + Scale factors from header line 11 are applied by default. They are applied + to the data columns only, so the ``ULOD_Value``, ``LLOD_Value`` and + ``uncertainty`` attributes are carried through exactly as the file states + them, unscaled. Limit of detection flags in the data are left untouched. + Parameters ---------- filename : str Name of file to read. - ict_format : int or str - ICARTT format to read. Only FFI 1001 is supported. return_None : bool, optional Catch IOError exception when file not found and return None. Default is False. + apply_scale_factors : bool, optional + Multiply each dependent variable by its scale factor. Default is True. + When False the values are returned exactly as the file records them and + the scale factor is left live in the ``scale_factor`` attribute, which + is a reserved CF name that xarray will act on if the Dataset is written + to netCDF and read back. **kwargs : keywords keywords to pass on through to Icartt.from_file. @@ -818,11 +960,6 @@ def read_icartt(filename, ict_format=1001, return_None=False, **kwargs): ds : xarray.Dataset (or None) ACT Xarray dataset (or None if no data file(s) found). """ - if str(ict_format) not in ('1001', 'FFI1001', 'Formats.FFI1001'): - raise NotImplementedError( - f'ACT supports the ICARTT FFI 1001 format only, got {ict_format!r}' - ) - try: ict = Icartt.from_file(filename, **kwargs) except (FileNotFoundError, OSError) as exception: @@ -834,7 +971,20 @@ def read_icartt(filename, ict_format=1001, return_None=False, **kwargs): return None raise - return ict.to_xarray() + ds = ict.to_xarray() + if apply_scale_factors: + return _apply_scale_factors(ds) + + if any(scale != DEFAULT_SCALE_FACTOR for scale in ict.VSCAL): + warnings.warn( + f'ICARTT file {ict.name} declares non-unity scale factors that were not ' + 'applied, so the values are as recorded in the file. The unapplied factor ' + "is left in each variable's 'scale_factor' attribute, which xarray will " + 'apply on a netCDF round trip. Pass apply_scale_factors=True to apply it ' + 'here instead.', + stacklevel=2, + ) + return ds def write_icartt(ds, filename, **kwargs): diff --git a/tests/io/test_icartt.py b/tests/io/test_icartt.py index f9877ac643..a0ecf84124 100644 --- a/tests/io/test_icartt.py +++ b/tests/io/test_icartt.py @@ -1,9 +1,12 @@ +import inspect +import warnings + import numpy as np import pytest import xarray as xr import act -from act.io.icartt import Icartt, read_icartt, write_icartt +from act.io.icartt import Icartt, apply_scale_factors, read_icartt, write_icartt # A minimal but spec-legal FFI 1001 normal comments section. Includes free-form # text ahead of the first keyword, a keyword value spanning several lines, one @@ -39,7 +42,15 @@ SAMPLE_DATA = ['0,20.5,1013.2', '1,-9999,1012.8', '2,21.0,-8888'] -def build_ict(tmp_path, name='TEST_20240315_R1.ict', ncom=None, data=None, nlhead=None): +def build_ict( + tmp_path, + name='TEST_20240315_R1.ict', + ncom=None, + data=None, + nlhead=None, + vscal='1, 1', + vmiss='-9999, -8888', +): """Write a synthetic FFI 1001 file and return its path.""" ncom = list(SAMPLE_NCOM if ncom is None else ncom) data = list(SAMPLE_DATA if data is None else data) @@ -57,8 +68,8 @@ def build_ict(tmp_path, name='TEST_20240315_R1.ict', ncom=None, data=None, nlhea '1.0', 'Start_UTC, seconds', str(nv), - '1, 1', - '-9999, -8888', + vscal, + vmiss, 'temperature, degC', 'pressure, hPa', str(nscoml), @@ -83,6 +94,9 @@ def test_read_icartt_lazy_loader(): assert act.io.read_icartt is read_icartt assert act.io.write_icartt is write_icartt assert act.io.Icartt is Icartt + # apply_scale_factors is deliberately not in the shared act.io namespace: + # the name does not say ICARTT, and it does not implement CF add_offset. + assert 'apply_scale_factors' not in act.io.__all__ def test_read_icartt_structure(): @@ -283,12 +297,16 @@ def test_read_icartt_bad_format_line(tmp_path): def test_read_icartt_unsupported_ffi(tmp_path): - path = tmp_path / 'FFI_20240315_R0.ict' - path.write_text('12, 2110\nDoe, Jane\n') - with pytest.raises(NotImplementedError, match='FFI 1001'): - read_icartt(str(path)) - with pytest.raises(NotImplementedError, match='FFI 1001'): - read_icartt(act.tests.EXAMPLE_AAF_ICARTT, ict_format=2110) + # The format is not selectable, so the only way to hit this is a file that + # declares an FFI other than 1001 on its first line. + for ffi in ('2110', '2310'): + path = tmp_path / f'FFI{ffi}_20240315_R0.ict' + path.write_text(f'12, {ffi}\nDoe, Jane\n') + with pytest.raises(NotImplementedError, match=f'declares {ffi}'): + read_icartt(str(path)) + + # A supported file reads normally, and read_icartt takes no format keyword. + assert 'ict_format' not in inspect.signature(read_icartt).parameters def test_read_icartt_variable_count_mismatch(tmp_path): @@ -314,3 +332,106 @@ def test_read_icartt_variable_count_mismatch(tmp_path): ) with pytest.raises(ValueError, match='scale factor line'): read_icartt(str(path)) + + +# ---------------------------------------------------------------------- +# Scale factors (ESDS-RFC-029v2 sections 2.1.4, 2.3.2.11) +# ---------------------------------------------------------------------- + +# Two dependent variables carrying a fractional and an exponential scale +# factor, with a distinct missing flag so the LOD flags stay visible in the +# data. temperature holds a real value, an LLOD flag and a ULOD flag. +SCALED_DATA = ['0,113178,1.5', '1,-8888,2.5', '2,-7777,-9999'] + + +def test_read_icartt_applies_scale_factors(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1.0e9', vmiss='-9999, -9999', data=SCALED_DATA) + ds = read_icartt(path) + + # 113178 * 0.0001, then the two LOD flags, which must not be scaled. + np.testing.assert_allclose(ds['temperature'].values, [11.3178, -8888.0, -7777.0]) + np.testing.assert_allclose(ds['pressure'].values, [1.5e9, 2.5e9, np.nan]) + + +def test_read_icartt_scale_factor_attrs(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1', vmiss='-9999, -9999', data=SCALED_DATA) + ds = read_icartt(path) + + # The spent factor is neutralised so no CF decoder applies it twice, and + # the original is kept for the write path. + assert ds['temperature'].attrs['scale_factor'] == 1.0 + assert ds['temperature'].attrs['scale_factor_applied'] == 0.0001 + # A unity factor is untouched and gains no provenance attribute. + assert ds['pressure'].attrs['scale_factor'] == 1.0 + assert 'scale_factor_applied' not in ds['pressure'].attrs + + +def test_read_icartt_scale_factors_leave_lod_values_unscaled(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1', vmiss='-9999, -9999', data=SCALED_DATA) + ds = read_icartt(path) + + # Data columns only: LOD values and uncertainty stay as the file states them. + assert ds['temperature'].attrs['ULOD_Value'] == '100' + assert ds['temperature'].attrs['LLOD_Value'] == '-50' + assert ds['temperature'].attrs['uncertainty'] == '0.5' + + +def test_read_icartt_scale_factors_opt_out(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1', vmiss='-9999, -9999', data=SCALED_DATA) + with pytest.warns(UserWarning, match='non-unity scale factors'): + ds = read_icartt(path, apply_scale_factors=False) + + np.testing.assert_allclose(ds['temperature'].values, [113178.0, -8888.0, -7777.0]) + assert ds['temperature'].attrs['scale_factor'] == 0.0001 + assert 'scale_factor_applied' not in ds['temperature'].attrs + + +def test_read_icartt_opt_out_does_not_warn_for_unity(tmp_path): + # The common all-1s file has nothing to warn about either way. + with warnings.catch_warnings(): + warnings.simplefilter('error') + read_icartt(build_ict(tmp_path), apply_scale_factors=False) + read_icartt(build_ict(tmp_path)) + + +def test_apply_scale_factors_standalone_and_idempotent(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1', vmiss='-9999, -9999', data=SCALED_DATA) + with pytest.warns(UserWarning, match='non-unity scale factors'): + ds = read_icartt(path, apply_scale_factors=False) + + apply_scale_factors(ds) + np.testing.assert_allclose(ds['temperature'].values, [11.3178, -8888.0, -7777.0]) + + # A second pass sees a unity factor and changes nothing. + apply_scale_factors(ds) + np.testing.assert_allclose(ds['temperature'].values, [11.3178, -8888.0, -7777.0]) + + +def test_write_icartt_reverses_scale_factors(tmp_path): + path = build_ict(tmp_path, vscal='0.0001, 1.0e9', vmiss='-9999, -9999', data=SCALED_DATA) + ds1 = read_icartt(path) + + out = tmp_path / 'OUT_20240315_R1.ict' + write_icartt(ds1, out) + written = out.read_text().splitlines() + + # Header line 11 and the data columns come back as the file had them. + assert [x.strip() for x in written[10].split(',')] == ['0.0001', '1000000000'] + assert written[-3:] == ['0,113178,1.5', '1,-8888,2.5', '2,-7777,-9999'] + + ds2 = read_icartt(str(out)) + xr.testing.assert_allclose(ds1, ds2, rtol=1e-9) + + +def test_write_icartt_without_applied_record(tmp_path): + # Nothing to reverse, so the scaled values are written against a scale of 1. + path = build_ict(tmp_path, vscal='0.0001, 1', vmiss='-9999, -9999', data=SCALED_DATA) + ds = read_icartt(path) + del ds['temperature'].attrs['scale_factor_applied'] + + out = tmp_path / 'NOREC_20240315_R1.ict' + write_icartt(ds, out) + written = out.read_text().splitlines() + + assert [x.strip() for x in written[10].split(',')] == ['1', '1'] + assert written[-3].startswith('0,11.3178,')