Allow for trailing whitespace in categories

If a line like "_cat1.foo bar" has trailing
whitespace before the newline, this is valid
mmCIF, so capture the whitespace in the token
group rather than reporting an error.

Author: benmwebb

ihm/format.py

@@ -570,15 +570,18 @@ def _skip_initial_whitespace(self, line, strlen, start_pos):
 
 class _CategoryTokenGroup:
     """A group of tokens which set a single data item"""
-    def __init__(self, vartoken, valtoken):
+    def __init__(self, vartoken, valtoken, whitespace):
         self.vartoken, self.valtoken = vartoken, valtoken
+        self.whitespace = whitespace
 
     def __str__(self):
         return ("<_CategoryTokenGroup(%s, %s)>"
                 % (self.vartoken.as_mmcif(), self.valtoken.token.as_mmcif()))
 
     def as_mmcif(self):
-        return self.vartoken.as_mmcif() + self.valtoken.as_mmcif() + "\n"
+        return (self.vartoken.as_mmcif() + self.valtoken.as_mmcif()
+                + (self.whitespace.as_mmcif() if self.whitespace else "")
+                + "\n")
 
     def __set_value(self, val):
         self.valtoken.value = val
@@ -1037,11 +1040,17 @@ def _read_value(self, vartoken):
                 "No valid value found for %s.%s on line %d"
                 % (vartoken.category, vartoken.keyword, self._linenum))
         eoltok = self._get_token()
+        # Handle whitespace at the end of the line, if any
+        if isinstance(eoltok, _WhitespaceToken):
+            whitespace = eoltok
+            eoltok = self._get_token()
+        else:
+            whitespace = None
         if not isinstance(eoltok, _EndOfLineToken):
             raise CifParserError(
                 "No end of line after %s.%s on line %d"
                 % (vartoken.category, vartoken.keyword, self._linenum))
-        return _CategoryTokenGroup(vartoken, spval)
+        return _CategoryTokenGroup(vartoken, spval, whitespace)
 
     def _read_loop(self, looptoken):
         """Handle a loop_ construct"""

test/test_format.py

@@ -1114,21 +1114,27 @@ def test_cif_token_reader(self):
         cif = """
 data_foo_bar
 #
-_cat1.foo ?
+_cat1.foo ?  
 #
 loop_
 _foo.bar
 _foo.baz
 a b c d
 x y
-"""
+_cat2.bar ?
+"""  # noqa: W291
         r = ihm.format.CifTokenReader(StringIO(cif))
         tokens = list(r.read_file())
         self.assertIsInstance(tokens[5], ihm.format._CategoryTokenGroup)
+        # cat1.foo has trailing whitespace
+        self.assertEqual(tokens[5].whitespace.as_mmcif(), "  ")
         self.assertIsInstance(tokens[8], ihm.format._LoopHeaderTokenGroup)
         self.assertIsInstance(tokens[9], ihm.format._LoopRowTokenGroup)
         self.assertIsInstance(tokens[10], ihm.format._LoopRowTokenGroup)
         self.assertIsInstance(tokens[11], ihm.format._LoopRowTokenGroup)
+        self.assertIsInstance(tokens[13], ihm.format._CategoryTokenGroup)
+        # cat2.bar has no trailing whitespace
+        self.assertIsNone(tokens[13].whitespace)
         new_cif = "".join(x.as_mmcif() for x in tokens)
         self.assertEqual(new_cif, cif)
 
@@ -1138,6 +1144,12 @@ def test_cif_token_reader_missing_value(self):
         r = ihm.format.CifTokenReader(StringIO(cif))
         self.assertRaises(ihm.format.CifParserError, list, r.read_file())
 
+    def test_cif_token_reader_category_no_eol(self):
+        """Category must end in end-of-line"""
+        cif = '_foo.bar foo bar baz\n'
+        r = ihm.format.CifTokenReader(StringIO(cif))
+        self.assertRaises(ihm.format.CifParserError, list, r.read_file())
+
     def test_cif_token_reader_loop_mixed_categories(self):
         """Test bad mmCIF loop with a mix of categories"""
         cif = 'loop_\n_atom_site.id\n_foo.bar\n'
@@ -1338,12 +1350,15 @@ def test_category_token_group(self):
         space = ihm.format._WhitespaceToken("   ")
         val = ihm.format._TextValueToken("baz", quote=None)
         tg = ihm.format._CategoryTokenGroup(
-            var, ihm.format._SpacedToken([space], val))
+            var, ihm.format._SpacedToken([space], val), None)
         self.assertEqual(str(tg), "<_CategoryTokenGroup(_foo.bar, baz)>")
         self.assertEqual(tg.as_mmcif(), '_foo.bar   baz\n')
         self.assertEqual(tg.category, "_foo")
         self.assertEqual(tg.keyword, "bar")
         self.assertEqual(tg.value, "baz")
+        tg = ihm.format._CategoryTokenGroup(
+            var, ihm.format._SpacedToken([space], val), space)
+        self.assertEqual(tg.as_mmcif(), '_foo.bar   baz   \n')
         tg.value = None
         self.assertIsNone(tg.value)
 
@@ -1406,7 +1421,7 @@ def test_change_value_filter_category(self):
         space = ihm.format._WhitespaceToken("   ")
         val = ihm.format._TextValueToken("baz", quote=None)
         tg = ihm.format._CategoryTokenGroup(
-            var, ihm.format._SpacedToken([space], val))
+            var, ihm.format._SpacedToken([space], val), None)
         # Value does not match
         f = ihm.format.ChangeValueFilter("_foo.bar", old='old', new='new')
         new_tg = f.filter_category(tg)