Add Container premitive

README.md

@@ -263,10 +263,11 @@ Plyse ships with a set of default types that should cover the basic needs pretty
 - IntegerComparison
 - Field
 - MultiField
+- Container
 
 The grammar can be manipulated programatically, removing or adding types used for the terms:
 ```python
-from plyse import GrammarFactory, IntegerComparison, QueryParser, Query
+from plyse import GrammarFactory, IntegerComparison, QueryParser, Query, Container
 
 grammar = GrammarFactory.build_default()
 grammar.value_types
@@ -278,16 +279,28 @@ grammar.value_types
 
 grammar.remove_type('integer_range')
 grammar.add_value_type(IntegerComparison(grammar.term_parser.integer_comparison_parse))
+grammar.add_value_type(Container(
+  grammar.term_parser.container_parse,
+  grammar.term_parser.integer_parse,
+  grammar.term_parser.partial_string_parse,
+  grammar.term_parser.quoted_string_parse
+  ))
 
 parser = QueryParser(grammar)
-q = qp.parse("age:>18")
+q = parser.parse("age:>18 name:[John, Doe]")
 
 print q.terms()
 
 #[{'field': 'age',
-#  'field_type': 'attribute',
+#  'field_type':
+#  'attribute',
 #  'val': 18,
-#  'val_type': 'greater_than'}]
+#  'val_type': 'greater_than'},
+# {'field': 'name',
+#  'field_type':
+#  'attribute',
+#  'val': ['John', 'Doe'],
+#  'val_type': 'container'}]
 ```
 
 For more examples take a look at the different tests covering the funcionality of each module [here](https://github.com/sebastiandev/plyse/tree/master/plyse/tests)

plyse/expressions/primitives.py

@@ -2,7 +2,7 @@
 from pyparsing import (Literal, Word, MatchFirst, CaselessKeyword, Regex, QuotedString as QString,
                        Suppress, Optional, Group, FollowedBy, Combine,
                        operatorPrecedence, opAssoc, ParseException,
-                       ParserElement, alphanums, And, OneOrMore)
+                       ParserElement, alphanums, And, OneOrMore, delimitedList)
 
 from ..util import load_module
 
@@ -17,19 +17,28 @@ class PrimitiveFactory(object):
     def build_from_conf(conf, parser):
         _cls = load_module(conf['class'])
 
-        if 'parse_method' in conf:
-            _kwargs = {'parse_method': getattr(parser, conf['parse_method'])}
+        _kwargs = {}
+        if not (set(conf.keys() - ['class']) <= set(_cls.__init__.__code__.co_varnames[1:])):
+            extra_parameters = set(conf.keys()) - (set(conf.keys()) & set(_cls.__init__.__code__.co_varnames[1:]))
+            raise PrimitiveFactoryError(
+                'Invalid primitive definition: configuration for class {} has extra parameters'.format(
+                    _cls.__name__, extra_parameters))
 
-        elif 'range_parse_method' in conf:
-            _kwargs = {
-                'range_parse_method': getattr(parser, conf['range_parse_method']),
-                'item_parse_method': getattr(parser, conf['item_parse_method'])
-            }
+        for var in _cls.__init__.__code__.co_varnames[1:]:
+            if var not in conf:
+                continue
 
-        else:
-            raise PrimitiveFactoryError("Invalid Primitive definition, parsing method invalid or not defined.")
+            if var.endswith('parse_method'):
+                try:
+                    _kwargs[var] = getattr(parser, conf[var])
+                except AttributeError:
+                    raise PrimitiveFactoryError(
+                        'Invalid Primitive definition. Parser {} does not have a method {}'.format(parser, conf[var]))
 
-        return _cls(precedence=conf['precedence'], **_kwargs)
+                continue
+
+            _kwargs[var] = conf[var]
+        return _cls(**_kwargs)
 
 
 def concatenate(elems, operator='OR', class_to_embed_elem=None):
@@ -159,6 +168,32 @@ def __init__(self, parse_method=None, precedence=6):
             self.addParseAction(parse_method)
 
 
+class Container(And, BaseType):
+
+    name = 'container'
+
+    def __init__(self,
+                 parse_method=None,
+                 int_parse_method=None,
+                 part_str_parse_method=None,
+                 qstr_parse_method=None,
+                 delim=',',
+                 precedence=8):
+        value = Integer(int_parse_method) ^ PartialString(part_str_parse_method) ^ QuotedString(qstr_parse_method)
+        And.__init__(
+            self,
+            [
+                Suppress(Literal('['))
+                + Group(delimitedList(value, delim=delim) + Optional(Suppress(Literal(delim))))
+                + Suppress(Literal(']'))
+            ],
+        )
+        BaseType.__init__(self, precedence)
+
+        if parse_method:
+            self.addParseAction(parse_method)
+
+
 class IntegerComparison(And, BaseType):
 
     name = 'integer_comparison'

plyse/grammar.py

@@ -13,17 +13,21 @@ class GrammarError(Exception):
 
 class GrammarFactory(object):
 
+    default_operators = [
+        Operator("not", ['!', '-', 'not']),
+        Operator('and', ['+', 'and']),
+        Operator('or', ['or'], True)]
+
     @staticmethod
     def build(term, parser, operators, keywords=None):
         return Grammar(term=term, operators=operators, term_parser=parser, keywords=keywords)
 
     @staticmethod
     def build_default(term_parser=None):
         t_parser = term_parser or TermParserFactory.build_default()
-        operators = [Operator("not", ['!', '-', 'not']), Operator('and', ['+', 'and']), Operator('or', ['or'], True)]
         term = TermFactory.build_default_term(t_parser)
 
-        return Grammar(operators=operators, term=term, keywords=[], term_parser=t_parser)
+        return Grammar(operators=GrammarFactory.default_operators, term=term, keywords=[], term_parser=t_parser)
 
     @staticmethod
     def build_from_conf(conf):
@@ -127,7 +131,8 @@ def add_keyword(self, keyword):
 
     def add_value_type(self, value):
         if not isinstance(value, ParserElement):
-            raise GrammarError("Value types should be PyParsing ParserElements or plyse.expressions.primitives.BaseType")
+            raise GrammarError(
+                "Value types should be PyParsing ParserElements or plyse.expressions.primitives.BaseType")
 
         new_term = TermFactory.build_term(self._term.field, self._term.values + [value], *self._term.parseAction)
         self._update_grammar(new_term)

plyse/term_parser.py

@@ -106,12 +106,29 @@ def field_parse(self, string, location, tokens):
         """
         if tokens:
             fields = [f for f in "".join(tokens).split(":") if f]
-            t = fields if len(fields) > 1 else fields[0]
+            t = fields[0]
 
             field_value = self._field_name_aliases.get(t, t)
 
             return self._build_field_data(field_value, Term.ATTRIBUTE)
 
+    def multifield_parse(self, string, location, tokens):
+        """
+        Fields are whatever comes before a separator and they are usually use for attribute/property matching. The value
+        of a field is parsed separately form the field name and it depends on the definition of the grammar and the
+        accepted/supported values. Thus this method receives a token list with <field name> <separator> <field name>.
+
+        Combined or nested fields are allowed, the pattern would be:
+            <field name> <separator> <field name> <separator> ...
+
+            > ej: address:zip:ABC1234 => token list would be ['address', ':', 'zip']
+        """
+        if tokens:
+            fields = [f for f in "".join(tokens).split(":") if f]
+            field_value = self._field_name_aliases.get(":".join(fields), ":".join(fields))
+
+            return self._build_field_data(field_value, Term.MULTI_ATTRIBUTE)
+
     def integer_parse(self, string, location, tokens):
         if tokens:
             r = self._build_value_data(int(tokens[0]), Term.INT)
@@ -146,6 +163,11 @@ def range_parse(self, string, location, tokens):
             return self._build_value_data([tokens[0][Term.VAL], tokens[2][Term.VAL]],
                                           Term.RANGE % tokens[0][Term.VAL_TYPE])
 
+    def container_parse(self, string, location, tokens):
+        if tokens:
+            value = [one[Term.VAL] for one in tokens[0]]
+            return self._build_value_data(value, Term.CONTAINER)
+
 
 class Term(dict):
 
@@ -159,11 +181,13 @@ class Term(dict):
     GREATER_EQUAL_THAN = 'greater_equal_than'
     LOWER_THAN = 'lower_than'
     LOWER_EQUAL_THAN = 'lower_equal_than'
+    CONTAINER = "container"
 
     # field types
     KEYWORD = 'keyword'
     DEFAULT = 'default'
     ATTRIBUTE = 'attribute'
+    MULTI_ATTRIBUTE = 'multi_attribute'
 
     # term keys
     FIELD = 'field'

plyse/tests/grammar_from_config_test.py

@@ -9,7 +9,7 @@
     'term_parser': 'plyse.term_parser.TermParser',
     'default_fields': ['default'],
     'aliases': {},
-    'operators': [ 
+    'operators': [
         {'not': {
             'implicit': False,
             'symbols': ['not', '-', '!']
@@ -54,7 +54,15 @@
                 'precedence': 5,
                 'range_parse_method': 'range_parse',
                 'item_parse_method': 'integer_parse'
-            }
+            },
+            {
+                "class": "plyse.expressions.primitives.Container",
+                "precedence": 10,
+                "parse_method": "container_parse",
+                "int_parse_method": "integer_parse",
+                "part_str_parse_method": "partial_string_parse",
+                "qstr_parse_method": "quoted_string_parse",
+            },
         ]
     }
 }
@@ -137,5 +145,6 @@ def test_remove_keyword_then_match_as_field(self):
         r = g.parse("is:something")
         self._check_values(r[0], 'is', 'something', Term.PARTIAL_STRING)
 
+
 if __name__ == "__main__":
     unittest.main()

plyse/tests/grammar_test.py

@@ -185,5 +185,6 @@ def test_parenthesis_OR_field_OR_field_AND_NOT_field(self):
         self._check_values(r[4][2][0], expected_val="NOT")
         self._check_values(r[4][2][1], "e", 0, Term.INT)
 
+
 if __name__ == '__main__':
     unittest.main()

plyse/tests/primitives_test.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 from plyse.expressions.primitives import *
+from pyparsing import ParseResults
 
 
 class PrimitiveTester(unittest.TestCase):
@@ -11,7 +12,11 @@ def assert_parsed_output(self, primitive, input_expected_val_dict):
             if exp:
                 output = primitive.parseString(inp)
                 for n, o in enumerate(output):
+                    # Workaround for container primitive due Group returns not a list but a ParseResults
+                    if isinstance(o, ParseResults):
+                        o = o.asList()
                     self.assertEqual(exp[n], o)
+                self.assertEqual(len(exp), len(output))
             else:
                 self.assertRaises(ParseException, primitive.parseString, inp)
 
@@ -65,10 +70,10 @@ def test_field(self):
         self.assert_parsed_output(f, {'name>test': ['name', '>'], 'age>10': ['age', '>'], 'aa': None})
 
     def test_multi_field(self):
-        mf = Field()
+        mf = MultiField()
         self.assert_parsed_output(mf, {'first:name:test': ['first', ':', 'name', ':'], 'age:10': ['age', ':']})
 
-        mf = Field(field_separator='>')
+        mf = MultiField(field_separator='>')
         self.assert_parsed_output(mf, {'first>name>test': ['first', '>', 'name', '>'], 'age>10': ['age', '>']})
 
     def test_any(self):
@@ -77,11 +82,23 @@ def test_any(self):
 
     def test_integer_comparisson(self):
         ic = IntegerComparison()
-        self.assert_parsed_output(ic, {'<10': ['<', '10'], '<=10': ['<=', '10'], '>10': ['>', '10'], '>=10': ['>=', '10'], '">e"': None})
+        self.assert_parsed_output(ic, {'<10': ['<', '10'], '<=10': ['<=', '10'], '>10': [
+                                  '>', '10'], '>=10': ['>=', '10'], '">e"': None})
 
     def test_string_proximity(self):
         sp = StringProximity()
         self.assert_parsed_output(sp, {"'hello world'~3": ['hello world', '~', '3']})
 
+    def test_container(self):
+        c = Container()
+        self.assert_parsed_output(c, {
+            "[a]": [["a"]],
+            "[aa,     bbbb]": [["aa", "bbbb"]],
+            "[a,b]": [["a", "b"]],
+            "[a,bbbbb]": [["a", "bbbbb"]],
+            "[1, a,]": [["1", "a"]],
+            "[a, '1']": [["a", "1"]]})
+
+
 if __name__ == '__main__':
     unittest.main()

plyse/tests/term_parser_test.py

@@ -3,7 +3,10 @@
 import unittest
 from plyse.grammar import GrammarFactory
 from plyse.term_parser import TermParserFactory, Term
-from plyse.expressions.primitives import IntegerComparison
+from plyse.expressions.primitives import (IntegerComparison, Container, MultiField, Integer,
+                                          IntegerRange, PartialString, QuotedString)
+from plyse.expressions.operators import Operator
+from plyse.expressions.terms import TermFactory
 
 
 class TermParserTester(unittest.TestCase):
@@ -20,8 +23,20 @@ def _init_and_parse(self, input_str):
         }
 
         term_parser = TermParserFactory.build_from_conf(conf)
-        g = GrammarFactory.build_default(term_parser)
+        values = [IntegerRange(range_parse_method=term_parser.range_parse, item_parse_method=term_parser.integer_parse),
+                  Integer(parse_method=term_parser.integer_parse),
+                  PartialString(parse_method=term_parser.partial_string_parse),
+                  QuotedString(parse_method=term_parser.quoted_string_parse)]
+        term = TermFactory.build_term(MultiField(parse_method=term_parser.multifield_parse),
+                                      values=values,
+                                      parse_method=term_parser.term_parse)
+        g = GrammarFactory.build(term=term, parser=term_parser, operators=GrammarFactory.default_operators)
         g.add_value_type(IntegerComparison(term_parser.integer_comparison_parse))
+        g.add_value_type(Container(
+            parse_method=term_parser.container_parse,
+            int_parse_method=term_parser.integer_parse,
+            part_str_parse_method=term_parser.partial_string_parse,
+            qstr_parse_method=term_parser.quoted_string_parse))
         self.assertTrue(g.parse(input_str, True))
 
         return g.parse(input_str, True)[0]
@@ -38,6 +53,14 @@ def test_default_field_partial_text(self):
         r = self._init_and_parse("texto")
         self._check_values(r, ['name', 'description'], "texto", Term.PARTIAL_STRING)
 
+    def test_default_multifield_partial_text(self):
+        r = self._init_and_parse("texto:texto3:texto4")
+        self._check_values(r, 'texto:texto3', "texto4", Term.PARTIAL_STRING)
+
+    def test_default_multifield_int(self):
+        r = self._init_and_parse("texto:texto3:1")
+        self._check_values(r, 'texto:texto3', 1, Term.INT)
+
     def test_default_field_partial_text_with_wildcard(self):
         r = self._init_and_parse("'texto*'")
         self._check_values(r, ['name', 'description'], "texto*", Term.PARTIAL_STRING)
@@ -79,5 +102,16 @@ def test_field_int_comparisson(self):
         r = self._init_and_parse("age:<=18")
         self._check_values(r, 'age', 18, Term.LOWER_EQUAL_THAN)
 
+    def test_filed_container(self):
+        r = self._init_and_parse("field:[a,b,c]")
+        self._check_values(r, 'field', ["a", "b", "c"], Term.CONTAINER)
+
+        r = self._init_and_parse("field:[1,b,c,]")
+        self._check_values(r, 'field', [1, "b", "c"], Term.CONTAINER)
+
+        r = self._init_and_parse("field:[aa,b,'1',]")
+        self._check_values(r, 'field', ["aa", "b", "1"], Term.CONTAINER)
+
+
 if __name__ == '__main__':
     unittest.main()