diff --git a/README.md b/README.md index 41a18b1..edc94b4 100644 --- a/README.md +++ b/README.md @@ -263,10 +263,11 @@ Plyse ships with a set of default types that should cover the basic needs pretty - IntegerComparison - Field - MultiField +- Container The grammar can be manipulated programatically, removing or adding types used for the terms: ```python -from plyse import GrammarFactory, IntegerComparison, QueryParser, Query +from plyse import GrammarFactory, IntegerComparison, QueryParser, Query, Container grammar = GrammarFactory.build_default() grammar.value_types @@ -278,16 +279,28 @@ grammar.value_types grammar.remove_type('integer_range') grammar.add_value_type(IntegerComparison(grammar.term_parser.integer_comparison_parse)) +grammar.add_value_type(Container( + grammar.term_parser.container_parse, + grammar.term_parser.integer_parse, + grammar.term_parser.partial_string_parse, + grammar.term_parser.quoted_string_parse + )) parser = QueryParser(grammar) -q = qp.parse("age:>18") +q = parser.parse("age:>18 name:[John, Doe]") print q.terms() #[{'field': 'age', -# 'field_type': 'attribute', +# 'field_type': +# 'attribute', # 'val': 18, -# 'val_type': 'greater_than'}] +# 'val_type': 'greater_than'}, +# {'field': 'name', +# 'field_type': +# 'attribute', +# 'val': ['John', 'Doe'], +# 'val_type': 'container'}] ``` For more examples take a look at the different tests covering the funcionality of each module [here](https://github.com/sebastiandev/plyse/tree/master/plyse/tests) diff --git a/plyse/expressions/primitives.py b/plyse/expressions/primitives.py index a533a07..c3f04c6 100644 --- a/plyse/expressions/primitives.py +++ b/plyse/expressions/primitives.py @@ -2,7 +2,7 @@ from pyparsing import (Literal, Word, MatchFirst, CaselessKeyword, Regex, QuotedString as QString, Suppress, Optional, Group, FollowedBy, Combine, operatorPrecedence, opAssoc, ParseException, - ParserElement, alphanums, And, OneOrMore) + ParserElement, alphanums, And, OneOrMore, delimitedList) from ..util import load_module @@ -17,19 +17,28 @@ class PrimitiveFactory(object): def build_from_conf(conf, parser): _cls = load_module(conf['class']) - if 'parse_method' in conf: - _kwargs = {'parse_method': getattr(parser, conf['parse_method'])} + _kwargs = {} + if not (set(conf.keys() - ['class']) <= set(_cls.__init__.__code__.co_varnames[1:])): + extra_parameters = set(conf.keys()) - (set(conf.keys()) & set(_cls.__init__.__code__.co_varnames[1:])) + raise PrimitiveFactoryError( + 'Invalid primitive definition: configuration for class {} has extra parameters'.format( + _cls.__name__, extra_parameters)) - elif 'range_parse_method' in conf: - _kwargs = { - 'range_parse_method': getattr(parser, conf['range_parse_method']), - 'item_parse_method': getattr(parser, conf['item_parse_method']) - } + for var in _cls.__init__.__code__.co_varnames[1:]: + if var not in conf: + continue - else: - raise PrimitiveFactoryError("Invalid Primitive definition, parsing method invalid or not defined.") + if var.endswith('parse_method'): + try: + _kwargs[var] = getattr(parser, conf[var]) + except AttributeError: + raise PrimitiveFactoryError( + 'Invalid Primitive definition. Parser {} does not have a method {}'.format(parser, conf[var])) - return _cls(precedence=conf['precedence'], **_kwargs) + continue + + _kwargs[var] = conf[var] + return _cls(**_kwargs) def concatenate(elems, operator='OR', class_to_embed_elem=None): @@ -159,6 +168,32 @@ def __init__(self, parse_method=None, precedence=6): self.addParseAction(parse_method) +class Container(And, BaseType): + + name = 'container' + + def __init__(self, + parse_method=None, + int_parse_method=None, + part_str_parse_method=None, + qstr_parse_method=None, + delim=',', + precedence=8): + value = Integer(int_parse_method) ^ PartialString(part_str_parse_method) ^ QuotedString(qstr_parse_method) + And.__init__( + self, + [ + Suppress(Literal('[')) + + Group(delimitedList(value, delim=delim) + Optional(Suppress(Literal(delim)))) + + Suppress(Literal(']')) + ], + ) + BaseType.__init__(self, precedence) + + if parse_method: + self.addParseAction(parse_method) + + class IntegerComparison(And, BaseType): name = 'integer_comparison' diff --git a/plyse/grammar.py b/plyse/grammar.py index bc43b19..753457a 100644 --- a/plyse/grammar.py +++ b/plyse/grammar.py @@ -13,6 +13,11 @@ class GrammarError(Exception): class GrammarFactory(object): + default_operators = [ + Operator("not", ['!', '-', 'not']), + Operator('and', ['+', 'and']), + Operator('or', ['or'], True)] + @staticmethod def build(term, parser, operators, keywords=None): return Grammar(term=term, operators=operators, term_parser=parser, keywords=keywords) @@ -20,10 +25,9 @@ def build(term, parser, operators, keywords=None): @staticmethod def build_default(term_parser=None): t_parser = term_parser or TermParserFactory.build_default() - operators = [Operator("not", ['!', '-', 'not']), Operator('and', ['+', 'and']), Operator('or', ['or'], True)] term = TermFactory.build_default_term(t_parser) - return Grammar(operators=operators, term=term, keywords=[], term_parser=t_parser) + return Grammar(operators=GrammarFactory.default_operators, term=term, keywords=[], term_parser=t_parser) @staticmethod def build_from_conf(conf): @@ -127,7 +131,8 @@ def add_keyword(self, keyword): def add_value_type(self, value): if not isinstance(value, ParserElement): - raise GrammarError("Value types should be PyParsing ParserElements or plyse.expressions.primitives.BaseType") + raise GrammarError( + "Value types should be PyParsing ParserElements or plyse.expressions.primitives.BaseType") new_term = TermFactory.build_term(self._term.field, self._term.values + [value], *self._term.parseAction) self._update_grammar(new_term) diff --git a/plyse/term_parser.py b/plyse/term_parser.py index f70b29e..b03c68d 100644 --- a/plyse/term_parser.py +++ b/plyse/term_parser.py @@ -106,12 +106,29 @@ def field_parse(self, string, location, tokens): """ if tokens: fields = [f for f in "".join(tokens).split(":") if f] - t = fields if len(fields) > 1 else fields[0] + t = fields[0] field_value = self._field_name_aliases.get(t, t) return self._build_field_data(field_value, Term.ATTRIBUTE) + def multifield_parse(self, string, location, tokens): + """ + Fields are whatever comes before a separator and they are usually use for attribute/property matching. The value + of a field is parsed separately form the field name and it depends on the definition of the grammar and the + accepted/supported values. Thus this method receives a token list with . + + Combined or nested fields are allowed, the pattern would be: + ... + + > ej: address:zip:ABC1234 => token list would be ['address', ':', 'zip'] + """ + if tokens: + fields = [f for f in "".join(tokens).split(":") if f] + field_value = self._field_name_aliases.get(":".join(fields), ":".join(fields)) + + return self._build_field_data(field_value, Term.MULTI_ATTRIBUTE) + def integer_parse(self, string, location, tokens): if tokens: r = self._build_value_data(int(tokens[0]), Term.INT) @@ -146,6 +163,11 @@ def range_parse(self, string, location, tokens): return self._build_value_data([tokens[0][Term.VAL], tokens[2][Term.VAL]], Term.RANGE % tokens[0][Term.VAL_TYPE]) + def container_parse(self, string, location, tokens): + if tokens: + value = [one[Term.VAL] for one in tokens[0]] + return self._build_value_data(value, Term.CONTAINER) + class Term(dict): @@ -159,11 +181,13 @@ class Term(dict): GREATER_EQUAL_THAN = 'greater_equal_than' LOWER_THAN = 'lower_than' LOWER_EQUAL_THAN = 'lower_equal_than' + CONTAINER = "container" # field types KEYWORD = 'keyword' DEFAULT = 'default' ATTRIBUTE = 'attribute' + MULTI_ATTRIBUTE = 'multi_attribute' # term keys FIELD = 'field' diff --git a/plyse/tests/grammar_from_config_test.py b/plyse/tests/grammar_from_config_test.py index e8f6422..b31319e 100644 --- a/plyse/tests/grammar_from_config_test.py +++ b/plyse/tests/grammar_from_config_test.py @@ -9,7 +9,7 @@ 'term_parser': 'plyse.term_parser.TermParser', 'default_fields': ['default'], 'aliases': {}, - 'operators': [ + 'operators': [ {'not': { 'implicit': False, 'symbols': ['not', '-', '!'] @@ -54,7 +54,15 @@ 'precedence': 5, 'range_parse_method': 'range_parse', 'item_parse_method': 'integer_parse' - } + }, + { + "class": "plyse.expressions.primitives.Container", + "precedence": 10, + "parse_method": "container_parse", + "int_parse_method": "integer_parse", + "part_str_parse_method": "partial_string_parse", + "qstr_parse_method": "quoted_string_parse", + }, ] } } @@ -137,5 +145,6 @@ def test_remove_keyword_then_match_as_field(self): r = g.parse("is:something") self._check_values(r[0], 'is', 'something', Term.PARTIAL_STRING) + if __name__ == "__main__": unittest.main() diff --git a/plyse/tests/grammar_test.py b/plyse/tests/grammar_test.py index 95e2f08..c3aabd6 100644 --- a/plyse/tests/grammar_test.py +++ b/plyse/tests/grammar_test.py @@ -185,5 +185,6 @@ def test_parenthesis_OR_field_OR_field_AND_NOT_field(self): self._check_values(r[4][2][0], expected_val="NOT") self._check_values(r[4][2][1], "e", 0, Term.INT) + if __name__ == '__main__': unittest.main() diff --git a/plyse/tests/primitives_test.py b/plyse/tests/primitives_test.py index aec0c02..c6051eb 100644 --- a/plyse/tests/primitives_test.py +++ b/plyse/tests/primitives_test.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import unittest from plyse.expressions.primitives import * +from pyparsing import ParseResults class PrimitiveTester(unittest.TestCase): @@ -11,7 +12,11 @@ def assert_parsed_output(self, primitive, input_expected_val_dict): if exp: output = primitive.parseString(inp) for n, o in enumerate(output): + # Workaround for container primitive due Group returns not a list but a ParseResults + if isinstance(o, ParseResults): + o = o.asList() self.assertEqual(exp[n], o) + self.assertEqual(len(exp), len(output)) else: self.assertRaises(ParseException, primitive.parseString, inp) @@ -65,10 +70,10 @@ def test_field(self): self.assert_parsed_output(f, {'name>test': ['name', '>'], 'age>10': ['age', '>'], 'aa': None}) def test_multi_field(self): - mf = Field() + mf = MultiField() self.assert_parsed_output(mf, {'first:name:test': ['first', ':', 'name', ':'], 'age:10': ['age', ':']}) - mf = Field(field_separator='>') + mf = MultiField(field_separator='>') self.assert_parsed_output(mf, {'first>name>test': ['first', '>', 'name', '>'], 'age>10': ['age', '>']}) def test_any(self): @@ -77,11 +82,23 @@ def test_any(self): def test_integer_comparisson(self): ic = IntegerComparison() - self.assert_parsed_output(ic, {'<10': ['<', '10'], '<=10': ['<=', '10'], '>10': ['>', '10'], '>=10': ['>=', '10'], '">e"': None}) + self.assert_parsed_output(ic, {'<10': ['<', '10'], '<=10': ['<=', '10'], '>10': [ + '>', '10'], '>=10': ['>=', '10'], '">e"': None}) def test_string_proximity(self): sp = StringProximity() self.assert_parsed_output(sp, {"'hello world'~3": ['hello world', '~', '3']}) + def test_container(self): + c = Container() + self.assert_parsed_output(c, { + "[a]": [["a"]], + "[aa, bbbb]": [["aa", "bbbb"]], + "[a,b]": [["a", "b"]], + "[a,bbbbb]": [["a", "bbbbb"]], + "[1, a,]": [["1", "a"]], + "[a, '1']": [["a", "1"]]}) + + if __name__ == '__main__': unittest.main() diff --git a/plyse/tests/term_parser_test.py b/plyse/tests/term_parser_test.py index a286037..a5787de 100644 --- a/plyse/tests/term_parser_test.py +++ b/plyse/tests/term_parser_test.py @@ -3,7 +3,10 @@ import unittest from plyse.grammar import GrammarFactory from plyse.term_parser import TermParserFactory, Term -from plyse.expressions.primitives import IntegerComparison +from plyse.expressions.primitives import (IntegerComparison, Container, MultiField, Integer, + IntegerRange, PartialString, QuotedString) +from plyse.expressions.operators import Operator +from plyse.expressions.terms import TermFactory class TermParserTester(unittest.TestCase): @@ -20,8 +23,20 @@ def _init_and_parse(self, input_str): } term_parser = TermParserFactory.build_from_conf(conf) - g = GrammarFactory.build_default(term_parser) + values = [IntegerRange(range_parse_method=term_parser.range_parse, item_parse_method=term_parser.integer_parse), + Integer(parse_method=term_parser.integer_parse), + PartialString(parse_method=term_parser.partial_string_parse), + QuotedString(parse_method=term_parser.quoted_string_parse)] + term = TermFactory.build_term(MultiField(parse_method=term_parser.multifield_parse), + values=values, + parse_method=term_parser.term_parse) + g = GrammarFactory.build(term=term, parser=term_parser, operators=GrammarFactory.default_operators) g.add_value_type(IntegerComparison(term_parser.integer_comparison_parse)) + g.add_value_type(Container( + parse_method=term_parser.container_parse, + int_parse_method=term_parser.integer_parse, + part_str_parse_method=term_parser.partial_string_parse, + qstr_parse_method=term_parser.quoted_string_parse)) self.assertTrue(g.parse(input_str, True)) return g.parse(input_str, True)[0] @@ -38,6 +53,14 @@ def test_default_field_partial_text(self): r = self._init_and_parse("texto") self._check_values(r, ['name', 'description'], "texto", Term.PARTIAL_STRING) + def test_default_multifield_partial_text(self): + r = self._init_and_parse("texto:texto3:texto4") + self._check_values(r, 'texto:texto3', "texto4", Term.PARTIAL_STRING) + + def test_default_multifield_int(self): + r = self._init_and_parse("texto:texto3:1") + self._check_values(r, 'texto:texto3', 1, Term.INT) + def test_default_field_partial_text_with_wildcard(self): r = self._init_and_parse("'texto*'") self._check_values(r, ['name', 'description'], "texto*", Term.PARTIAL_STRING) @@ -79,5 +102,16 @@ def test_field_int_comparisson(self): r = self._init_and_parse("age:<=18") self._check_values(r, 'age', 18, Term.LOWER_EQUAL_THAN) + def test_filed_container(self): + r = self._init_and_parse("field:[a,b,c]") + self._check_values(r, 'field', ["a", "b", "c"], Term.CONTAINER) + + r = self._init_and_parse("field:[1,b,c,]") + self._check_values(r, 'field', [1, "b", "c"], Term.CONTAINER) + + r = self._init_and_parse("field:[aa,b,'1',]") + self._check_values(r, 'field', ["aa", "b", "1"], Term.CONTAINER) + + if __name__ == '__main__': unittest.main() diff --git a/plyse/tests/terms_test.py b/plyse/tests/terms_test.py index 5fb60c5..ab566a0 100644 --- a/plyse/tests/terms_test.py +++ b/plyse/tests/terms_test.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- import unittest +from pyparsing import ParseResults from plyse.expressions.terms import * from plyse.expressions.primitives import * @@ -11,11 +12,20 @@ def assert_parsed_output(self, primitive, input_list): for inp in input_list: output = primitive.parseString(inp)[0] for o in output: + # Workaround for ParseResults in Container primitive + if isinstance(o, ParseResults): + o = str(o.asList()) self.assertIn(o, inp) def test_term(self): - t = TermFactory.build_term(Field(), [Integer(), QuotedString(), PartialString()]) - self.assert_parsed_output(t, ["name:tester", "age:30", "nickname:'test'", "freetext"]) + t = TermFactory.build_term(Field(), [Integer(), QuotedString(), PartialString(), Container()]) + self.assert_parsed_output(t, [ + "name:tester", "age:30", "nickname:'test'", "freetext", "department:['qa', 'dev']"]) + + def test_multifiled_term(self): + t = TermFactory.build_term(MultiField(), [Integer(), QuotedString(), PartialString(), Container()]) + self.assert_parsed_output(t, [ + "name:last:tester", "age:30", "nickname:'test'", "freetext", "department:['qa', 'dev']"]) def test_keyword(self): k = KeywordTerm("has", ["message", "comment", "notification"]) @@ -41,6 +51,6 @@ def test_keyword(self): k = KeywordTerm("has", ["message", "comment", "notification"], allow_other_values=False) self.assertRaises(ParseException, k.parseString, "has:10") + if __name__ == '__main__': unittest.main() - diff --git a/requirements.txt b/requirements.txt index c2356ab..6e2e33a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pyparsing==2.3.1 +pyparsing<3.0.0 diff --git a/setup.py b/setup.py index 9af2b88..eac7d77 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,10 @@ def get_version(package): setup( name='plyse', setup_requires=[ - 'pyparsing', + 'pyparsing<3.0.0', ], install_requires=[ - 'pyparsing', + 'pyparsing<3.0.0', ], version=get_version('plyse'), url='https://github.com/sebastiandev/plyse',