From 3ed300930a16f6b65b3b6c2792b0f1749f573920 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 25 Apr 2015 23:34:18 +0300 Subject: [PATCH 01/17] Add a BaseValidationError abstract class and move error-related stuff to a new errors module --- valideer/__init__.py | 3 +- valideer/base.py | 65 ++-------------------------- valideer/errors.py | 70 +++++++++++++++++++++++++++++++ valideer/tests/test_validators.py | 2 +- valideer/validators.py | 3 +- 5 files changed, 79 insertions(+), 64 deletions(-) create mode 100644 valideer/errors.py diff --git a/valideer/__init__.py b/valideer/__init__.py index 83cdcf6..c106dac 100644 --- a/valideer/__init__.py +++ b/valideer/__init__.py @@ -1,2 +1,3 @@ from .base import * -from .validators import * +from .errors import * +from .validators import * diff --git a/valideer/base.py b/valideer/base.py index 75d31d6..5ca14a9 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -2,12 +2,14 @@ from contextlib import contextmanager from threading import RLock from decorator import decorator + from .compat import with_metaclass +from .errors import SchemaError, ValidationError + __all__ = [ - "ValidationError", "SchemaError", "Validator", "accepts", "returns", "adapts", "parse", "parsing", "register", "register_factory", - "set_name_for_types", "reset_type_names", + "Validator", "accepts", "returns", "adapts", ] _NAMED_VALIDATORS = {} @@ -15,48 +17,6 @@ _VALIDATOR_FACTORIES_LOCK = RLock() -class SchemaError(Exception): - """An object cannot be parsed as a validator.""" - - -class ValidationError(ValueError): - """A value is invalid for a given validator.""" - - _UNDEFINED = object() - - def __init__(self, msg, value=_UNDEFINED): - self.msg = msg - self.value = value - self.context = [] - super(ValidationError, self).__init__() - - def __str__(self): - return self.to_string() - - @property - def message(self): - return self.to_string() - - @property - def args(self): - return (self.to_string(),) - - def to_string(self, repr_value=repr): - msg = self.msg - if self.value is not self._UNDEFINED: - msg = "Invalid value %s (%s): %s" % (repr_value(self.value), - get_type_name(self.value.__class__), - msg) - if self.context: - msg += " (at %s)" % "".join("[%r]" % context if i > 0 else str(context) - for i, context in enumerate(reversed(self.context))) - return msg - - def add_context(self, context): - self.context.append(context) - return self - - def parse(obj, required_properties=None, additional_properties=None): """Try to parse the given ``obj`` as a validator instance. @@ -335,20 +295,3 @@ def adapting(func, *args, **kwargs): return func(*adapted_posargs, **adapted_keywords) return adapting - - -_TYPE_NAMES = {} - - -def set_name_for_types(name, *types): - """Associate one or more types with an alternative human-friendly name.""" - for t in types: - _TYPE_NAMES[t] = name - - -def reset_type_names(): - _TYPE_NAMES.clear() - - -def get_type_name(type): - return _TYPE_NAMES.get(type) or type.__name__ diff --git a/valideer/errors.py b/valideer/errors.py new file mode 100644 index 0000000..cee829e --- /dev/null +++ b/valideer/errors.py @@ -0,0 +1,70 @@ +__all__ = [ + "SchemaError", + "BaseValidationError", "ValidationError", + "set_name_for_types", "reset_type_names", +] + +_TYPE_NAMES = {} + + +def set_name_for_types(name, *types): + """Associate one or more types with an alternative human-friendly name.""" + for t in types: + _TYPE_NAMES[t] = name + + +def reset_type_names(): + _TYPE_NAMES.clear() + + +def get_type_name(type): + return _TYPE_NAMES.get(type) or type.__name__ + + +class SchemaError(Exception): + """An object cannot be parsed as a validator.""" + + +class BaseValidationError(ValueError): + """Abstract base class of all validation errors.""" + + def __str__(self): + return self.to_string() + + @property + def message(self): + return self.to_string() + + @property + def args(self): + return (self.to_string(),) + + def to_string(self, repr_value=repr): + raise NotImplementedError('Abstract method') + + +class ValidationError(BaseValidationError): + """A value is invalid for a given validator.""" + + _UNDEFINED = object() + + def __init__(self, msg, value=_UNDEFINED): + self.msg = msg + self.value = value + self.context = [] + super(ValidationError, self).__init__() + + def to_string(self, repr_value=repr): + msg = self.msg + if self.value is not self._UNDEFINED: + msg = "Invalid value %s (%s): %s" % (repr_value(self.value), + get_type_name(self.value.__class__), + msg) + if self.context: + msg += " (at %s)" % "".join("[%r]" % context if i > 0 else str(context) + for i, context in enumerate(reversed(self.context))) + return msg + + def add_context(self, context): + self.context.append(context) + return self diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 23ee462..ee35583 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -29,7 +29,7 @@ class TestValidator(unittest.TestCase): def setUp(self): V.Object.REQUIRED_PROPERTIES = True - V.base.reset_type_names() + V.reset_type_names() self.complex_validator = self.parse({ "n": "number", "?i": V.Nullable("integer", 0), diff --git a/valideer/validators.py b/valideer/validators.py index d48fd07..2ec8b2c 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -1,4 +1,5 @@ -from .base import Validator, ValidationError, parse, get_type_name +from .errors import get_type_name +from .base import Validator, ValidationError, parse from .compat import string_types, izip, imap, iteritems import collections import datetime From 3bf123c29e9910eac21e88d15bbeb8079f640fb5 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 00:04:06 +0300 Subject: [PATCH 02/17] Add MultipleValidationError class --- valideer/errors.py | 30 +++++++++++++++++++++++++++--- valideer/tests/test_validators.py | 24 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/valideer/errors.py b/valideer/errors.py index cee829e..518d461 100644 --- a/valideer/errors.py +++ b/valideer/errors.py @@ -1,6 +1,6 @@ __all__ = [ "SchemaError", - "BaseValidationError", "ValidationError", + "BaseValidationError", "ValidationError", "MultipleValidationError", "set_name_for_types", "reset_type_names", ] @@ -40,7 +40,7 @@ def args(self): return (self.to_string(),) def to_string(self, repr_value=repr): - raise NotImplementedError('Abstract method') + raise NotImplementedError("Abstract method") class ValidationError(BaseValidationError): @@ -49,10 +49,10 @@ class ValidationError(BaseValidationError): _UNDEFINED = object() def __init__(self, msg, value=_UNDEFINED): + super(ValidationError, self).__init__() self.msg = msg self.value = value self.context = [] - super(ValidationError, self).__init__() def to_string(self, repr_value=repr): msg = self.msg @@ -68,3 +68,27 @@ def to_string(self, repr_value=repr): def add_context(self, context): self.context.append(context) return self + + +class MultipleValidationError(BaseValidationError): + """Encapsulates multiple validation errors for a given value.""" + + def __init__(self, *errors): + super(MultipleValidationError, self).__init__() + self.errors = [] + self.add_errors(*errors) + + def to_string(self, repr_value=repr): + lines = ["Multiple validation errors:"] + lines.extend("- " + e.to_string(repr_value) for e in self.errors) + return "\n".join(lines) + + def add_errors(self, *errors): + for error in errors: + if isinstance(error, MultipleValidationError): + self.errors.extend(error.errors) + elif isinstance(error, BaseValidationError): + self.errors.append(error) + else: + raise TypeError("error should be a BaseValidationError instance; " + "%r given" % error.__class__.__name__) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index ee35583..7601f3e 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -871,6 +871,30 @@ def test_error_message_json_type_names(self): ({"foo": 3, "opt": 12}, "Invalid value 12 (integer): must be string (at opt)")]) + def test_multiple_multiple_validation_error_message(self): + ex = V.MultipleValidationError( + V.ValidationError('More cowbell', 'moo'), + V.MultipleValidationError( + V.ValidationError('Less blink', 'blink'), + V.ValidationError('More cowbell', 'mooooo'), + ), + V.ValidationError('Boring', 'stuff'), + ) + self.assertEqual(len(ex.errors), 4) + self.assertEqual(str(ex), "Multiple validation errors:\n" + "- Invalid value 'moo' (str): More cowbell\n" + "- Invalid value 'blink' (str): Less blink\n" + "- Invalid value 'mooooo' (str): More cowbell\n" + "- Invalid value 'stuff' (str): Boring") + with self.assertRaises(TypeError): + V.MultipleValidationError( + V.ValidationError('More cowbell'), + V.MultipleValidationError( + ValueError('oops'), + V.ValidationError('Less cowbell'), + ), + ) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) From 389bd92b691b3799a66e2f3ba7b0f131b32ff98f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 01:10:27 +0300 Subject: [PATCH 03/17] Introduce Validator.full_validate() for accumulating all validation errors instead of raising just the first one --- valideer/base.py | 31 ++++++++++++++++++++++++++++--- valideer/tests/test_validators.py | 25 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 5ca14a9..5f07c42 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -4,7 +4,12 @@ from decorator import decorator from .compat import with_metaclass -from .errors import SchemaError, ValidationError +from .errors import ( + SchemaError, + BaseValidationError, + ValidationError, + MultipleValidationError, +) __all__ = [ @@ -178,7 +183,9 @@ class Validator(object): name = None def validate(self, value, adapt=True): - """Check if ``value`` is valid and if so adapt it. + """ + Check if ``value`` is valid and if so adapt it, otherwise raise a + ``ValidationError`` for the first encountered error. :param adapt: If ``False``, it indicates that the caller is interested only on whether ``value`` is valid, not on adapting it. This is @@ -190,6 +197,24 @@ def validate(self, value, adapt=True): """ raise NotImplementedError + def full_validate(self, value, adapt=True): + """ + Same as ``validate`` but raise ``MultipleValidationError`` that holds + all validation errors if ``value`` is invalid. + + :param adapt: If ``False``, it indicates that the caller is interested + only on whether ``value`` is valid, not on adapting it. This is + essentially an optimization hint for cases that validation can be + done more efficiently than adaptation. + + :raises MultipleValidationError: If ``value`` is invalid. + :returns: The adapted value if ``adapt`` is ``True``, otherwise anything. + """ + try: + return self.validate(value, adapt) + except ValidationError as ex: + raise MultipleValidationError(ex) + def is_valid(self, value): """Check if the value is valid. @@ -198,7 +223,7 @@ def is_valid(self, value): try: self.validate(value, adapt=False) return True - except ValidationError: + except BaseValidationError: return False def error(self, value): diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 7601f3e..8a41e7d 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -895,6 +895,19 @@ def test_multiple_multiple_validation_error_message(self): ), ) + def test_full_validate_single_error(self): + obj = {"+foo": "number", "?bar": ["integer"]} + self._testFullValidationErrors(obj, 42, + ["Invalid value 42 (int): must be Mapping"]) + self._testFullValidationErrors(obj, {}, + ["Invalid value {} (dict): missing required properties: ['foo']"]) + self._testFullValidationErrors(obj, {"foo": "3"}, + ["Invalid value '3' (str): must be number (at foo)"]) + self._testFullValidationErrors(obj, {"foo": 3, "bar": None}, + ["Invalid value None (NoneType): must be Sequence (at bar)"]) + self._testFullValidationErrors(obj, {"foo": 3, "bar": [1, "2", 3]}, + ["Invalid value '2' (str): must be integer (at bar[1])"]) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) @@ -914,7 +927,17 @@ def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), validator.validate(value) except V.ValidationError as ex: error_repr = ex.to_string(error_value_repr) - self.assertEqual(error_repr, error, "Actual error: %r" % error_repr) + self.assertEqual(error_repr, error) + + def _testFullValidationErrors(self, obj, value, errors, error_value_repr=repr): + validator = self.parse(obj) + try: + validator.full_validate(value) + except V.MultipleValidationError as ex: + self.assertEqual(len(ex.errors), len(errors)) + for sub_ex, error in zip(ex.errors, errors): + sub_ex_repr = sub_ex.to_string(error_value_repr) + self.assertEqual(sub_ex_repr, error) class TestValidatorModuleParse(TestValidator): From 367392fc2d216a7458c92ecc350ea868cd03fa3b Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 17:04:07 +0300 Subject: [PATCH 04/17] Add MultipleValidationError.add_context() and merge BaseValidationError into ValidationError --- valideer/base.py | 9 ++----- valideer/errors.py | 43 ++++++++++++++----------------- valideer/tests/test_validators.py | 18 ++++++------- 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 5f07c42..7c487fd 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -4,12 +4,7 @@ from decorator import decorator from .compat import with_metaclass -from .errors import ( - SchemaError, - BaseValidationError, - ValidationError, - MultipleValidationError, -) +from .errors import SchemaError, ValidationError, MultipleValidationError __all__ = [ @@ -223,7 +218,7 @@ def is_valid(self, value): try: self.validate(value, adapt=False) return True - except BaseValidationError: + except ValidationError: return False def error(self, value): diff --git a/valideer/errors.py b/valideer/errors.py index 518d461..fad0977 100644 --- a/valideer/errors.py +++ b/valideer/errors.py @@ -1,6 +1,5 @@ __all__ = [ - "SchemaError", - "BaseValidationError", "ValidationError", "MultipleValidationError", + "SchemaError", "ValidationError", "MultipleValidationError", "set_name_for_types", "reset_type_names", ] @@ -25,8 +24,15 @@ class SchemaError(Exception): """An object cannot be parsed as a validator.""" -class BaseValidationError(ValueError): - """Abstract base class of all validation errors.""" +class ValidationError(ValueError): + """A value is invalid for a given validator.""" + + _UNDEFINED = object() + + def __init__(self, msg, value=_UNDEFINED): + self.msg = msg + self.value = value + self.context = [] def __str__(self): return self.to_string() @@ -39,21 +45,6 @@ def message(self): def args(self): return (self.to_string(),) - def to_string(self, repr_value=repr): - raise NotImplementedError("Abstract method") - - -class ValidationError(BaseValidationError): - """A value is invalid for a given validator.""" - - _UNDEFINED = object() - - def __init__(self, msg, value=_UNDEFINED): - super(ValidationError, self).__init__() - self.msg = msg - self.value = value - self.context = [] - def to_string(self, repr_value=repr): msg = self.msg if self.value is not self._UNDEFINED: @@ -70,11 +61,10 @@ def add_context(self, context): return self -class MultipleValidationError(BaseValidationError): +class MultipleValidationError(ValidationError): """Encapsulates multiple validation errors for a given value.""" def __init__(self, *errors): - super(MultipleValidationError, self).__init__() self.errors = [] self.add_errors(*errors) @@ -83,12 +73,17 @@ def to_string(self, repr_value=repr): lines.extend("- " + e.to_string(repr_value) for e in self.errors) return "\n".join(lines) + def add_context(self, context): + for error in self.errors: + error.add_context(context) + return self + def add_errors(self, *errors): for error in errors: if isinstance(error, MultipleValidationError): self.errors.extend(error.errors) - elif isinstance(error, BaseValidationError): + elif isinstance(error, ValidationError): self.errors.append(error) else: - raise TypeError("error should be a BaseValidationError instance; " - "%r given" % error.__class__.__name__) + raise TypeError("ValidationError instance expected, %r given" + % error.__class__.__name__) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 8a41e7d..4b3431d 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -871,21 +871,21 @@ def test_error_message_json_type_names(self): ({"foo": 3, "opt": 12}, "Invalid value 12 (integer): must be string (at opt)")]) - def test_multiple_multiple_validation_error_message(self): + def test_multiple_validation_error_message(self): ex = V.MultipleValidationError( - V.ValidationError('More cowbell', 'moo'), + V.ValidationError('More cowbell', 'moo').add_context(0), V.MultipleValidationError( - V.ValidationError('Less blink', 'blink'), - V.ValidationError('More cowbell', 'mooooo'), + V.ValidationError('Less blink', 'blink').add_context('x').add_context('1'), + V.ValidationError('More cowbell', 'mooooo').add_context('y').add_context('1'), ), - V.ValidationError('Boring', 'stuff'), + V.ValidationError('Boring', 'stuff').add_context(2), ) self.assertEqual(len(ex.errors), 4) self.assertEqual(str(ex), "Multiple validation errors:\n" - "- Invalid value 'moo' (str): More cowbell\n" - "- Invalid value 'blink' (str): Less blink\n" - "- Invalid value 'mooooo' (str): More cowbell\n" - "- Invalid value 'stuff' (str): Boring") + "- Invalid value 'moo' (str): More cowbell (at 0)\n" + "- Invalid value 'blink' (str): Less blink (at 1['x'])\n" + "- Invalid value 'mooooo' (str): More cowbell (at 1['y'])\n" + "- Invalid value 'stuff' (str): Boring (at 2)") with self.assertRaises(TypeError): V.MultipleValidationError( V.ValidationError('More cowbell'), From 946dff8f86a2049212b7291c0eaee477b5c45a43 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 17:59:16 +0300 Subject: [PATCH 05/17] Add convenient FullValidator base class for validators that need to report multiple errors --- valideer/base.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 7c487fd..c6e7eab 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -194,8 +194,11 @@ def validate(self, value, adapt=True): def full_validate(self, value, adapt=True): """ - Same as ``validate`` but raise ``MultipleValidationError`` that holds - all validation errors if ``value`` is invalid. + Same as :py:meth:`validate` but raise :py:class:`MultipleValidationError` + that holds all validation errors if ``value`` is invalid. + + The default implementation simply calls :py:meth:`validate` and wraps a + :py:class:`ValidationError` into a :py:class:`MultipleValidationError`. :param adapt: If ``False``, it indicates that the caller is interested only on whether ``value`` is valid, not on adapting it. This is @@ -240,6 +243,54 @@ def humanized_name(self): register_factory = staticmethod(register_factory) +class FullValidator(Validator): + """ + Handy abstract base class for validators that need to report multiple + errors without duplicating the logic between :py:meth:`validate` and + :py:meth:`full_validate` or making the former less efficient than necessary + by delegating to the latter. + + Concrete subclasses need to implement just :py:meth:`_iter_errors` as a + generator that: + 1. Yields all validation errors. + 2. Raises a :py:class:`_Value` exception if the value is valid. In this case + (and only if no errors have been yielded) the returned (possibly adapted) + value is the ``message`` of the :py:class:`_Value` exception. If no errors + are yielded and no :py:class:`_Value` exception is raised, the input + ``value`` is considered valid and returned. + """ + + def validate(self, value, adapt=True): + try: + error = self._iter_errors(value, adapt, full=False).next() + except self._Value as ex: + return ex.message + except StopIteration: + return value + else: + raise error + + def full_validate(self, value, adapt=True): + errors = [] + ex = None + try: + for error in self._iter_errors(value, adapt, full=True): + errors.append(error) + except self._Value as ex: + pass + + if errors: + raise MultipleValidationError(*errors) + + return ex.message if ex else value + + def _iter_errors(self, value, adapt, full): + raise NotImplementedError + + class _Value(Exception): + pass + + def accepts(**schemas): """Create a decorator for validating function parameters. From 5828cb16da1ab49fb94e704907b3e241d7c32aec Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 18:09:13 +0300 Subject: [PATCH 06/17] Change HomogeneousSequence to a FullValidator --- valideer/tests/test_validators.py | 22 +++++++++-- valideer/validators.py | 65 ++++++++++++++++++++----------- 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 4b3431d..3d88cce 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -5,6 +5,7 @@ import json import re import unittest + import valideer as V from valideer.compat import long, unicode, xrange, string_types, int_types @@ -908,6 +909,15 @@ def test_full_validate_single_error(self): self._testFullValidationErrors(obj, {"foo": 3, "bar": [1, "2", 3]}, ["Invalid value '2' (str): must be integer (at bar[1])"]) + def test_full_validate_homogeneous_sequence(self): + obj = [{"foo": ["string"]}] + value = [1, {"foo": 2.5}, {"foo": ["x", True, "y"]}] + self._testFullValidationErrors(obj, value, errors=[ + "Invalid value 1 (int): must be Mapping (at 0)", + "Invalid value 2.5 (float): must be Sequence (at 1['foo'])", + "Invalid value True (bool): must be string (at 2['foo'][1])", + ]) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) @@ -934,10 +944,14 @@ def _testFullValidationErrors(self, obj, value, errors, error_value_repr=repr): try: validator.full_validate(value) except V.MultipleValidationError as ex: - self.assertEqual(len(ex.errors), len(errors)) - for sub_ex, error in zip(ex.errors, errors): - sub_ex_repr = sub_ex.to_string(error_value_repr) - self.assertEqual(sub_ex_repr, error) + found_errors = ex.errors + else: + found_errors = [] + + self.assertEqual(len(found_errors), len(errors)) + for found_error, error in zip(found_errors, errors): + found_error_repr = found_error.to_string(error_value_repr) + self.assertEqual(found_error_repr, error) class TestValidatorModuleParse(TestValidator): diff --git a/valideer/validators.py b/valideer/validators.py index 2ec8b2c..e788f99 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -1,12 +1,15 @@ -from .errors import get_type_name -from .base import Validator, ValidationError, parse -from .compat import string_types, izip, imap, iteritems import collections import datetime import inspect +import itertools import numbers import re +from .base import Validator, FullValidator, ValidationError, parse +from .compat import string_types, izip, imap, iteritems +from .errors import get_type_name + + __all__ = [ "AnyOf", "AllOf", "ChainOf", "Nullable", "NonNullable", "Enum", "Condition", "AdaptBy", "AdaptTo", @@ -16,6 +19,14 @@ ] +def _partition_pairs(iterable): + """Partition a (bool, value) iterable into an iterable of false and true values""" + t1, t2 = itertools.tee(iterable) + false_it = (v for p, v in t1 if not p) + true_it = (v for p, v in t2 if p) + return false_it, true_it + + class AnyOf(Validator): """A composite validator that accepts values accepted by any of its component validators. @@ -464,18 +475,16 @@ def _PatternFactory(obj): return Pattern(obj) -class HomogeneousSequence(Type): +class HomogeneousSequence(FullValidator): """A validator that accepts homogeneous, non-fixed size sequences.""" - accept_types = collections.Sequence - reject_types = string_types - def __init__(self, item_schema=None, min_length=None, max_length=None): """Instantiate a :py:class:`HomogeneousSequence` validator. :param item_schema: If not None, the schema of the items of the list. """ - super(HomogeneousSequence, self).__init__() + self._type_validator = Type(accept_types=collections.Sequence, + reject_types=string_types) if item_schema is not None: self._item_validator = parse(item_schema) else: @@ -483,28 +492,40 @@ def __init__(self, item_schema=None, min_length=None, max_length=None): self._min_length = min_length self._max_length = max_length - def validate(self, value, adapt=True): - super(HomogeneousSequence, self).validate(value) + def _iter_errors(self, value, adapt, full): + try: + self._type_validator.validate(value) + except ValidationError as ex: + yield ex + return + if self._min_length is not None and len(value) < self._min_length: - raise ValidationError("must contain at least %d elements" % + yield ValidationError("must contain at least %d elements" % self._min_length, value) + if self._max_length is not None and len(value) > self._max_length: - raise ValidationError("must contain at most %d elements" % + yield ValidationError("must contain at most %d elements" % self._max_length, value) - if self._item_validator is None: - return value - if adapt: - return value.__class__(self._iter_validated_items(value, adapt)) - for _ in self._iter_validated_items(value, adapt): - pass - def _iter_validated_items(self, value, adapt): - validate_item = self._item_validator.validate + if self._item_validator is not None: + iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) + item_errors, items = _partition_pairs(iter_pairs) + for item_error in item_errors: + yield item_error + if adapt: + raise self._Value(value.__class__(items)) + + def _iter_validated_items_and_errors(self, value, adapt, full): + if full: + validate_item = self._item_validator.full_validate + else: + validate_item = self._item_validator.validate + for i, item in enumerate(value): try: - yield validate_item(item, adapt) + yield (True, validate_item(item, adapt=adapt)) except ValidationError as ex: - raise ex.add_context(i) + yield (False, ex.add_context(i)) @HomogeneousSequence.register_factory From 7ed122161cdb55b8353313902d2b694eec87c031 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 18:27:21 +0300 Subject: [PATCH 07/17] Use next() builtin for Python3 compatibility --- valideer/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/valideer/base.py b/valideer/base.py index c6e7eab..8962974 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -262,7 +262,7 @@ class FullValidator(Validator): def validate(self, value, adapt=True): try: - error = self._iter_errors(value, adapt, full=False).next() + error = next(self._iter_errors(value, adapt, full=False)) except self._Value as ex: return ex.message except StopIteration: From 7972893f6e8ea2bdfd18a189ccbe128ce601d42d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 19:10:31 +0300 Subject: [PATCH 08/17] Replaced custom compat module with six dependency --- setup.py | 2 +- valideer/base.py | 5 ++--- valideer/compat.py | 32 ------------------------------- valideer/tests/test_validators.py | 20 +++++++++++-------- valideer/validators.py | 17 ++++++++-------- 5 files changed, 24 insertions(+), 52 deletions(-) delete mode 100644 valideer/compat.py diff --git a/setup.py b/setup.py index 7f6ae03..40ecf20 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ author="George Sakkis", author_email="george.sakkis@gmail.com", packages=find_packages(), - install_requires=["decorator"], + install_requires=["decorator", "six"], test_suite="valideer.tests", platforms=["any"], keywords="validation adaptation typechecking jsonschema", diff --git a/valideer/base.py b/valideer/base.py index 8962974..4782fbf 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -3,7 +3,7 @@ from threading import RLock from decorator import decorator -from .compat import with_metaclass +from six import with_metaclass from .errors import SchemaError, ValidationError, MultipleValidationError @@ -166,8 +166,7 @@ def __new__(mcs, name, bases, attrs): # @NoSelf return validator_type -@with_metaclass(_MetaValidator) -class Validator(object): +class Validator(with_metaclass(_MetaValidator)): """Abstract base class of all validators. Concrete subclasses must implement :py:meth:`validate`. A subclass may optionally diff --git a/valideer/compat.py b/valideer/compat.py deleted file mode 100644 index d98b19a..0000000 --- a/valideer/compat.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys - -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY2: # pragma: no cover - string_types = basestring - int_types = (int, long) - from itertools import izip, imap - long = long - unicode = unicode - xrange = xrange - iteritems = dict.iteritems -else: # pragma: no cover - string_types = str - int_types = (int,) - izip = zip - imap = map - long = int - unicode = str - iteritems = dict.items - xrange = range - - -def with_metaclass(mcls): - def decorator(cls): - body = vars(cls).copy() - # clean out class body - body.pop('__dict__', None) - body.pop('__weakref__', None) - return mcls(cls.__name__, cls.__bases__, body) - return decorator diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 3d88cce..6fadec4 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -5,9 +5,13 @@ import json import re import unittest +from six import string_types, integer_types, binary_type, text_type, PY3 +from six.moves import xrange, zip import valideer as V -from valideer.compat import long, unicode, xrange, string_types, int_types + +if PY3: + long = int class Fraction(V.Type): @@ -40,7 +44,7 @@ def setUp(self): "?s": V.String(min_length=1, max_length=8), "?p": V.Nullable(re.compile(r"\d{1,4}$")), "?l": [{"+s2": "string"}], - "?t": (unicode, "number"), + "?t": (text_type, "number"), "?h": V.Mapping(int, ["string"]), "?o": V.NonNullable({"+i2": "integer"}), }) @@ -48,7 +52,7 @@ def setUp(self): def test_none(self): for obj in ["boolean", "integer", "number", "string", V.HomogeneousSequence, V.HeterogeneousSequence, - V.Mapping, V.Object, int, float, str, unicode, + V.Mapping, V.Object, int, float, binary_type, text_type, Fraction, Fraction(), Gender, Gender()]: self.assertFalse(self.parse(obj).is_valid(None)) @@ -763,7 +767,7 @@ def test_complex_adaptation(self): ]: adapted = self.complex_validator.validate(value) self.assertTrue(isinstance(adapted["n"], (int, long, float, Decimal))) - self.assertTrue(isinstance(adapted["i"], int_types)) + self.assertTrue(isinstance(adapted["i"], integer_types)) self.assertTrue(adapted.get("b") is None or isinstance(adapted["b"], bool)) self.assertTrue(adapted.get("d") is None or isinstance(adapted["d"], (date, datetime))) self.assertTrue(adapted.get("e") is None or adapted["e"] in "rgb") @@ -776,7 +780,7 @@ def test_complex_adaptation(self): for item in adapted["l"])) if adapted.get("t") is not None: self.assertEqual(len(adapted["t"]), 2) - self.assertTrue(isinstance(adapted["t"][0], unicode)) + self.assertTrue(isinstance(adapted["t"][0], text_type)) self.assertTrue(isinstance(adapted["t"][1], float)) if adapted.get("h") is not None: self.assertTrue(all(isinstance(key, int) @@ -785,7 +789,7 @@ def test_complex_adaptation(self): for value in adapted["h"].values() for value_item in value)) if adapted.get("o") is not None: - self.assertTrue(isinstance(adapted["o"]["i2"], int_types)) + self.assertTrue(isinstance(adapted["o"]["i2"], integer_types)) def test_humanized_names(self): class DummyValidator(V.Validator): @@ -847,7 +851,7 @@ def test_error_message_json_type_names(self): V.set_name_for_types("null", type(None)) V.set_name_for_types("integer", int, long) V.set_name_for_types("number", float) - V.set_name_for_types("string", str, unicode) + V.set_name_for_types("string", binary_type, text_type) V.set_name_for_types("array", list, collections.Sequence) V.set_name_for_types("object", dict, collections.Mapping) @@ -973,7 +977,7 @@ def setUp(self): "s": V.String(min_length=1, max_length=8), "p": V.Nullable(re.compile(r"\d{1,4}$")), "l": [{"+s2": "string"}], - "t": (unicode, "number"), + "t": (text_type, "number"), "h": V.Mapping(int, ["string"]), "o": V.NonNullable({"+i2": "integer"}), }) diff --git a/valideer/validators.py b/valideer/validators.py index e788f99..389d696 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -4,9 +4,10 @@ import itertools import numbers import re +from six import string_types, iteritems +from six.moves import zip, map from .base import Validator, FullValidator, ValidationError, parse -from .compat import string_types, izip, imap, iteritems from .errors import get_type_name @@ -36,7 +37,7 @@ class AnyOf(Validator): """ def __init__(self, *schemas): - self._validators = list(imap(parse, schemas)) + self._validators = list(map(parse, schemas)) def validate(self, value, adapt=True): msgs = [] @@ -60,7 +61,7 @@ class AllOf(Validator): """ def __init__(self, *schemas): - self._validators = list(imap(parse, schemas)) + self._validators = list(map(parse, schemas)) def validate(self, value, adapt=True): result = value @@ -80,7 +81,7 @@ class ChainOf(Validator): """ def __init__(self, *schemas): - self._validators = list(imap(parse, schemas)) + self._validators = list(map(parse, schemas)) def validate(self, value, adapt=True): for validator in self._validators: @@ -204,7 +205,7 @@ def validate(self, value, adapt=True): @property def humanized_name(self): - return "one of {%s}" % ", ".join(list(imap(repr, self.values))) + return "one of {%s}" % ", ".join(list(map(repr, self.values))) class Condition(Validator): @@ -550,7 +551,7 @@ def __init__(self, *item_schemas): :param item_schemas: The schema of each element of the the tuple. """ super(HeterogeneousSequence, self).__init__() - self._item_validators = list(imap(parse, item_schemas)) + self._item_validators = list(map(parse, item_schemas)) def validate(self, value, adapt=True): super(HeterogeneousSequence, self).validate(value) @@ -563,7 +564,7 @@ def validate(self, value, adapt=True): pass def _iter_validated_items(self, value, adapt): - for i, (validator, item) in enumerate(izip(self._item_validators, value)): + for i, (validator, item) in enumerate(zip(self._item_validators, value)): try: yield validator.validate(item, adapt) except ValidationError as ex: @@ -748,7 +749,7 @@ def _ObjectFactory(obj, required_properties=None, additional_properties=None): def _format_types(types): if inspect.isclass(types): types = (types,) - names = list(imap(get_type_name, types)) + names = list(map(get_type_name, types)) s = names[-1] if len(names) > 1: s = ", ".join(names[:-1]) + " or " + s From e979bcddd375e5e455eedde7bf53842985922985 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 19:28:43 +0300 Subject: [PATCH 09/17] Python 3 exceptions don't have a message attribute --- valideer/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 4782fbf..0889e13 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -254,7 +254,7 @@ class FullValidator(Validator): 1. Yields all validation errors. 2. Raises a :py:class:`_Value` exception if the value is valid. In this case (and only if no errors have been yielded) the returned (possibly adapted) - value is the ``message`` of the :py:class:`_Value` exception. If no errors + value is the ``args[0]`` of the :py:class:`_Value` exception. If no errors are yielded and no :py:class:`_Value` exception is raised, the input ``value`` is considered valid and returned. """ @@ -263,7 +263,7 @@ def validate(self, value, adapt=True): try: error = next(self._iter_errors(value, adapt, full=False)) except self._Value as ex: - return ex.message + return ex.args[0] except StopIteration: return value else: @@ -281,7 +281,7 @@ def full_validate(self, value, adapt=True): if errors: raise MultipleValidationError(*errors) - return ex.message if ex else value + return ex.args[0] if ex else value def _iter_errors(self, value, adapt, full): raise NotImplementedError From 858b86b072aaaac5bf8a6ff0a06055eb68fe952c Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 26 Apr 2015 19:44:37 +0300 Subject: [PATCH 10/17] Change HeterogeneousSequence to a FullValidator --- valideer/tests/test_validators.py | 9 ++++++++ valideer/validators.py | 35 ++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 6fadec4..87a982e 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -922,6 +922,15 @@ def test_full_validate_homogeneous_sequence(self): "Invalid value True (bool): must be string (at 2['foo'][1])", ]) + def test_full_validate_heterogeneous_sequence(self): + obj = ("integer", ("string", "number")) + value = ("3", (4, True)) + self._testFullValidationErrors(obj, value, errors=[ + "Invalid value '3' (str): must be integer (at 0)", + "Invalid value 4 (int): must be string (at 1[0])", + "Invalid value True (bool): must be number (at 1[1])", + ]) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) diff --git a/valideer/validators.py b/valideer/validators.py index 389d696..1f267b6 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -539,36 +539,43 @@ def _HomogeneousSequenceFactory(obj): return HomogeneousSequence(*obj) -class HeterogeneousSequence(Type): +class HeterogeneousSequence(FullValidator): """A validator that accepts heterogeneous, fixed size sequences.""" - accept_types = collections.Sequence - reject_types = string_types - def __init__(self, *item_schemas): """Instantiate a :py:class:`HeterogeneousSequence` validator. :param item_schemas: The schema of each element of the the tuple. """ - super(HeterogeneousSequence, self).__init__() + self._type_validator = Type(accept_types=collections.Sequence, + reject_types=string_types) self._item_validators = list(map(parse, item_schemas)) - def validate(self, value, adapt=True): - super(HeterogeneousSequence, self).validate(value) + def _iter_errors(self, value, adapt, full): + try: + self._type_validator.validate(value) + except ValidationError as ex: + yield ex + return + if len(value) != len(self._item_validators): - raise ValidationError("%d items expected, %d found" % + yield ValidationError("%d items expected, %d found" % (len(self._item_validators), len(value)), value) + + iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) + item_errors, items = _partition_pairs(iter_pairs) + for item_error in item_errors: + yield item_error if adapt: - return value.__class__(self._iter_validated_items(value, adapt)) - for _ in self._iter_validated_items(value, adapt): - pass + raise self._Value(value.__class__(items)) - def _iter_validated_items(self, value, adapt): + def _iter_validated_items_and_errors(self, value, adapt, full): + method = 'full_validate' if full else 'validate' for i, (validator, item) in enumerate(zip(self._item_validators, value)): try: - yield validator.validate(item, adapt) + yield (True, getattr(validator, method)(item, adapt=adapt)) except ValidationError as ex: - raise ex.add_context(i) + yield (False, ex.add_context(i)) @HeterogeneousSequence.register_factory From fffece62afdf8e1ff10b05f664b183269cbdd9ea Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 27 Apr 2015 02:02:26 +0300 Subject: [PATCH 11/17] Change Mapping to a FullValidator --- valideer/tests/test_validators.py | 18 +++++++++++ valideer/validators.py | 54 +++++++++++++++++++++++++------ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 87a982e..9219301 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -931,6 +931,24 @@ def test_full_validate_heterogeneous_sequence(self): "Invalid value True (bool): must be number (at 1[1])", ]) + def test_full_validate_mapping(self): + obj = V.Mapping("string", V.Mapping("integer", ["number"])) + value = { + 1: { + "a": [] + }, + "x": { + 0: [4.2, -1], + True: [1, "z", 2] + } + } + self._testFullValidationErrors(obj, value, errors=[ + "Invalid value 1 (int): must be string", + "Invalid value 'a' (str): must be integer (at 1)", + "Invalid value True (bool): must be integer (at x)", + "Invalid value 'z' (str): must be number (at x[True][1])", + ]) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) diff --git a/valideer/validators.py b/valideer/validators.py index 1f267b6..1c1700b 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -588,18 +588,16 @@ def _HeterogeneousSequenceFactory(obj): return HeterogeneousSequence(*obj) -class Mapping(Type): +class Mapping(FullValidator): """A validator that accepts mappings (:py:class:`collections.Mapping` instances).""" - accept_types = collections.Mapping - def __init__(self, key_schema=None, value_schema=None): """Instantiate a :py:class:`Mapping` validator. :param key_schema: If not None, the schema of the dict keys. :param value_schema: If not None, the schema of the dict values. """ - super(Mapping, self).__init__() + self._type_validator = Type(accept_types=collections.Mapping) if key_schema is not None: self._key_validator = parse(key_schema) else: @@ -609,12 +607,49 @@ def __init__(self, key_schema=None, value_schema=None): else: self._value_validator = None - def validate(self, value, adapt=True): - super(Mapping, self).validate(value) + def _iter_errors(self, value, adapt, full): + try: + self._type_validator.validate(value) + except ValidationError as ex: + yield ex + return + + iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) + item_errors, items = _partition_pairs(iter_pairs) + for item_error in item_errors: + yield item_error if adapt: - return dict(self._iter_validated_items(value, adapt)) - for _ in self._iter_validated_items(value, adapt): - pass + raise self._Value(dict(items)) + + def _iter_validated_items_and_errors(self, value, adapt, full): + if self._key_validator is None: + validate_key = None + elif full: + validate_key = self._key_validator.full_validate + else: + validate_key = self._key_validator.validate + + if self._value_validator is None: + validate_value = None + elif full: + validate_value = self._value_validator.full_validate + else: + validate_value = self._value_validator.validate + + for k, v in iteritems(value): + if validate_key is not None: + try: + k = validate_key(k, adapt) + except ValidationError as ex: + yield (False, ex) + + if validate_value is not None: + try: + v = validate_value(v, adapt) + except ValidationError as ex: + yield (False, ex.add_context(k)) + + yield (True, (k, v)) def _iter_validated_items(self, value, adapt): validate_key = validate_value = None @@ -622,6 +657,7 @@ def _iter_validated_items(self, value, adapt): validate_key = self._key_validator.validate if self._value_validator is not None: validate_value = self._value_validator.validate + for k, v in iteritems(value): if validate_value is not None: try: From 7b7080c5fba65cbea39d273dd81732846a8e3ca7 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 27 Apr 2015 02:36:14 +0300 Subject: [PATCH 12/17] Fix exception scope for Python 3 --- valideer/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 0889e13..832cc48 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -271,17 +271,16 @@ def validate(self, value, adapt=True): def full_validate(self, value, adapt=True): errors = [] - ex = None try: for error in self._iter_errors(value, adapt, full=True): errors.append(error) except self._Value as ex: - pass + value = ex.args[0] if errors: raise MultipleValidationError(*errors) - return ex.args[0] if ex else value + return value def _iter_errors(self, value, adapt, full): raise NotImplementedError From 1be1ac2098559631e0c5ac0bbbf9be5054e35925 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 29 Apr 2015 05:30:27 +0300 Subject: [PATCH 13/17] Replace FullValidator with ContainerValidator --- valideer/base.py | 69 ++++++++++++++++--------------- valideer/validators.py | 93 +++++++++++------------------------------- 2 files changed, 58 insertions(+), 104 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 832cc48..2925053 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -1,4 +1,5 @@ import inspect +import itertools from contextlib import contextmanager from threading import RLock from decorator import decorator @@ -242,51 +243,51 @@ def humanized_name(self): register_factory = staticmethod(register_factory) -class FullValidator(Validator): +class ContainerValidator(Validator): """ - Handy abstract base class for validators that need to report multiple - errors without duplicating the logic between :py:meth:`validate` and - :py:meth:`full_validate` or making the former less efficient than necessary - by delegating to the latter. - - Concrete subclasses need to implement just :py:meth:`_iter_errors` as a - generator that: - 1. Yields all validation errors. - 2. Raises a :py:class:`_Value` exception if the value is valid. In this case - (and only if no errors have been yielded) the returned (possibly adapted) - value is the ``args[0]`` of the :py:class:`_Value` exception. If no errors - are yielded and no :py:class:`_Value` exception is raised, the input - ``value`` is considered valid and returned. + Convenient abstract base class for validators of container-like values that + need to report multiple errors for their items without duplicating the logic + between :py:meth:`validate` and :py:meth:`full_validate` or making the + former less efficient than necessary by delegating to the latter. + + Concrete subclasses have to implement :py:meth:`_iter_errors_and_items` as a + generator that yields all validation errors and items of the container value. + If there are no validation errors and `adapt=True`, the final adapted value + is produced by passing the yielded items to :py:meth:`_reduce_items`. The + default :py:meth:`_reduce_items` instantiates `value.__class__` with the + iterator of items but subclasses can override it if necessary. """ def validate(self, value, adapt=True): - try: - error = next(self._iter_errors(value, adapt, full=False)) - except self._Value as ex: - return ex.args[0] - except StopIteration: - return value - else: - raise error + return self._validate(value, adapt, full=False) def full_validate(self, value, adapt=True): - errors = [] - try: - for error in self._iter_errors(value, adapt, full=True): - errors.append(error) - except self._Value as ex: - value = ex.args[0] + return self._validate(value, adapt, full=True) + + def _validate(self, value, adapt, full): + iterable = self._iter_errors_and_items(value, adapt, full) + t1, t2 = itertools.tee(iterable) + iter_errors = (x for x in t1 if isinstance(x, ValidationError)) + if full: + multi_error = MultipleValidationError(*iter_errors) + if multi_error.errors: + raise multi_error + else: + error = next(iter_errors, None) + if error: + raise error - if errors: - raise MultipleValidationError(*errors) + if adapt: + iter_items = (x for x in t2 if not isinstance(x, ValidationError)) + return self._reduce_items(iter_items, value) return value - def _iter_errors(self, value, adapt, full): - raise NotImplementedError + def _reduce_items(self, iterable, value): + return value.__class__(iterable) - class _Value(Exception): - pass + def _iter_errors_and_items(self, value, adapt, full): + raise NotImplementedError def accepts(**schemas): diff --git a/valideer/validators.py b/valideer/validators.py index 1c1700b..33ade2b 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -1,13 +1,12 @@ import collections import datetime import inspect -import itertools import numbers import re from six import string_types, iteritems from six.moves import zip, map -from .base import Validator, FullValidator, ValidationError, parse +from .base import Validator, ContainerValidator, ValidationError, parse from .errors import get_type_name @@ -20,14 +19,6 @@ ] -def _partition_pairs(iterable): - """Partition a (bool, value) iterable into an iterable of false and true values""" - t1, t2 = itertools.tee(iterable) - false_it = (v for p, v in t1 if not p) - true_it = (v for p, v in t2 if p) - return false_it, true_it - - class AnyOf(Validator): """A composite validator that accepts values accepted by any of its component validators. @@ -476,7 +467,7 @@ def _PatternFactory(obj): return Pattern(obj) -class HomogeneousSequence(FullValidator): +class HomogeneousSequence(ContainerValidator): """A validator that accepts homogeneous, non-fixed size sequences.""" def __init__(self, item_schema=None, min_length=None, max_length=None): @@ -493,7 +484,7 @@ def __init__(self, item_schema=None, min_length=None, max_length=None): self._min_length = min_length self._max_length = max_length - def _iter_errors(self, value, adapt, full): + def _iter_errors_and_items(self, value, adapt, full): try: self._type_validator.validate(value) except ValidationError as ex: @@ -508,25 +499,20 @@ def _iter_errors(self, value, adapt, full): yield ValidationError("must contain at most %d elements" % self._max_length, value) - if self._item_validator is not None: - iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) - item_errors, items = _partition_pairs(iter_pairs) - for item_error in item_errors: - yield item_error - if adapt: - raise self._Value(value.__class__(items)) - - def _iter_validated_items_and_errors(self, value, adapt, full): - if full: - validate_item = self._item_validator.full_validate + if self._item_validator is None: + for item in value: + yield item else: - validate_item = self._item_validator.validate + if full: + validate_item = self._item_validator.full_validate + else: + validate_item = self._item_validator.validate - for i, item in enumerate(value): - try: - yield (True, validate_item(item, adapt=adapt)) - except ValidationError as ex: - yield (False, ex.add_context(i)) + for i, item in enumerate(value): + try: + yield validate_item(item, adapt=adapt) + except ValidationError as ex: + yield ex.add_context(i) @HomogeneousSequence.register_factory @@ -539,7 +525,7 @@ def _HomogeneousSequenceFactory(obj): return HomogeneousSequence(*obj) -class HeterogeneousSequence(FullValidator): +class HeterogeneousSequence(ContainerValidator): """A validator that accepts heterogeneous, fixed size sequences.""" def __init__(self, *item_schemas): @@ -551,7 +537,7 @@ def __init__(self, *item_schemas): reject_types=string_types) self._item_validators = list(map(parse, item_schemas)) - def _iter_errors(self, value, adapt, full): + def _iter_errors_and_items(self, value, adapt, full): try: self._type_validator.validate(value) except ValidationError as ex: @@ -562,20 +548,12 @@ def _iter_errors(self, value, adapt, full): yield ValidationError("%d items expected, %d found" % (len(self._item_validators), len(value)), value) - iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) - item_errors, items = _partition_pairs(iter_pairs) - for item_error in item_errors: - yield item_error - if adapt: - raise self._Value(value.__class__(items)) - - def _iter_validated_items_and_errors(self, value, adapt, full): method = 'full_validate' if full else 'validate' for i, (validator, item) in enumerate(zip(self._item_validators, value)): try: - yield (True, getattr(validator, method)(item, adapt=adapt)) + yield getattr(validator, method)(item, adapt=adapt) except ValidationError as ex: - yield (False, ex.add_context(i)) + yield ex.add_context(i) @HeterogeneousSequence.register_factory @@ -588,7 +566,7 @@ def _HeterogeneousSequenceFactory(obj): return HeterogeneousSequence(*obj) -class Mapping(FullValidator): +class Mapping(ContainerValidator): """A validator that accepts mappings (:py:class:`collections.Mapping` instances).""" def __init__(self, key_schema=None, value_schema=None): @@ -607,21 +585,13 @@ def __init__(self, key_schema=None, value_schema=None): else: self._value_validator = None - def _iter_errors(self, value, adapt, full): + def _iter_errors_and_items(self, value, adapt, full): try: self._type_validator.validate(value) except ValidationError as ex: yield ex return - iter_pairs = self._iter_validated_items_and_errors(value, adapt, full) - item_errors, items = _partition_pairs(iter_pairs) - for item_error in item_errors: - yield item_error - if adapt: - raise self._Value(dict(items)) - - def _iter_validated_items_and_errors(self, value, adapt, full): if self._key_validator is None: validate_key = None elif full: @@ -641,31 +611,14 @@ def _iter_validated_items_and_errors(self, value, adapt, full): try: k = validate_key(k, adapt) except ValidationError as ex: - yield (False, ex) + yield ex if validate_value is not None: try: v = validate_value(v, adapt) except ValidationError as ex: - yield (False, ex.add_context(k)) - - yield (True, (k, v)) + yield ex.add_context(k) - def _iter_validated_items(self, value, adapt): - validate_key = validate_value = None - if self._key_validator is not None: - validate_key = self._key_validator.validate - if self._value_validator is not None: - validate_value = self._value_validator.validate - - for k, v in iteritems(value): - if validate_value is not None: - try: - v = validate_value(v, adapt) - except ValidationError as ex: - raise ex.add_context(k) - if validate_key is not None: - k = validate_key(k, adapt) yield (k, v) From 3486e024967209e85e82abe9f395806bc407d8ad Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 29 Apr 2015 05:44:11 +0300 Subject: [PATCH 14/17] Change signature of MultipleValidationError to accept iterable instead of varargs --- valideer/base.py | 4 ++-- valideer/errors.py | 6 +++--- valideer/tests/test_validators.py | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index 2925053..ece6dc3 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -211,7 +211,7 @@ def full_validate(self, value, adapt=True): try: return self.validate(value, adapt) except ValidationError as ex: - raise MultipleValidationError(ex) + raise MultipleValidationError([ex]) def is_valid(self, value): """Check if the value is valid. @@ -269,7 +269,7 @@ def _validate(self, value, adapt, full): t1, t2 = itertools.tee(iterable) iter_errors = (x for x in t1 if isinstance(x, ValidationError)) if full: - multi_error = MultipleValidationError(*iter_errors) + multi_error = MultipleValidationError(iter_errors) if multi_error.errors: raise multi_error else: diff --git a/valideer/errors.py b/valideer/errors.py index fad0977..eb81e46 100644 --- a/valideer/errors.py +++ b/valideer/errors.py @@ -64,9 +64,9 @@ def add_context(self, context): class MultipleValidationError(ValidationError): """Encapsulates multiple validation errors for a given value.""" - def __init__(self, *errors): + def __init__(self, errors): self.errors = [] - self.add_errors(*errors) + self.add_errors(errors) def to_string(self, repr_value=repr): lines = ["Multiple validation errors:"] @@ -78,7 +78,7 @@ def add_context(self, context): error.add_context(context) return self - def add_errors(self, *errors): + def add_errors(self, errors): for error in errors: if isinstance(error, MultipleValidationError): self.errors.extend(error.errors) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 9219301..29960cd 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -877,14 +877,14 @@ def test_error_message_json_type_names(self): "Invalid value 12 (integer): must be string (at opt)")]) def test_multiple_validation_error_message(self): - ex = V.MultipleValidationError( + ex = V.MultipleValidationError([ V.ValidationError('More cowbell', 'moo').add_context(0), - V.MultipleValidationError( + V.MultipleValidationError([ V.ValidationError('Less blink', 'blink').add_context('x').add_context('1'), V.ValidationError('More cowbell', 'mooooo').add_context('y').add_context('1'), - ), + ]), V.ValidationError('Boring', 'stuff').add_context(2), - ) + ]) self.assertEqual(len(ex.errors), 4) self.assertEqual(str(ex), "Multiple validation errors:\n" "- Invalid value 'moo' (str): More cowbell (at 0)\n" @@ -892,13 +892,13 @@ def test_multiple_validation_error_message(self): "- Invalid value 'mooooo' (str): More cowbell (at 1['y'])\n" "- Invalid value 'stuff' (str): Boring (at 2)") with self.assertRaises(TypeError): - V.MultipleValidationError( + V.MultipleValidationError([ V.ValidationError('More cowbell'), - V.MultipleValidationError( + V.MultipleValidationError([ ValueError('oops'), V.ValidationError('Less cowbell'), - ), - ) + ]), + ]) def test_full_validate_single_error(self): obj = {"+foo": "number", "?bar": ["integer"]} From 1cc76733b4122c80e8cc23d004d4c0d8dc672319 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 4 May 2015 02:41:47 +0300 Subject: [PATCH 15/17] Change Object to a FullValidator --- valideer/tests/test_validators.py | 30 +++++++++++++ valideer/validators.py | 74 +++++++++++++++---------------- 2 files changed, 67 insertions(+), 37 deletions(-) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 29960cd..de25fba 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -949,6 +949,36 @@ def test_full_validate_mapping(self): "Invalid value 'z' (str): must be number (at x[True][1])", ]) + def test_full_validate_object(self): + obj = { + "+foo": "number", + "+bar": "string", + "?baz": { + "?a": "boolean", + "?b": V.Nullable("integer", 1), + } + } + value = {"baz": {"a": 1, "x": 4.5}} + common_errors = [ + "Invalid value {'baz': {'a': 1, 'x': 4.5}} (dict): missing required properties: ['foo', 'bar']", + "Invalid value 1 (int): must be boolean (at baz['a'])", + ] + + self._testFullValidationErrors(obj, value, errors=common_errors) + + with V.parsing(additional_properties=V.Object.REMOVE): + self._testFullValidationErrors(obj, value, errors=common_errors) + + with V.parsing(additional_properties=False): + self._testFullValidationErrors(obj, value, errors=common_errors + [ + "Invalid value {'a': 1, 'x': 4.5} (dict): additional properties: ['x'] (at baz)" + ]) + + with V.parsing(additional_properties="string"): + self._testFullValidationErrors(obj, value, errors=common_errors + [ + "Invalid value 4.5 (float): must be string (at baz['x'])" + ]) + def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), error_value_repr=repr): validator = self.parse(obj) diff --git a/valideer/validators.py b/valideer/validators.py index 33ade2b..e7e44e1 100644 --- a/valideer/validators.py +++ b/valideer/validators.py @@ -622,15 +622,13 @@ def _iter_errors_and_items(self, value, adapt, full): yield (k, v) -class Object(Type): +class Object(ContainerValidator): """A validator that accepts json-like objects. A ``json-like object`` here is meant as a dict with a predefined set of "properties", i.e. string keys. """ - accept_types = collections.Mapping - REQUIRED_PROPERTIES = False ADDITIONAL_PROPERTIES = True REMOVE = object() @@ -652,7 +650,7 @@ def __init__(self, optional={}, required={}, additional=None): - ``None`` to use the value of the ``ADDITIONAL_PROPERTIES`` class attribute. """ - super(Object, self).__init__() + self._type_validator = Type(accept_types=collections.Mapping) if additional is None: additional = self.ADDITIONAL_PROPERTIES if not isinstance(additional, bool) and additional is not self.REMOVE: @@ -665,49 +663,51 @@ def __init__(self, optional={}, required={}, additional=None): self._all_keys = set(name for name, _ in self._named_validators) self._additional = additional - def validate(self, value, adapt=True): - super(Object, self).validate(value) + def _iter_errors_and_items(self, value, adapt, full): + try: + self._type_validator.validate(value) + except ValidationError as ex: + yield ex + return + missing_required = self._required_keys.difference(value) if missing_required: - raise ValidationError("missing required properties: %s" % + yield ValidationError("missing required properties: %s" % list(missing_required), value) - result = dict(value) if adapt else None + method = 'full_validate' if full else 'validate' for name, validator in self._named_validators: if name in value: try: - adapted = validator.validate(value[name], adapt) - if result is not None: - result[name] = adapted + adapted = getattr(validator, method)(value[name], adapt=adapt) + if adapt: + yield (name, adapted) except ValidationError as ex: - raise ex.add_context(name) - elif result is not None and isinstance(validator, Nullable): + yield ex.add_context(name) + elif adapt and isinstance(validator, Nullable): default = validator.default_object_property if default is not Nullable._UNDEFINED: - result[name] = default - - if self._additional is not True: - all_keys = self._all_keys - additional_properties = [k for k in value if k not in all_keys] - if additional_properties: - if self._additional is False: - raise ValidationError("additional properties: %s" % - additional_properties, value) - elif self._additional is self.REMOVE: - if result is not None: - for name in additional_properties: - del result[name] - else: - additional_validate = self._additional.validate - for name in additional_properties: - try: - adapted = additional_validate(value[name], adapt) - if result is not None: - result[name] = adapted - except ValidationError as ex: - raise ex.add_context(name) - - return result + yield (name, default) + + additional = self._additional + all_keys = self._all_keys + additional_properties = [k for k in value if k not in all_keys] + if additional_properties and additional is not self.REMOVE: + if additional is False: + yield ValidationError("additional properties: %s" % + additional_properties, value) + elif additional is True: + for name in additional_properties: + yield (name, value[name]) + else: + for name in additional_properties: + try: + adapted = getattr(additional, method)(value[name], + adapt=adapt) + if adapt: + yield (name, adapted) + except ValidationError as ex: + yield ex.add_context(name) @Object.register_factory From 3d9b990b6f40503ce75980343abf7715bf800bb5 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 4 May 2015 02:52:53 +0300 Subject: [PATCH 16/17] Boost test coverage to 100% --- valideer/base.py | 2 +- valideer/tests/test_validators.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/valideer/base.py b/valideer/base.py index ece6dc3..b14e962 100644 --- a/valideer/base.py +++ b/valideer/base.py @@ -287,7 +287,7 @@ def _reduce_items(self, iterable, value): return value.__class__(iterable) def _iter_errors_and_items(self, value, adapt, full): - raise NotImplementedError + raise NotImplementedError # pragma: no cover def accepts(**schemas): diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index de25fba..0c62ecc 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -795,9 +795,6 @@ def test_humanized_names(self): class DummyValidator(V.Validator): name = "dummy" - def validate(self, value, adapt=True): - return value - self.assertEqual(DummyValidator().humanized_name, "dummy") self.assertEqual(V.Nullable(DummyValidator()).humanized_name, "dummy or null") self.assertEqual(V.AnyOf("boolean", DummyValidator()).humanized_name, @@ -930,6 +927,9 @@ def test_full_validate_heterogeneous_sequence(self): "Invalid value 4 (int): must be string (at 1[0])", "Invalid value True (bool): must be number (at 1[1])", ]) + self._testFullValidationErrors(obj, {}, errors=[ + "Invalid value {} (dict): must be Sequence", + ]) def test_full_validate_mapping(self): obj = V.Mapping("string", V.Mapping("integer", ["number"])) @@ -948,6 +948,9 @@ def test_full_validate_mapping(self): "Invalid value True (bool): must be integer (at x)", "Invalid value 'z' (str): must be number (at x[True][1])", ]) + self._testFullValidationErrors(obj, [], errors=[ + "Invalid value [] (list): must be Mapping", + ]) def test_full_validate_object(self): obj = { @@ -1002,12 +1005,11 @@ def _testValidation(self, obj, invalid=(), valid=(), adapted=(), errors=(), def _testFullValidationErrors(self, obj, value, errors, error_value_repr=repr): validator = self.parse(obj) + found_errors = [] try: validator.full_validate(value) except V.MultipleValidationError as ex: - found_errors = ex.errors - else: - found_errors = [] + found_errors.extend(ex.errors) self.assertEqual(len(found_errors), len(errors)) for found_error, error in zip(found_errors, errors): @@ -1048,7 +1050,3 @@ def test_required_properties_global(self): {"foo": 3}, {"bar": False, "baz": "yo"}, {"bar": True, "foo": 3.1}]) - - -if __name__ == '__main__': - unittest.main() From ae44ee124168d95552c742370f6dcb853307d5f8 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 31 May 2015 19:42:07 +0300 Subject: [PATCH 17/17] Document Validator.full_validate() and tweak MultipleValidationError.to_string() a bit --- README.rst | 28 +++++++++++++++++++++++++++- valideer/errors.py | 2 +- valideer/tests/test_validators.py | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 265ee6e..1c1360a 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,7 @@ You may run the unit tests with:: OK + Basic Usage ----------- @@ -181,6 +182,32 @@ typecheck_):: >>> get_total_price(product2) ValidationError: Invalid value {'price': 123, 'id': 1} (dict): missing required properties: ['name'] (at product) +Catching all errors +################### + +The ``validate`` method raises a ``ValidationError`` at the first encountered +input error. In case one wants to catch all input errors (for example, to show +them all to the user after validating a web form), the ``full_validate`` method +raises a ``MultipleValidationError``, a ``ValidationError`` subclass whose +``errors`` attribute is a (flat) list of all individual errors:: + + >>> product3 = { + >>> "id": "1", + >>> "price": -10, + >>> "tags": "foo", + >>> "stock": { + >>> "retail": True, + >>> } + >>> } + + >>> validator.full_validate(product3) + MultipleValidationError: + - Invalid value {'price': -10, 'stock': {'retail': True}, 'id': '1', 'tags': 'foo'} (dict): missing required properties: ['name'] + - Invalid value '1' (str): must be number (at id) + - Invalid value -10 (int): must not be less than 0 (at price) + - Invalid value True (bool): must be number (at stock['retail']) + - Invalid value 'foo' (str): must be Sequence (at tags) + Adaptation ########## @@ -273,7 +300,6 @@ optional are allowed by default. This default can be overriden by calling ... V.parse(schema).validate(data) ValidationError: Invalid value 12 (int): must be string (at duration['seconds']) - Explicit Instantiation ###################### diff --git a/valideer/errors.py b/valideer/errors.py index eb81e46..7821f07 100644 --- a/valideer/errors.py +++ b/valideer/errors.py @@ -69,7 +69,7 @@ def __init__(self, errors): self.add_errors(errors) def to_string(self, repr_value=repr): - lines = ["Multiple validation errors:"] + lines = [""] lines.extend("- " + e.to_string(repr_value) for e in self.errors) return "\n".join(lines) diff --git a/valideer/tests/test_validators.py b/valideer/tests/test_validators.py index 0c62ecc..40ccda2 100644 --- a/valideer/tests/test_validators.py +++ b/valideer/tests/test_validators.py @@ -883,7 +883,7 @@ def test_multiple_validation_error_message(self): V.ValidationError('Boring', 'stuff').add_context(2), ]) self.assertEqual(len(ex.errors), 4) - self.assertEqual(str(ex), "Multiple validation errors:\n" + self.assertEqual(str(ex), "\n" "- Invalid value 'moo' (str): More cowbell (at 0)\n" "- Invalid value 'blink' (str): Less blink (at 1['x'])\n" "- Invalid value 'mooooo' (str): More cowbell (at 1['y'])\n"