File: stone_validators.py

package info (click to toggle)
python-stone 3.3.8-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 1,956 kB
  • sloc: python: 21,786; objc: 498; sh: 29; makefile: 11
file content (726 lines) | stat: -rw-r--r-- 26,277 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
"""
Defines classes to represent each Stone type in Python. These classes should
be used to validate Python objects and normalize them for a given type.

The data types defined here should not be specific to an RPC or serialization
format.
"""


import datetime
import hashlib
import math
import numbers
import re
from abc import ABCMeta, abstractmethod


_MYPY = False
if _MYPY:
    import typing  # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression

# See <http://python3porting.com/differences.html#buffer>
_binary_types = (bytes, memoryview)  # noqa: E501,F821 # pylint: disable=undefined-variable,useless-suppression


class ValidationError(Exception):
    """Raised when a value doesn't pass validation by its validator."""

    def __init__(self, message, parent=None):
        """
        Args:
            message (str): Error message detailing validation failure.
            parent (str): Adds the parent as the closest reference point for
                the error. Use :meth:`add_parent` to add more.
        """
        super().__init__(message)
        self.message = message
        self._parents = []
        if parent:
            self._parents.append(parent)

    def add_parent(self, parent):
        """
        Args:
            parent (str): Adds the parent to the top of the tree of references
                that lead to the validator that failed.
        """
        self._parents.append(parent)

    def __str__(self):
        """
        Returns:
            str: A descriptive message of the validation error that may also
                include the path to the validator that failed.
        """
        if self._parents:
            return '{}: {}'.format('.'.join(self._parents[::-1]), self.message)
        else:
            return self.message

    def __repr__(self):
        # Not a perfect repr, but includes the error location information.
        return 'ValidationError(%r)' % str(self)


def type_name_with_module(t):
    # type: (typing.Type[typing.Any]) -> typing.Any
    return '{}.{}'.format(t.__module__, t.__name__)


def generic_type_name(v):
    # type: (typing.Any) -> typing.Any
    """Return a descriptive type name that isn't Python specific. For example,
    an int value will return 'integer' rather than 'int'."""
    if isinstance(v, bool):
        # Must come before any numbers checks since booleans are integers too
        return 'boolean'
    elif isinstance(v, numbers.Integral):
        # Must come before real numbers check since integrals are reals too
        return 'integer'
    elif isinstance(v, numbers.Real):
        return 'float'
    elif isinstance(v, (tuple, list)):
        return 'list'
    elif isinstance(v, str):
        return 'string'
    elif v is None:
        return 'null'
    else:
        return type_name_with_module(type(v))


def get_value_string(v, max_length=1000):
    # type: (typing.Any, int) -> str
    """Return a truncated version of the input string.  If the input string is longer than
       1000 characters, this will return the first 1000 characters and append with '[The
       string has been truncated due to its length]' to indicate that it has been truncated."""
    v_str = str(v)
    if len(v_str) > max_length:
        return v_str[:max_length] + ' [The string has been truncated due to its length]'
    return v_str


class Validator(metaclass=ABCMeta):
    """All primitive and composite data types should be a subclass of this."""
    __slots__ = ("_redact",)

    @abstractmethod
    def validate(self, val):
        """Validates that val is of this data type.

        Returns: A normalized value if validation succeeds.
        Raises: ValidationError
        """

    def has_default(self):
        return False

    def get_default(self):
        raise AssertionError('No default available.')


class Primitive(Validator):  # pylint: disable=abstract-method
    """A basic type that is defined by Stone."""
    __slots__ = ()


class Boolean(Primitive):
    __slots__ = ()

    def validate(self, val):
        if not isinstance(val, bool):
            raise ValidationError('%r is not a valid boolean' % val)
        return val


class Integer(Primitive):
    """
    Do not use this class directly. Extend it and specify a 'default_minimum' and
    'default_maximum' value as class variables for a more restrictive integer range.
    """
    __slots__ = ("minimum", "maximum")

    default_minimum = None  # type: typing.Optional[int]
    default_maximum = None  # type: typing.Optional[int]

    def __init__(self, min_value=None, max_value=None):
        """
        A more restrictive minimum or maximum value can be specified than the
        range inherent to the defined type.
        """
        if min_value is not None:
            assert isinstance(min_value, numbers.Integral), \
                'min_value must be an integral number'
            assert min_value >= self.default_minimum, \
                'min_value cannot be less than the minimum value for this ' \
                'type (%d < %d)' % (min_value, self.default_minimum)
            self.minimum = min_value
        else:
            self.minimum = self.default_minimum
        if max_value is not None:
            assert isinstance(max_value, numbers.Integral), \
                'max_value must be an integral number'
            assert max_value <= self.default_maximum, \
                'max_value cannot be greater than the maximum value for ' \
                'this type (%d < %d)' % (max_value, self.default_maximum)
            self.maximum = max_value
        else:
            self.maximum = self.default_maximum

    def validate(self, val):
        if not isinstance(val, numbers.Integral):
            raise ValidationError('expected integer, got %s'
                                  % generic_type_name(val))
        elif not (self.minimum <= val <= self.maximum):
            raise ValidationError('%d is not within range [%d, %d]'
                                  % (val, self.minimum, self.maximum))
        return val

    def __repr__(self):
        return '%s()' % self.__class__.__name__


class Int32(Integer):
    __slots__ = ()

    default_minimum = -2**31
    default_maximum = 2**31 - 1


class UInt32(Integer):
    __slots__ = ()

    default_minimum = 0
    default_maximum = 2**32 - 1


class Int64(Integer):
    __slots__ = ()

    default_minimum = -2**63
    default_maximum = 2**63 - 1


class UInt64(Integer):
    __slots__ = ()

    default_minimum = 0
    default_maximum = 2**64 - 1


class Real(Primitive):
    """
    Do not use this class directly. Extend it and optionally set a 'default_minimum'
    and 'default_maximum' value to enforce a range that's a subset of the Python float
    implementation. Python floats are doubles.
    """
    __slots__ = ("minimum", "maximum")

    default_minimum = None  # type: typing.Optional[float]
    default_maximum = None  # type: typing.Optional[float]

    def __init__(self, min_value=None, max_value=None):
        """
        A more restrictive minimum or maximum value can be specified than the
        range inherent to the defined type.
        """
        if min_value is not None:
            assert isinstance(min_value, numbers.Real), \
                'min_value must be a real number'
            if not isinstance(min_value, float):
                try:
                    min_value = float(min_value)
                except OverflowError:
                    raise AssertionError('min_value is too small for a float')
            if self.default_minimum is not None and min_value < self.default_minimum:
                raise AssertionError('min_value cannot be less than the '
                                     'minimum value for this type (%f < %f)' %
                                     (min_value, self.default_minimum))
            self.minimum = min_value
        else:
            self.minimum = self.default_minimum
        if max_value is not None:
            assert isinstance(max_value, numbers.Real), \
                'max_value must be a real number'
            if not isinstance(max_value, float):
                try:
                    max_value = float(max_value)
                except OverflowError:
                    raise AssertionError('max_value is too large for a float')
            if self.default_maximum is not None and max_value > self.default_maximum:
                raise AssertionError('max_value cannot be greater than the '
                                     'maximum value for this type (%f < %f)' %
                                     (max_value, self.default_maximum))
            self.maximum = max_value
        else:
            self.maximum = self.default_maximum

    def validate(self, val):
        if not isinstance(val, numbers.Real):
            raise ValidationError('expected real number, got %s' %
                                  generic_type_name(val))
        if not isinstance(val, float):
            # This checks for the case where a number is passed in with a
            # magnitude larger than supported by float64.
            try:
                val = float(val)
            except OverflowError:
                raise ValidationError('too large for float')
        if math.isnan(val) or math.isinf(val):
            raise ValidationError('%f values are not supported' % val)
        if self.minimum is not None and val < self.minimum:
            raise ValidationError('%f is not greater than %f' %
                                  (val, self.minimum))
        if self.maximum is not None and val > self.maximum:
            raise ValidationError('%f is not less than %f' %
                                  (val, self.maximum))
        return val

    def __repr__(self):
        return '%s()' % self.__class__.__name__


class Float32(Real):
    __slots__ = ()

    # Maximum and minimums from the IEEE 754-1985 standard
    default_minimum = -3.40282 * 10**38
    default_maximum = 3.40282 * 10**38


class Float64(Real):
    __slots__ = ()


class String(Primitive):
    """Represents a unicode string."""
    __slots__ = ("min_length", "max_length", "pattern", "pattern_re")

    def __init__(self, min_length=None, max_length=None, pattern=None):
        if min_length is not None:
            assert isinstance(min_length, numbers.Integral), \
                'min_length must be an integral number'
            assert min_length >= 0, 'min_length must be >= 0'
        if max_length is not None:
            assert isinstance(max_length, numbers.Integral), \
                'max_length must be an integral number'
            assert max_length > 0, 'max_length must be > 0'
        if min_length and max_length:
            assert max_length >= min_length, 'max_length must be >= min_length'
        if pattern is not None:
            assert isinstance(pattern, str), \
                'pattern must be a string'

        self.min_length = min_length
        self.max_length = max_length
        self.pattern = pattern
        self.pattern_re = None

        if pattern:
            try:
                self.pattern_re = re.compile(r"\A(?:" + pattern + r")\Z")
            except re.error as e:
                raise AssertionError('Regex {!r} failed: {}'.format(
                    pattern, e.args[0]))

    def validate(self, val):
        """
        A unicode string of the correct length and pattern will pass validation.
        In PY2, we enforce that a str type must be valid utf-8, and a unicode
        string will be returned.
        """
        if not isinstance(val, str):
            raise ValidationError("'%s' expected to be a string, got %s"
                                  % (get_value_string(val), generic_type_name(val)))

        if self.max_length is not None and len(val) > self.max_length:
            raise ValidationError("'%s' must be at most %d characters, got %d"
                                  % (get_value_string(val), self.max_length, len(val)))
        if self.min_length is not None and len(val) < self.min_length:
            raise ValidationError("'%s' must be at least %d characters, got %d"
                                  % (get_value_string(val), self.min_length, len(val)))

        if self.pattern and not self.pattern_re.match(val):
            # Detect if pattern is matching an email address and return redacted error message.
            if self.pattern == "^['#&A-Za-z0-9._%+-]+@[A-Za-z0-9-][A-Za-z0-9.-]*\\.[A-Za-z]{2,15}$":
                val = "*****"

            raise ValidationError("'%s' did not match pattern '%s'"
                                  % (get_value_string(val), self.pattern))
        return val


class Bytes(Primitive):
    __slots__ = ("min_length", "max_length")

    def __init__(self, min_length=None, max_length=None):
        if min_length is not None:
            assert isinstance(min_length, numbers.Integral), \
                'min_length must be an integral number'
            assert min_length >= 0, 'min_length must be >= 0'
        if max_length is not None:
            assert isinstance(max_length, numbers.Integral), \
                'max_length must be an integral number'
            assert max_length > 0, 'max_length must be > 0'
        if min_length is not None and max_length is not None:
            assert max_length >= min_length, 'max_length must be >= min_length'

        self.min_length = min_length
        self.max_length = max_length

    def validate(self, val):
        if not isinstance(val, _binary_types):
            raise ValidationError("expected bytes type, got %s"
                                  % generic_type_name(val))
        elif self.max_length is not None and len(val) > self.max_length:
            raise ValidationError("'%s' must have at most %d bytes, got %d"
                                  % (get_value_string(val), self.max_length, len(val)))
        elif self.min_length is not None and len(val) < self.min_length:
            raise ValidationError("'%s' has fewer than %d bytes, got %d"
                                  % (get_value_string(val), self.min_length, len(val)))
        return val


class Timestamp(Primitive):
    """Note that while a format is specified, it isn't used in validation
    since a native Python datetime object is preferred. The format, however,
    can and should be used by serializers."""
    __slots__ = ("format",)

    def __init__(self, fmt):
        """fmt must be composed of format codes that the C standard (1989)
        supports, most notably in its strftime() function."""
        assert isinstance(fmt, str), 'format must be a string'
        self.format = fmt

    def validate(self, val):
        if not isinstance(val, datetime.datetime):
            raise ValidationError('expected timestamp, got %s'
                                  % generic_type_name(val))
        elif val.tzinfo is not None and \
                val.tzinfo.utcoffset(val).total_seconds() != 0:
            raise ValidationError('timestamp should have either a UTC '
                                  'timezone or none set at all')
        return val


class Composite(Validator):  # pylint: disable=abstract-method
    """Validator for a type that builds on other primitive and composite
    types."""
    __slots__ = ()


class List(Composite):
    """Assumes list contents are homogeneous with respect to types."""
    __slots__ = ("item_validator", "min_items", "max_items")

    def __init__(self, item_validator, min_items=None, max_items=None):
        """Every list item will be validated with item_validator."""
        self.item_validator = item_validator
        if min_items is not None:
            assert isinstance(min_items, numbers.Integral), \
                'min_items must be an integral number'
            assert min_items >= 0, 'min_items must be >= 0'
        if max_items is not None:
            assert isinstance(max_items, numbers.Integral), \
                'max_items must be an integral number'
            assert max_items > 0, 'max_items must be > 0'
        if min_items is not None and max_items is not None:
            assert max_items >= min_items, 'max_items must be >= min_items'

        self.min_items = min_items
        self.max_items = max_items

    def validate(self, val):
        if not isinstance(val, (tuple, list)):
            raise ValidationError('%r is not a valid list' % get_value_string(val))
        elif self.max_items is not None and len(val) > self.max_items:
            raise ValidationError('%r has more than %s items'
                                  % (get_value_string(val), self.max_items))
        elif self.min_items is not None and len(val) < self.min_items:
            raise ValidationError('%r has fewer than %s items'
                                  % (get_value_string(val), self.min_items))
        return [self.item_validator.validate(item) for item in val]


class Map(Composite):
    """Assumes map keys and values are homogeneous with respect to types."""
    __slots__ = ("key_validator", "value_validator")

    def __init__(self, key_validator, value_validator):
        """
        Every Map key/value pair will be validated with item_validator.
        key validators must be a subclass of a String validator
        """
        self.key_validator = key_validator
        self.value_validator = value_validator

    def validate(self, val):
        if not isinstance(val, dict):
            raise ValidationError('%r is not a valid dict' % get_value_string(val))
        return {
            self.key_validator.validate(key):
                self.value_validator.validate(value) for key, value in val.items()
        }


class Struct(Composite):
    __slots__ = ("definition",)

    def __init__(self, definition):
        """
        Args:
            definition (class): A generated class representing a Stone struct
                from a spec. Must have a _fields_ attribute with the following
                structure:

                _fields_ = [(field_name, validator), ...]

                where
                    field_name: Name of the field (str).
                    validator: Validator object.
        """
        super().__init__()
        self.definition = definition

    def validate(self, val):
        """
        For a val to pass validation, val must be of the correct type and have
        all required fields present.
        """
        self.validate_type_only(val)
        self.validate_fields_only(val)
        return val

    def validate_with_permissions(self, val, caller_permissions):
        """
        For a val to pass validation, val must be of the correct type and have
        all required permissioned fields present. Should only be called
        for callers with extra permissions.
        """
        self.validate(val)
        self.validate_fields_only_with_permissions(val, caller_permissions)
        return val

    def validate_fields_only(self, val):
        """
        To pass field validation, no required field should be missing.

        This method assumes that the contents of each field have already been
        validated on assignment, so it's merely a presence check.

        FIXME(kelkabany): Since the definition object does not maintain a list
        of which fields are required, all fields are scanned.
        """
        for field_name in self.definition._all_field_names_:
            if not hasattr(val, field_name):
                raise ValidationError("missing required field '%s'" %
                                      field_name)

    def validate_fields_only_with_permissions(self, val, caller_permissions):
        """
        To pass field validation, no required field should be missing.
        This method assumes that the contents of each field have already been
        validated on assignment, so it's merely a presence check.
        Should only be called for callers with extra permissions.
        """
        self.validate_fields_only(val)

        # check if type has been patched
        for extra_permission in caller_permissions.permissions:
            all_field_names = '_all_{}_field_names_'.format(extra_permission)
            for field_name in getattr(self.definition, all_field_names, set()):
                if not hasattr(val, field_name):
                    raise ValidationError("missing required field '%s'" % field_name)

    def validate_type_only(self, val):
        """
        Use this when you only want to validate that the type of an object
        is correct, but not yet validate each field.
        """
        # Since the definition maintains the list of fields for serialization,
        # we're okay with a subclass that might have extra information. This
        # makes it easier to return one subclass for two routes, one of which
        # relies on the parent class.
        if not isinstance(val, self.definition):
            raise ValidationError('expected type %s, got %s' %
                (
                    type_name_with_module(self.definition),
                    generic_type_name(val),
                ),
            )

    def has_default(self):
        return not self.definition._has_required_fields

    def get_default(self):
        assert not self.definition._has_required_fields, 'No default available.'
        return self.definition()


class StructTree(Struct):
    """Validator for structs with enumerated subtypes.

    NOTE: validate_fields_only() validates the fields known to this base
    struct, but does not do any validation specific to the subtype.
    """
    __slots__ = ()

    # See PyCQA/pylint#1043 for why this is disabled; this should show up
    # as a usless-suppression (and can be removed) once a fix is released
    def __init__(self, definition):  # pylint: disable=useless-super-delegation
        super().__init__(definition)


class Union(Composite):
    __slots__ = ("definition",)

    def __init__(self, definition):
        """
        Args:
            definition (class): A generated class representing a Stone union
                from a spec. Must have a _tagmap attribute with the following
                structure:

                _tagmap = {field_name: validator, ...}

                where
                    field_name (str): Tag name.
                    validator (Validator): Tag value validator.
        """
        self.definition = definition

    def validate(self, val):
        """
        For a val to pass validation, it must have a _tag set. This assumes
        that the object validated that _tag is a valid tag, and that any
        associated value has also been validated.
        """
        self.validate_type_only(val)
        if not hasattr(val, '_tag') or val._tag is None:
            raise ValidationError('no tag set')
        return val

    def validate_type_only(self, val):
        """
        Use this when you only want to validate that the type of an object
        is correct, but not yet validate each field.

        We check whether val is a Python parent class of the definition. This
        is because Union subtyping works in the opposite direction of Python
        inheritance. For example, if a union U2 extends U1 in Python, this
        validator will accept U1 in places where U2 is expected.
        """
        if not issubclass(self.definition, type(val)):
            raise ValidationError('expected type %s or subtype, got %s' %
                (
                    type_name_with_module(self.definition),
                    generic_type_name(val),
                ),
            )


class Void(Primitive):
    __slots__ = ()

    def validate(self, val):
        if val is not None:
            raise ValidationError('expected NoneType, got %s' %
                                  generic_type_name(val))

    def has_default(self):
        return True

    def get_default(self):
        return None


class Nullable(Validator):
    __slots__ = ("validator",)

    def __init__(self, validator):
        super().__init__()
        assert isinstance(validator, (Primitive, Composite)), \
            'validator must be for a primitive or composite type'
        assert not isinstance(validator, Nullable), \
            'nullables cannot be stacked'
        assert not isinstance(validator, Void), \
            'void cannot be made nullable'
        self.validator = validator

    def validate(self, val):
        if val is None:
            return
        else:
            return self.validator.validate(val)

    def validate_type_only(self, val):
        """Use this only if Nullable is wrapping a Composite."""
        if val is None:
            return
        else:
            return self.validator.validate_type_only(val)

    def has_default(self):
        return True

    def get_default(self):
        return None


class Redactor:
    __slots__ = ("regex",)

    def __init__(self, regex):
        """
        Args:
            regex: What parts of the field to redact.
        """
        self.regex = regex

    @abstractmethod
    def apply(self, val):
        """Redacts information from annotated field.
        Returns: A redacted version of the string provided.
        """

    def _get_matches(self, val):
        if not self.regex:
            return None
        try:
            return re.search(self.regex, val)
        except TypeError:
            return None


class HashRedactor(Redactor):
    __slots__ = ()

    def apply(self, val):
        matches = self._get_matches(val)

        val_to_hash = str(val) if isinstance(val, int) or isinstance(val, float) else val

        try:
            # add string literal to ensure unicode
            hashed = hashlib.md5(val_to_hash.encode('utf-8')).hexdigest() + ''
        except [AttributeError, ValueError]:
            hashed = None

        if matches:
            blotted = '***'.join(matches.groups())
            if hashed:
                return '{} ({})'.format(hashed, blotted)
            return blotted
        return hashed


class BlotRedactor(Redactor):
    __slots__ = ()

    def apply(self, val):
        matches = self._get_matches(val)
        if matches:
            return '***'.join(matches.groups())
        return '********'