diff --git a/Pipfile b/Pipfile index a8da44535bad..5d41687bbca1 100644 --- a/Pipfile +++ b/Pipfile @@ -16,4 +16,4 @@ python-hglib = "==2.4" requests = "==2.9.1" six = "==1.10.0" virtualenv = "==15.2.0" -voluptuous = "==0.10.5" +voluptuous = "==0.11.5" diff --git a/Pipfile.lock b/Pipfile.lock index 51ba7ce30dd4..107ab3064483 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "609a35f65e9a4c07e0e1473ec982c6b5028622e9a795b6cfb8555ad8574804f3" + "sha256": "f718e0b6ec2c030d4becf157f8ca0fd1b2f32ca277d5d3d2407a2dee33119441" }, "pipfile-spec": 6, "requires": {}, @@ -69,11 +69,11 @@ }, "more-itertools": { "hashes": [ - "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8", - "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3", - "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0" + "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", + "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", + "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], - "version": "==4.2.0" + "version": "==4.3.0" }, "pipenv": { "hashes": [ @@ -146,10 +146,11 @@ }, "voluptuous": { "hashes": [ - "sha256:7a7466f8dc3666a292d186d1d871a47bf2120836ccb900d5ba904674957a2396" + "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1", + "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef" ], "index": "pypi", - "version": "==0.10.5" + "version": "==0.11.5" } }, "develop": {} diff --git a/third_party/python/more-itertools/MANIFEST.in b/third_party/python/more-itertools/MANIFEST.in index ec800e3e02a6..21d674258617 100644 --- a/third_party/python/more-itertools/MANIFEST.in +++ b/third_party/python/more-itertools/MANIFEST.in @@ -4,5 +4,6 @@ include docs/*.rst include docs/Makefile include docs/make.bat include docs/conf.py +include docs/_static/* include fabfile.py include tox.ini diff --git a/third_party/python/more-itertools/PKG-INFO b/third_party/python/more-itertools/PKG-INFO index 8c802bc8af65..95d111bf6b7e 100644 --- a/third_party/python/more-itertools/PKG-INFO +++ b/third_party/python/more-itertools/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: more-itertools -Version: 4.2.0 +Version: 4.3.0 Summary: More routines for operating on iterables, beyond itertools Home-page: https://github.com/erikrose/more-itertools Author: Erik Rose @@ -18,6 +18,101 @@ Description: ============== we collect additional building blocks, recipes, and routines for working with Python iterables. + ---- + + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Grouping | `chunked `_, | + | | `sliced `_, | + | | `distribute `_, | + | | `divide `_, | + | | `split_at `_, | + | | `split_before `_, | + | | `split_after `_, | + | | `bucket `_, | + | | `grouper `_, | + | | `partition `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Lookahead and lookback | `spy `_, | + | | `peekable `_, | + | | `seekable `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Windowing | `windowed `_, | + | | `stagger `_, | + | | `pairwise `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Augmenting | `count_cycle `_, | + | | `intersperse `_, | + | | `padded `_, | + | | `adjacent `_, | + | | `groupby_transform `_, | + | | `padnone `_, | + | | `ncycles `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Combining | `collapse `_, | + | | `sort_together `_, | + | | `interleave `_, | + | | `interleave_longest `_, | + | | `collate `_, | + | | `zip_offset `_, | + | | `dotproduct `_, | + | | `flatten `_, | + | | `roundrobin `_, | + | | `prepend `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Summarizing | `ilen `_, | + | | `first `_, | + | | `last `_, | + | | `one `_, | + | | `unique_to_each `_, | + | | `locate `_, | + | | `rlocate `_, | + | | `consecutive_groups `_, | + | | `exactly_n `_, | + | | `run_length `_, | + | | `map_reduce `_, | + | | `all_equal `_, | + | | `first_true `_, | + | | `nth `_, | + | | `quantify `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Selecting | `islice_extended `_, | + | | `strip `_, | + | | `lstrip `_, | + | | `rstrip `_, | + | | `take `_, | + | | `tail `_, | + | | `unique_everseen `_, | + | | `unique_justseen `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Combinatorics | `distinct_permutations `_, | + | | `circular_shifts `_, | + | | `powerset `_, | + | | `random_product `_, | + | | `random_permutation `_, | + | | `random_combination `_, | + | | `random_combination_with_replacement `_, | + | | `nth_combination `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Wrapping | `always_iterable `_, | + | | `consumer `_, | + | | `with_iter `_, | + | | `iter_except `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Others | `replace `_, | + | | `numeric_range `_, | + | | `always_reversible `_, | + | | `side_effect `_, | + | | `iterate `_, | + | | `difference `_, | + | | `make_decorator `_, | + | | `SequenceView `_, | + | | `consume `_, | + | | `accumulate `_, | + | | `tabulate `_, | + | | `repeatfunc `_ | + +------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + Getting started =============== @@ -72,6 +167,20 @@ Description: ============== + 4.3.0 + ----- + + * New itertools: + * last (thanks to tmshn) + * replace (thanks to pylang) + * rlocate (thanks to jferard and pylang) + + * Improvements to existing itertools: + * locate can now search for multiple items + + * Other changes: + * The docs now include a nice table of tools (thanks MSeifert04) + 4.2.0 ----- diff --git a/third_party/python/more-itertools/README.rst b/third_party/python/more-itertools/README.rst index 252b394737ac..d918eb684ffb 100644 --- a/third_party/python/more-itertools/README.rst +++ b/third_party/python/more-itertools/README.rst @@ -10,6 +10,101 @@ for a variety of problems with the functions it provides. In ``more-itertools`` we collect additional building blocks, recipes, and routines for working with Python iterables. +---- + ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Grouping | `chunked `_, | +| | `sliced `_, | +| | `distribute `_, | +| | `divide `_, | +| | `split_at `_, | +| | `split_before `_, | +| | `split_after `_, | +| | `bucket `_, | +| | `grouper `_, | +| | `partition `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Lookahead and lookback | `spy `_, | +| | `peekable `_, | +| | `seekable `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Windowing | `windowed `_, | +| | `stagger `_, | +| | `pairwise `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Augmenting | `count_cycle `_, | +| | `intersperse `_, | +| | `padded `_, | +| | `adjacent `_, | +| | `groupby_transform `_, | +| | `padnone `_, | +| | `ncycles `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Combining | `collapse `_, | +| | `sort_together `_, | +| | `interleave `_, | +| | `interleave_longest `_, | +| | `collate `_, | +| | `zip_offset `_, | +| | `dotproduct `_, | +| | `flatten `_, | +| | `roundrobin `_, | +| | `prepend `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Summarizing | `ilen `_, | +| | `first `_, | +| | `last `_, | +| | `one `_, | +| | `unique_to_each `_, | +| | `locate `_, | +| | `rlocate `_, | +| | `consecutive_groups `_, | +| | `exactly_n `_, | +| | `run_length `_, | +| | `map_reduce `_, | +| | `all_equal `_, | +| | `first_true `_, | +| | `nth `_, | +| | `quantify `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Selecting | `islice_extended `_, | +| | `strip `_, | +| | `lstrip `_, | +| | `rstrip `_, | +| | `take `_, | +| | `tail `_, | +| | `unique_everseen `_, | +| | `unique_justseen `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Combinatorics | `distinct_permutations `_, | +| | `circular_shifts `_, | +| | `powerset `_, | +| | `random_product `_, | +| | `random_permutation `_, | +| | `random_combination `_, | +| | `random_combination_with_replacement `_, | +| | `nth_combination `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Wrapping | `always_iterable `_, | +| | `consumer `_, | +| | `with_iter `_, | +| | `iter_except `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Others | `replace `_, | +| | `numeric_range `_, | +| | `always_reversible `_, | +| | `side_effect `_, | +| | `iterate `_, | +| | `difference `_, | +| | `make_decorator `_, | +| | `SequenceView `_, | +| | `consume `_, | +| | `accumulate `_, | +| | `tabulate `_, | +| | `repeatfunc `_ | ++------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + Getting started =============== diff --git a/third_party/python/more-itertools/docs/_static/theme_overrides.css b/third_party/python/more-itertools/docs/_static/theme_overrides.css new file mode 100644 index 000000000000..3a451aeae168 --- /dev/null +++ b/third_party/python/more-itertools/docs/_static/theme_overrides.css @@ -0,0 +1,14 @@ +/* https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html */ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} diff --git a/third_party/python/more-itertools/docs/api.rst b/third_party/python/more-itertools/docs/api.rst index 63e5d7f45001..65cb009310d5 100644 --- a/third_party/python/more-itertools/docs/api.rst +++ b/third_party/python/more-itertools/docs/api.rst @@ -124,9 +124,11 @@ These tools return summarized or aggregated data from an iterable. .. autofunction:: ilen .. autofunction:: first(iterable[, default]) +.. autofunction:: last(iterable[, default]) .. autofunction:: one .. autofunction:: unique_to_each .. autofunction:: locate(iterable, pred=bool) +.. autofunction:: rlocate(iterable, pred=bool) .. autofunction:: consecutive_groups(iterable, ordering=lambda x: x) .. autofunction:: exactly_n(iterable, n, predicate=bool) .. autoclass:: run_length @@ -216,6 +218,7 @@ Others **New itertools** +.. autofunction:: replace .. autofunction:: numeric_range(start, stop, step) .. autofunction:: always_reversible .. autofunction:: side_effect diff --git a/third_party/python/more-itertools/docs/conf.py b/third_party/python/more-itertools/docs/conf.py index e38c71aeaacd..8adf15fca95a 100644 --- a/third_party/python/more-itertools/docs/conf.py +++ b/third_party/python/more-itertools/docs/conf.py @@ -50,7 +50,7 @@ copyright = u'2012, Erik Rose' # built documents. # # The short X.Y version. -version = '4.2.0' +version = '4.3.0' # The full version, including alpha/beta/rc tags. release = version @@ -124,6 +124,11 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_context = { + # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + 'css_files': ['_static/theme_overrides.css'], +} + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' diff --git a/third_party/python/more-itertools/docs/versions.rst b/third_party/python/more-itertools/docs/versions.rst index e50ac4393dd8..b8a7f197409f 100644 --- a/third_party/python/more-itertools/docs/versions.rst +++ b/third_party/python/more-itertools/docs/versions.rst @@ -4,6 +4,20 @@ Version History .. automodule:: more_itertools +4.3.0 +----- + +* New itertools: + * :func:`last` (thanks to tmshn) + * :func:`replace` (thanks to pylang) + * :func:`rlocate` (thanks to jferard and pylang) + +* Improvements to existing itertools: + * :func:`locate` can now search for multiple items + +* Other changes: + * The docs now include a nice table of tools (thanks MSeifert04) + 4.2.0 ----- diff --git a/third_party/python/more-itertools/more_itertools/more.py b/third_party/python/more-itertools/more_itertools/more.py index d517250242f3..05e851eefab9 100644 --- a/third_party/python/more-itertools/more_itertools/more.py +++ b/third_party/python/more-itertools/more_itertools/more.py @@ -12,6 +12,7 @@ from itertools import ( groupby, islice, repeat, + starmap, takewhile, tee ) @@ -52,6 +53,7 @@ __all__ = [ 'intersperse', 'islice_extended', 'iterate', + 'last', 'locate', 'lstrip', 'make_decorator', @@ -60,6 +62,8 @@ __all__ = [ 'one', 'padded', 'peekable', + 'replace', + 'rlocate', 'rstrip', 'run_length', 'seekable', @@ -136,6 +140,32 @@ def first(iterable, default=_marker): return default +def last(iterable, default=_marker): + """Return the last item of *iterable*, or *default* if *iterable* is + empty. + + >>> last([0, 1, 2, 3]) + 3 + >>> last([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + """ + try: + try: + # Try to access the last item directly + return iterable[-1] + except (TypeError, AttributeError, KeyError): + # If not slice-able, iterate entirely using length-1 deque + return deque(iterable, maxlen=1)[0] + except IndexError: # If the iterable was empty + if default is _marker: + raise ValueError('last() was called on an empty iterable, and no ' + 'default value was provided.') + return default + + class peekable(object): """Wrap an iterator to allow lookahead and prepending elements. @@ -1435,7 +1465,7 @@ def count_cycle(iterable, n=None): return ((i, item) for i in counter for item in iterable) -def locate(iterable, pred=bool): +def locate(iterable, pred=bool, window_size=None): """Yield the index of each item in *iterable* for which *pred* returns ``True``. @@ -1445,18 +1475,17 @@ def locate(iterable, pred=bool): [1, 2, 4] Set *pred* to a custom function to, e.g., find the indexes for a particular - item: + item. >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) [1, 3] - Use with :func:`windowed` to find the indexes of a sub-sequence: + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: - >>> from more_itertools import windowed >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> sub = [1, 2, 3] - >>> pred = lambda w: w == tuple(sub) # windowed() returns tuples - >>> list(locate(windowed(iterable, len(sub)), pred=pred)) + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(locate(iterable, pred=pred, window_size=3)) [1, 5, 9] Use with :func:`seekable` to find indexes and then retrieve the associated @@ -1474,7 +1503,14 @@ def locate(iterable, pred=bool): 106 """ - return compress(count(), map(pred, iterable)) + if window_size is None: + return compress(count(), map(pred, iterable)) + + if window_size < 1: + raise ValueError('window size must be at least 1') + + it = windowed(iterable, window_size, fillvalue=_marker) + return compress(count(), starmap(pred, it)) def lstrip(iterable, pred): @@ -2032,7 +2068,7 @@ def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): [('A', 1), ('B', 2), ('C', 3)] You may want to filter the input iterable before applying the map/reduce - proecdure: + procedure: >>> all_items = range(30) >>> items = [x for x in all_items if 10 <= x <= 20] # Filter @@ -2066,3 +2102,110 @@ def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): ret.default_factory = None return ret + + +def rlocate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``, starting from the right and moving left. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 + [4, 2, 1] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item: + + >>> iterable = iter('abcb') + >>> pred = lambda x: x == 'b' + >>> list(rlocate(iterable, pred)) + [3, 1] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(rlocate(iterable, pred=pred, window_size=3)) + [9, 5, 1] + + Beware, this function won't return anything for infinite iterables. + If *iterable* is reversible, ``rlocate`` will reverse it and search from + the right. Otherwise, it will search from the left and return the results + in reverse order. + + See :func:`locate` to for other example applications. + + """ + if window_size is None: + try: + len_iter = len(iterable) + return ( + len_iter - i - 1 for i in locate(reversed(iterable), pred) + ) + except TypeError: + pass + + return reversed(list(locate(iterable, pred, window_size))) + + +def replace(iterable, pred, substitutes, count=None, window_size=1): + """Yield the items from *iterable*, replacing the items for which *pred* + returns ``True`` with the items from the iterable *substitutes*. + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] + >>> pred = lambda x: x == 0 + >>> substitutes = (2, 3) + >>> list(replace(iterable, pred, substitutes)) + [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] + + If *count* is given, the number of replacements will be limited: + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] + >>> pred = lambda x: x == 0 + >>> substitutes = [None] + >>> list(replace(iterable, pred, substitutes, count=2)) + [1, 1, None, 1, 1, None, 1, 1, 0] + + Use *window_size* to control the number of items passed as arguments to + *pred*. This allows for locating and replacing subsequences. + + >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] + >>> window_size = 3 + >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred + >>> substitutes = [3, 4] # Splice in these items + >>> list(replace(iterable, pred, substitutes, window_size=window_size)) + [3, 4, 5, 3, 4, 5] + + """ + if window_size < 1: + raise ValueError('window_size must be at least 1') + + # Save the substitutes iterable, since it's used more than once + substitutes = tuple(substitutes) + + # Add padding such that the number of windows matches the length of the + # iterable + it = chain(iterable, [_marker] * (window_size - 1)) + windows = windowed(it, window_size) + + n = 0 + for w in windows: + # If the current window matches our predicate (and we haven't hit + # our maximum number of replacements), splice in the substitutes + # and then consume the following windows that overlap with this one. + # For example, if the iterable is (0, 1, 2, 3, 4...) + # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... + # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) + if pred(*w): + if (count is None) or (n < count): + n += 1 + for s in substitutes: + yield s + consume(windows, window_size - 1) + continue + + # If there was no match (or we've reached the replacement limit), + # yield the first item from the window. + if w and (w[0] is not _marker): + yield w[0] diff --git a/third_party/python/more-itertools/more_itertools/tests/test_more.py b/third_party/python/more-itertools/more_itertools/tests/test_more.py index 2023ba6a4cbd..a1b1e43198cc 100644 --- a/third_party/python/more-itertools/more_itertools/tests/test_more.py +++ b/third_party/python/more-itertools/more_itertools/tests/test_more.py @@ -1,5 +1,6 @@ from __future__ import division, print_function, unicode_literals +from collections import OrderedDict from decimal import Decimal from doctest import DocTestSuite from fractions import Fraction @@ -114,6 +115,90 @@ class FirstTests(TestCase): self.assertEqual(mi.first([], 'boo'), 'boo') +class IterOnlyRange: + """User-defined iterable class which only support __iter__. + + It is not specified to inherit ``object``, so indexing on a instance will + raise an ``AttributeError`` rather than ``TypeError`` in Python 2. + + >>> r = IterOnlyRange(5) + >>> r[0] + AttributeError: IterOnlyRange instance has no attribute '__getitem__' + + Note: In Python 3, ``TypeError`` will be raised because ``object`` is + inherited implicitly by default. + + >>> r[0] + TypeError: 'IterOnlyRange' object does not support indexing + """ + def __init__(self, n): + """Set the length of the range.""" + self.n = n + + def __iter__(self): + """Works same as range().""" + return iter(range(self.n)) + + +class LastTests(TestCase): + """Tests for ``last()``""" + + def test_many_nonsliceable(self): + """Test that it works on many-item non-slice-able iterables.""" + # Also try it on a generator expression to make sure it works on + # whatever those return, across Python versions. + self.assertEqual(mi.last(x for x in range(4)), 3) + + def test_one_nonsliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last(x for x in range(1)), 0) + + def test_empty_stop_iteration_nonsliceable(self): + """It should raise ValueError for empty non-slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last(x for x in range(0))) + + def test_default_nonsliceable(self): + """It should return the provided default arg for empty non-slice-able + iterables. + """ + self.assertEqual(mi.last((x for x in range(0)), 'boo'), 'boo') + + def test_many_sliceable(self): + """Test that it works on many-item slice-able iterables.""" + self.assertEqual(mi.last([0, 1, 2, 3]), 3) + + def test_one_sliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last([3]), 3) + + def test_empty_stop_iteration_sliceable(self): + """It should raise ValueError for empty slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last([])) + + def test_default_sliceable(self): + """It should return the provided default arg for empty slice-able + iterables. + """ + self.assertEqual(mi.last([], 'boo'), 'boo') + + def test_dict(self): + """last(dic) and last(dic.keys()) should return same result.""" + dic = {'a': 1, 'b': 2, 'c': 3} + self.assertEqual(mi.last(dic), mi.last(dic.keys())) + + def test_ordereddict(self): + """last(dic) should return the last key.""" + od = OrderedDict() + od['a'] = 1 + od['b'] = 2 + od['c'] = 3 + self.assertEqual(mi.last(od), 'c') + + def test_customrange(self): + """It should work on custom class where [] raises AttributeError.""" + self.assertEqual(mi.last(IterOnlyRange(5)), 4) + + class PeekableTests(TestCase): """Tests for ``peekable()`` behavor not incidentally covered by testing ``collate()`` @@ -1462,6 +1547,26 @@ class LocateTests(TestCase): expected = [0, 3, 5, 6] self.assertEqual(actual, expected) + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + actual = list(mi.locate(iterable, pred, window_size=2)) + expected = [0, 3] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + actual = list(mi.locate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + class StripFunctionTests(TestCase): def test_hashable(self): @@ -1846,3 +1951,124 @@ class MapReduceTests(TestCase): d = mi.map_reduce([1, 0, 2, 0, 1, 0], bool) self.assertEqual(d, {False: [0, 0, 0], True: [1, 2, 1]}) self.assertRaises(KeyError, lambda: d[None].append(1)) + + +class RlocateTests(TestCase): + def test_default_pred(self): + iterable = [0, 1, 1, 0, 1, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [4, 2, 1] + self.assertEqual(actual, expected) + + def test_no_matches(self): + iterable = [0, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [] + self.assertEqual(actual, expected) + + def test_custom_pred(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda x: x == '0' + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it, pred)) + expected = [6, 5, 3, 0] + self.assertEqual(actual, expected) + + def test_efficient_reversal(self): + iterable = range(10 ** 10) # Is efficiently reversible + target = 10 ** 10 - 2 + pred = lambda x: x == target # Find-able from the right + actual = next(mi.rlocate(iterable, pred)) + self.assertEqual(actual, target) + + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(it, pred, window_size=2)) + expected = [3, 0] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + for it in (iterable, iter(iterable)): + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + + +class ReplaceTests(TestCase): + def test_basic(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes)) + expected = [1, 3, 5, 7, 9] + self.assertEqual(actual, expected) + + def test_count(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, count=4)) + expected = [1, 3, 5, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size(self): + iterable = range(10) + pred = lambda *args: args == (0, 1, 2) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_end(self): + iterable = range(10) + pred = lambda *args: args == (7, 8, 9) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [0, 1, 2, 3, 4, 5, 6] + self.assertEqual(actual, expected) + + def test_window_size_count(self): + iterable = range(10) + pred = lambda *args: (args == (0, 1, 2)) or (args == (7, 8, 9)) + substitutes = [] + actual = list( + mi.replace(iterable, pred, substitutes, count=1, window_size=3) + ) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = range(4) + pred = lambda a, b, c, d, e: True + substitutes = [5, 6, 7] + actual = list(mi.replace(iterable, pred, substitutes, window_size=5)) + expected = [5, 6, 7] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = range(10) + pred = lambda *args: True + substitutes = [] + with self.assertRaises(ValueError): + list(mi.replace(iterable, pred, substitutes, window_size=0)) + + def test_iterable_substitutes(self): + iterable = range(5) + pred = lambda x: x % 2 == 0 + substitutes = iter('__') + actual = list(mi.replace(iterable, pred, substitutes)) + expected = ['_', '_', 1, '_', '_', 3, '_', '_'] + self.assertEqual(actual, expected) diff --git a/third_party/python/more-itertools/more_itertools/tests/test_recipes.py b/third_party/python/more-itertools/more_itertools/tests/test_recipes.py index 81721fdf9fe9..98981fe8e660 100644 --- a/third_party/python/more-itertools/more_itertools/tests/test_recipes.py +++ b/third_party/python/more-itertools/more_itertools/tests/test_recipes.py @@ -590,6 +590,15 @@ class NthCombinationTests(TestCase): expected = (2, 12, 35, 126) self.assertEqual(actual, expected) + def test_invalid_r(self): + for r in (-1, 3): + with self.assertRaises(ValueError): + mi.nth_combination([], r, 0) + + def test_invalid_index(self): + with self.assertRaises(IndexError): + mi.nth_combination('abcdefg', 3, -36) + class PrependTests(TestCase): def test_basic(self): diff --git a/third_party/python/more-itertools/setup.py b/third_party/python/more-itertools/setup.py index 484e4d06f799..277265387557 100644 --- a/third_party/python/more-itertools/setup.py +++ b/third_party/python/more-itertools/setup.py @@ -28,7 +28,7 @@ def get_long_description(): setup( name='more-itertools', - version='4.2.0', + version='4.3.0', description='More routines for operating on iterables, beyond itertools', long_description=get_long_description(), author='Erik Rose', diff --git a/third_party/python/voluptuous/CHANGELOG.md b/third_party/python/voluptuous/CHANGELOG.md index 2887fd3d19ca..90d644f34a4f 100644 --- a/third_party/python/voluptuous/CHANGELOG.md +++ b/third_party/python/voluptuous/CHANGELOG.md @@ -1,6 +1,35 @@ # Changelog -## [Unreleased] +## [0.11.0] + +**Changes**: + +- [#293](https://github.com/alecthomas/voluptuous/pull/293): Support Python 3.6. +- [#294](https://github.com/alecthomas/voluptuous/pull/294): Drop support for Python 2.6, 3.1 and 3.2. +- [#318](https://github.com/alecthomas/voluptuous/pull/318): Allow to use nested schema and allow any validator to be compiled. +- [#324](https://github.com/alecthomas/voluptuous/pull/324): + Default values MUST now pass validation just as any regular value. This is a backward incompatible change if a schema uses default values that don't pass validation against the specified schema. +- [#328](https://github.com/alecthomas/voluptuous/pull/328): + Modify `__lt__` in Marker class to allow comparison with non Marker objects, such as str and int. + +**New**: + +- [#307](https://github.com/alecthomas/voluptuous/pull/307): Add description field to `Marker` instances. +- [#311](https://github.com/alecthomas/voluptuous/pull/311): Add `Schema.infer` method for basic schema inference. +- [#314](https://github.com/alecthomas/voluptuous/pull/314): Add `SomeOf` validator. + +**Fixes**: + +- [#279](https://github.com/alecthomas/voluptuous/pull/279): + Treat Python 2 old-style classes like types when validating. +- [#280](https://github.com/alecthomas/voluptuous/pull/280): Make + `IsDir()`, `IsFile()` and `PathExists()` consistent between different Python versions. +- [#290](https://github.com/alecthomas/voluptuous/pull/290): Use absolute imports to avoid import conflicts. +- [#291](https://github.com/alecthomas/voluptuous/pull/291): Fix `Coerce` validator to catch `decimal.InvalidOperation`. +- [#298](https://github.com/alecthomas/voluptuous/pull/298): Make `Schema([])` usage consistent with `Schema({})`. +- [#303](https://github.com/alecthomas/voluptuous/pull/303): Allow partial validation when using validate decorator. +- [#316](https://github.com/alecthomas/voluptuous/pull/316): Make `Schema.__eq__` deterministic. +- [#319](https://github.com/alecthomas/voluptuous/pull/319): Replace implementation of `Maybe(s)` with `Any(None, s)` to allow it to be compiled. ## [0.10.5] diff --git a/third_party/python/voluptuous/PKG-INFO b/third_party/python/voluptuous/PKG-INFO index 2c26019e50e1..999071c44eaf 100644 --- a/third_party/python/voluptuous/PKG-INFO +++ b/third_party/python/voluptuous/PKG-INFO @@ -1,16 +1,16 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: voluptuous -Version: 0.10.5 -Summary: Voluptuous is a Python data validation library +Version: 0.11.5 +Summary: # Voluptuous is a Python data validation library Home-page: https://github.com/alecthomas/voluptuous Author: Alec Thomas Author-email: alec@swapoff.org License: BSD Download-URL: https://pypi.python.org/pypi/voluptuous -Description: Voluptuous is a Python data validation library - ============================================== +Description: # Voluptuous is a Python data validation library - |Build Status| |Coverage Status| |Gitter chat| + [![Build Status](https://travis-ci.org/alecthomas/voluptuous.png)](https://travis-ci.org/alecthomas/voluptuous) + [![Coverage Status](https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master)](https://coveralls.io/github/alecthomas/voluptuous?branch=master) [![Gitter chat](https://badges.gitter.im/alecthomas.png)](https://gitter.im/alecthomas/Lobby) Voluptuous, *despite* the name, is a Python data validation library. It is primarily intended for validating data coming into Python as JSON, @@ -18,251 +18,302 @@ Description: Voluptuous is a Python data validation library It has three goals: - 1. Simplicity. - 2. Support for complex data structures. - 3. Provide useful error messages. + 1. Simplicity. + 2. Support for complex data structures. + 3. Provide useful error messages. - Contact - ------- + ## Contact Voluptuous now has a mailing list! Send a mail to - ` `__ to - subscribe. Instructions will follow. + [](mailto:voluptuous@librelist.com) to subscribe. Instructions + will follow. - You can also contact me directly via `email `__ - or `Twitter `__. + You can also contact me directly via [email](mailto:alec@swapoff.org) or + [Twitter](https://twitter.com/alecthomas). - To file a bug, create a `new - issue `__ on GitHub - with a short example of how to replicate the issue. + To file a bug, create a [new issue](https://github.com/alecthomas/voluptuous/issues/new) on GitHub with a short example of how to replicate the issue. - Documentation - ------------- + ## Documentation - The documentation is provided [here] - (http://alecthomas.github.io/voluptuous/). + The documentation is provided [here](http://alecthomas.github.io/voluptuous/). - Changelog - --------- + ## Changelog - See `CHANGELOG.md `__. + See [CHANGELOG.md](https://github.com/alecthomas/voluptuous/blob/master/CHANGELOG.md). - Show me an example - ------------------ + ## Show me an example - Twitter's `user search - API `__ accepts + Twitter's [user search API](https://dev.twitter.com/rest/reference/get/users/search) accepts query URLs like: - :: - - $ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' + ``` + $ curl 'https://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' + ``` To validate this we might use a schema like: - .. code:: pycon + ```pycon + >>> from voluptuous import Schema + >>> schema = Schema({ + ... 'q': str, + ... 'per_page': int, + ... 'page': int, + ... }) - >>> from voluptuous import Schema - >>> schema = Schema({ - ... 'q': str, - ... 'per_page': int, - ... 'page': int, - ... }) + ``` This schema very succinctly and roughly describes the data required by the API, and will work fine. But it has a few problems. Firstly, it doesn't fully express the constraints of the API. According to the API, - ``per_page`` should be restricted to at most 20, defaulting to 5, for + `per_page` should be restricted to at most 20, defaulting to 5, for example. To describe the semantics of the API more accurately, our schema will need to be more thoroughly defined: - .. code:: pycon + ```pycon + >>> from voluptuous import Required, All, Length, Range + >>> schema = Schema({ + ... Required('q'): All(str, Length(min=1)), + ... Required('per_page', default=5): All(int, Range(min=1, max=20)), + ... 'page': All(int, Range(min=0)), + ... }) - >>> from voluptuous import Required, All, Length, Range - >>> schema = Schema({ - ... Required('q'): All(str, Length(min=1)), - ... Required('per_page', default=5): All(int, Range(min=1, max=20)), - ... 'page': All(int, Range(min=0)), - ... }) + ``` This schema fully enforces the interface defined in Twitter's documentation, and goes a little further for completeness. "q" is required: - .. code:: pycon + ```pycon + >>> from voluptuous import MultipleInvalid, Invalid + >>> try: + ... schema({}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data['q']" + True - >>> from voluptuous import MultipleInvalid, Invalid - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data['q']" - True + ``` ...must be a string: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': 123}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected str for dictionary value @ data['q']" + True - >>> try: - ... schema({'q': 123}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected str for dictionary value @ data['q']" - True + ``` ...and must be at least one character in length: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': ''}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" + True + >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} + True - >>> try: - ... schema({'q': ''}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" - True - >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} - True + ``` "per\_page" is a positive integer no greater than 20: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': '#topic', 'per_page': 900}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" + True + >>> try: + ... schema({'q': '#topic', 'per_page': -10}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" + True - >>> try: - ... schema({'q': '#topic', 'per_page': 900}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" - True - >>> try: - ... schema({'q': '#topic', 'per_page': -10}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" - True + ``` - "page" is an integer >= 0: + "page" is an integer \>= 0: - .. code:: pycon + ```pycon + >>> try: + ... schema({'q': '#topic', 'per_page': 'one'}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) + "expected int for dictionary value @ data['per_page']" + >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} + True - >>> try: - ... schema({'q': '#topic', 'per_page': 'one'}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "expected int for dictionary value @ data['per_page']" - >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} - True + ``` - Defining schemas - ---------------- + ## Defining schemas Schemas are nested data structures consisting of dictionaries, lists, scalars and *validators*. Each node in the input schema is pattern matched against corresponding nodes in the input data. - Literals - ~~~~~~~~ + ### Literals Literals in the schema are matched using normal equality checks: - .. code:: pycon + ```pycon + >>> schema = Schema(1) + >>> schema(1) + 1 + >>> schema = Schema('a string') + >>> schema('a string') + 'a string' - >>> schema = Schema(1) - >>> schema(1) - 1 - >>> schema = Schema('a string') - >>> schema('a string') - 'a string' + ``` - Types - ~~~~~ + ### Types Types in the schema are matched by checking if the corresponding value is an instance of the type: - .. code:: pycon + ```pycon + >>> schema = Schema(int) + >>> schema(1) + 1 + >>> try: + ... schema('one') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected int" + True - >>> schema = Schema(int) - >>> schema(1) - 1 - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected int" - True + ``` - URL's - ~~~~~ + ### URL's - URL's in the schema are matched by using ``urlparse`` library. + URL's in the schema are matched by using `urlparse` library. - .. code:: pycon + ```pycon + >>> from voluptuous import Url + >>> schema = Schema(Url()) + >>> schema('http://w3.org') + 'http://w3.org' + >>> try: + ... schema('one') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "expected a URL" + True - >>> from voluptuous import Url - >>> schema = Schema(Url()) - >>> schema('http://w3.org') - 'http://w3.org' - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected a URL" - True + ``` - Lists - ~~~~~ + ### Lists Lists in the schema are treated as a set of valid values. Each element in the schema list is compared to each value in the input data: - .. code:: pycon + ```pycon + >>> schema = Schema([1, 'a', 'string']) + >>> schema([1]) + [1] + >>> schema([1, 1, 1]) + [1, 1, 1] + >>> schema(['a', 1, 'string', 1, 'string']) + ['a', 1, 'string', 1, 'string'] - >>> schema = Schema([1, 'a', 'string']) - >>> schema([1]) - [1] - >>> schema([1, 1, 1]) - [1, 1, 1] - >>> schema(['a', 1, 'string', 1, 'string']) - ['a', 1, 'string', 1, 'string'] + ``` - However, an empty list (``[]``) is treated as is. If you want to specify - a list that can contain anything, specify it as ``list``: + However, an empty list (`[]`) is treated as is. If you want to specify a list that can + contain anything, specify it as `list`: - .. code:: pycon + ```pycon + >>> schema = Schema([]) + >>> try: + ... schema([1]) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value @ data[1]" + True + >>> schema([]) + [] + >>> schema = Schema(list) + >>> schema([]) + [] + >>> schema([1, 2]) + [1, 2] - >>> schema = Schema([]) - >>> try: - ... schema([1]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema([]) - [] - >>> schema = Schema(list) - >>> schema([]) - [] - >>> schema([1, 2]) - [1, 2] + ``` - Validation functions - ~~~~~~~~~~~~~~~~~~~~ + ### Sets and frozensets - Validators are simple callables that raise an ``Invalid`` exception when + Sets and frozensets are treated as a set of valid values. Each element + in the schema set is compared to each value in the input data: + + ```pycon + >>> schema = Schema({42}) + >>> schema({42}) == {42} + True + >>> try: + ... schema({43}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "invalid value in set" + True + >>> schema = Schema({int}) + >>> schema({1, 2, 3}) == {1, 2, 3} + True + >>> schema = Schema({int, str}) + >>> schema({1, 2, 'abc'}) == {1, 2, 'abc'} + True + >>> schema = Schema(frozenset([int])) + >>> try: + ... schema({3}) + ... raise AssertionError('Invalid not raised') + ... except Invalid as e: + ... exc = e + >>> str(exc) == 'expected a frozenset' + True + + ``` + + However, an empty set (`set()`) is treated as is. If you want to specify a set + that can contain anything, specify it as `set`: + + ```pycon + >>> schema = Schema(set()) + >>> try: + ... schema({1}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "invalid value in set" + True + >>> schema(set()) == set() + True + >>> schema = Schema(set) + >>> schema({1, 2}) == {1, 2} + True + + ``` + + ### Validation functions + + Validators are simple callables that raise an `Invalid` exception when they encounter invalid data. The criteria for determining validity is entirely up to the implementation; it may check that a value is a valid - username with ``pwd.getpwnam()``, it may check that a value is of a + username with `pwd.getpwnam()`, it may check that a value is of a specific type, and so on. The simplest kind of validator is a Python function that raises @@ -270,388 +321,416 @@ Description: Voluptuous is a Python data validation library Python functions have this property. Here's an example of a date validator: - .. code:: pycon + ```pycon + >>> from datetime import datetime + >>> def Date(fmt='%Y-%m-%d'): + ... return lambda v: datetime.strptime(v, fmt) - >>> from datetime import datetime - >>> def Date(fmt='%Y-%m-%d'): - ... return lambda v: datetime.strptime(v, fmt) + ``` - .. code:: pycon + ```pycon + >>> schema = Schema(Date()) + >>> schema('2013-03-03') + datetime.datetime(2013, 3, 3, 0, 0) + >>> try: + ... schema('2013-03') + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value" + True - >>> schema = Schema(Date()) - >>> schema('2013-03-03') - datetime.datetime(2013, 3, 3, 0, 0) - >>> try: - ... schema('2013-03') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True + ``` In addition to simply determining if a value is valid, validators may mutate the value into a valid form. An example of this is the - ``Coerce(type)`` function, which returns a function that coerces its + `Coerce(type)` function, which returns a function that coerces its argument to the given type: - .. code:: python + ```python + def Coerce(type, msg=None): + """Coerce a value to a type. - def Coerce(type, msg=None): - """Coerce a value to a type. + If the type constructor throws a ValueError, the value will be marked as + Invalid. + """ + def f(v): + try: + return type(v) + except ValueError: + raise Invalid(msg or ('expected %s' % type.__name__)) + return f - If the type constructor throws a ValueError, the value will be marked as - Invalid. - """ - def f(v): - try: - return type(v) - except ValueError: - raise Invalid(msg or ('expected %s' % type.__name__)) - return f + ``` This example also shows a common idiom where an optional human-readable message can be provided. This can vastly improve the usefulness of the resulting error messages. - Dictionaries - ~~~~~~~~~~~~ + ### Dictionaries Each key-value pair in a schema dictionary is validated against each key-value pair in the corresponding data dictionary: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 'one', 2: 'two'}) + >>> schema({1: 'one'}) + {1: 'one'} - >>> schema = Schema({1: 'one', 2: 'two'}) - >>> schema({1: 'one'}) - {1: 'one'} + ``` - Extra dictionary keys - ^^^^^^^^^^^^^^^^^^^^^ + #### Extra dictionary keys By default any additional keys in the data, not in the schema will trigger exceptions: - .. code:: pycon + ```pycon + >>> schema = Schema({2: 3}) + >>> try: + ... schema({1: 2, 2: 3}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "extra keys not allowed @ data[1]" + True - >>> schema = Schema({2: 3}) - >>> try: - ... schema({1: 2, 2: 3}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[1]" - True + ``` - This behaviour can be altered on a per-schema basis. To allow additional - keys use ``Schema(..., extra=ALLOW_EXTRA)``: + This behaviour can be altered on a per-schema basis. To allow + additional keys use + `Schema(..., extra=ALLOW_EXTRA)`: - .. code:: pycon + ```pycon + >>> from voluptuous import ALLOW_EXTRA + >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) + >>> schema({1: 2, 2: 3}) + {1: 2, 2: 3} - >>> from voluptuous import ALLOW_EXTRA - >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) - >>> schema({1: 2, 2: 3}) - {1: 2, 2: 3} + ``` - To remove additional keys use ``Schema(..., extra=REMOVE_EXTRA)``: + To remove additional keys use + `Schema(..., extra=REMOVE_EXTRA)`: - .. code:: pycon + ```pycon + >>> from voluptuous import REMOVE_EXTRA + >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) + >>> schema({1: 2, 2: 3}) + {2: 3} - >>> from voluptuous import REMOVE_EXTRA - >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) - >>> schema({1: 2, 2: 3}) - {2: 3} + ``` It can also be overridden per-dictionary by using the catch-all marker - token ``extra`` as a key: + token `extra` as a key: - .. code:: pycon + ```pycon + >>> from voluptuous import Extra + >>> schema = Schema({1: {Extra: object}}) + >>> schema({1: {'foo': 'bar'}}) + {1: {'foo': 'bar'}} - >>> from voluptuous import Extra - >>> schema = Schema({1: {Extra: object}}) - >>> schema({1: {'foo': 'bar'}}) - {1: {'foo': 'bar'}} + ``` - However, an empty dict (``{}``) is treated as is. If you want to specify - a list that can contain anything, specify it as ``dict``: - - .. code:: pycon - - >>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this - >>> try: - ... schema({'extra': 1}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema({}) - {} - >>> schema = Schema(dict) # do this instead - >>> schema({}) - {} - >>> schema({'extra': 1}) - {'extra': 1} - - Required dictionary keys - ^^^^^^^^^^^^^^^^^^^^^^^^ + #### Required dictionary keys By default, keys in the schema are not required to be in the data: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 2, 3: 4}) + >>> schema({3: 4}) + {3: 4} - >>> schema = Schema({1: 2, 3: 4}) - >>> schema({3: 4}) - {3: 4} + ``` Similarly to how extra\_ keys work, this behaviour can be overridden per-schema: - .. code:: pycon + ```pycon + >>> schema = Schema({1: 2, 3: 4}, required=True) + >>> try: + ... schema({3: 4}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True - >>> schema = Schema({1: 2, 3: 4}, required=True) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True + ``` - And per-key, with the marker token ``Required(key)``: + And per-key, with the marker token `Required(key)`: - .. code:: pycon + ```pycon + >>> schema = Schema({Required(1): 2, 3: 4}) + >>> try: + ... schema({3: 4}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True + >>> schema({1: 2}) + {1: 2} - >>> schema = Schema({Required(1): 2, 3: 4}) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} + ``` - Optional dictionary keys - ^^^^^^^^^^^^^^^^^^^^^^^^ + #### Optional dictionary keys - If a schema has ``required=True``, keys may be individually marked as - optional using the marker token ``Optional(key)``: + If a schema has `required=True`, keys may be individually marked as + optional using the marker token `Optional(key)`: - .. code:: pycon + ```pycon + >>> from voluptuous import Optional + >>> schema = Schema({1: 2, Optional(3): 4}, required=True) + >>> try: + ... schema({}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "required key not provided @ data[1]" + True + >>> schema({1: 2}) + {1: 2} + >>> try: + ... schema({1: 2, 4: 5}) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "extra keys not allowed @ data[4]" + True - >>> from voluptuous import Optional - >>> schema = Schema({1: 2, Optional(3): 4}, required=True) - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - >>> try: - ... schema({1: 2, 4: 5}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[4]" - True + ``` - .. code:: pycon + ```pycon + >>> schema({1: 2, 3: 4}) + {1: 2, 3: 4} - >>> schema({1: 2, 3: 4}) - {1: 2, 3: 4} + ``` - Recursive schema - ~~~~~~~~~~~~~~~~ + ### Recursive / nested schema - There is no syntax to have a recursive schema. The best way to do it is - to have a wrapper like this: + You can use `voluptuous.Self` to define a nested schema: - .. code:: pycon + ```pycon + >>> from voluptuous import Schema, Self + >>> recursive = Schema({"more": Self, "value": int}) + >>> recursive({"more": {"value": 42}, "value": 41}) == {'more': {'value': 42}, 'value': 41} + True - >>> from voluptuous import Schema, Any - >>> def s2(v): - ... return s1(v) - ... - >>> s1 = Schema({"key": Any(s2, "value")}) - >>> s1({"key": {"key": "value"}}) - {'key': {'key': 'value'}} + ``` - Extending an existing Schema - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ### Extending an existing Schema - Often it comes handy to have a base ``Schema`` that is extended with - more requirements. In that case you can use ``Schema.extend`` to create - a new ``Schema``: + Often it comes handy to have a base `Schema` that is extended with more + requirements. In that case you can use `Schema.extend` to create a new + `Schema`: - .. code:: pycon + ```pycon + >>> from voluptuous import Schema + >>> person = Schema({'name': str}) + >>> person_with_age = person.extend({'age': int}) + >>> sorted(list(person_with_age.schema.keys())) + ['age', 'name'] - >>> from voluptuous import Schema - >>> person = Schema({'name': str}) - >>> person_with_age = person.extend({'age': int}) - >>> sorted(list(person_with_age.schema.keys())) - ['age', 'name'] + ``` - The original ``Schema`` remains unchanged. + The original `Schema` remains unchanged. - Objects - ~~~~~~~ + ### Objects Each key-value pair in a schema dictionary is validated against each attribute-value pair in the corresponding object: - .. code:: pycon + ```pycon + >>> from voluptuous import Object + >>> class Structure(object): + ... def __init__(self, q=None): + ... self.q = q + ... def __repr__(self): + ... return ''.format(self) + ... + >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) + >>> schema(Structure(q='one')) + - >>> from voluptuous import Object - >>> class Structure(object): - ... def __init__(self, q=None): - ... self.q = q - ... def __repr__(self): - ... return ''.format(self) - ... - >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) - >>> schema(Structure(q='one')) - + ``` - Allow None values - ~~~~~~~~~~~~~~~~~ + ### Allow None values To allow value to be None as well, use Any: - .. code:: pycon + ```pycon + >>> from voluptuous import Any - >>> from voluptuous import Any + >>> schema = Schema(Any(None, int)) + >>> schema(None) + >>> schema(5) + 5 - >>> schema = Schema(Any(None, int)) - >>> schema(None) - >>> schema(5) - 5 + ``` - Error reporting - --------------- + ## Error reporting - Validators must throw an ``Invalid`` exception if invalid data is passed + Validators must throw an `Invalid` exception if invalid data is passed to them. All other exceptions are treated as errors in the validator and will not be caught. - Each ``Invalid`` exception has an associated ``path`` attribute - representing the path in the data structure to our currently validating - value, as well as an ``error_message`` attribute that contains the - message of the original exception. This is especially useful when you - want to catch ``Invalid`` exceptions and give some feedback to the user, - for instance in the context of an HTTP API. + Each `Invalid` exception has an associated `path` attribute representing + the path in the data structure to our currently validating value, as well + as an `error_message` attribute that contains the message of the original + exception. This is especially useful when you want to catch `Invalid` + exceptions and give some feedback to the user, for instance in the context of + an HTTP API. - .. code:: pycon - >>> def validate_email(email): - ... """Validate email.""" - ... if not "@" in email: - ... raise Invalid("This email is invalid.") - ... return email - >>> schema = Schema({"email": validate_email}) - >>> exc = None - >>> try: - ... schema({"email": "whatever"}) - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "This email is invalid. for dictionary value @ data['email']" - >>> exc.path - ['email'] - >>> exc.msg - 'This email is invalid.' - >>> exc.error_message - 'This email is invalid.' + ```pycon + >>> def validate_email(email): + ... """Validate email.""" + ... if not "@" in email: + ... raise Invalid("This email is invalid.") + ... return email + >>> schema = Schema({"email": validate_email}) + >>> exc = None + >>> try: + ... schema({"email": "whatever"}) + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) + "This email is invalid. for dictionary value @ data['email']" + >>> exc.path + ['email'] + >>> exc.msg + 'This email is invalid.' + >>> exc.error_message + 'This email is invalid.' - The ``path`` attribute is used during error reporting, but also during - matching to determine whether an error should be reported to the user or - if the next match should be attempted. This is determined by comparing - the depth of the path where the check is, to the depth of the path where - the error occurred. If the error is more than one level deeper, it is - reported. + ``` + + The `path` attribute is used during error reporting, but also during matching + to determine whether an error should be reported to the user or if the next + match should be attempted. This is determined by comparing the depth of the + path where the check is, to the depth of the path where the error occurred. If + the error is more than one level deeper, it is reported. The upshot of this is that *matching is depth-first and fail-fast*. To illustrate this, here is an example schema: - .. code:: pycon + ```pycon + >>> schema = Schema([[2, 3], 6]) - >>> schema = Schema([[2, 3], 6]) + ``` Each value in the top-level list is matched depth-first in-order. Given - input data of ``[[6]]``, the inner list will match the first element of - the schema, but the literal ``6`` will not match any of the elements of + input data of `[[6]]`, the inner list will match the first element of + the schema, but the literal `6` will not match any of the elements of that list. This error will be reported back to the user immediately. No backtracking is attempted: - .. code:: pycon + ```pycon + >>> try: + ... schema([[6]]) + ... raise AssertionError('MultipleInvalid not raised') + ... except MultipleInvalid as e: + ... exc = e + >>> str(exc) == "not a valid value @ data[0][0]" + True - >>> try: - ... schema([[6]]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value @ data[0][0]" - True + ``` - If we pass the data ``[6]``, the ``6`` is not a list type and so will - not recurse into the first element of the schema. Matching will continue - on to the second element in the schema, and succeed: + If we pass the data `[6]`, the `6` is not a list type and so will not + recurse into the first element of the schema. Matching will continue on + to the second element in the schema, and succeed: - .. code:: pycon + ```pycon + >>> schema([6]) + [6] - >>> schema([6]) - [6] + ``` - Running tests. - -------------- + ## Multi-field validation + + Validation rules that involve multiple fields can be implemented as + custom validators. It's recommended to use `All()` to do a two-pass + validation - the first pass checking the basic structure of the data, + and only after that, the second pass applying your cross-field + validator: + + ```python + def passwords_must_match(passwords): + if passwords['password'] != passwords['password_again']: + raise Invalid('passwords must match') + return passwords + + s=Schema(All( + # First "pass" for field types + {'password':str, 'password_again':str}, + # Follow up the first "pass" with your multi-field rules + passwords_must_match + )) + + # valid + s({'password':'123', 'password_again':'123'}) + + # raises MultipleInvalid: passwords must match + s({'password':'123', 'password_again':'and now for something completely different'}) + + ``` + + With this structure, your multi-field validator will run with + pre-validated data from the first "pass" and so will not have to do + its own type checking on its inputs. + + The flipside is that if the first "pass" of validation fails, your + cross-field validator will not run: + + ``` + # raises Invalid because password_again is not a string + # passwords_must_match() will not run because first-pass validation already failed + s({'password':'123', 'password_again': 1337}) + ``` + + ## Running tests. Voluptuous is using nosetests: - :: - $ nosetests - Why use Voluptuous over another validation library? - --------------------------------------------------- + + ## Why use Voluptuous over another validation library? **Validators are simple callables** - No need to subclass anything, just use a function. - **Errors are simple exceptions.** - A validator can just ``raise Invalid(msg)`` and expect the user to - get useful messages. - **Schemas are basic Python data structures.** - Should your data be a dictionary of integer keys to strings? - ``{int: str}`` does what you expect. List of integers, floats or - strings? ``[int, float, str]``. - **Designed from the ground up for validating more than just forms.** - Nested data structures are treated in the same way as any other - type. Need a list of dictionaries? ``[{}]`` - **Consistency.** - Types in the schema are checked as types. Values are compared as - values. Callables are called to validate. Simple. + : No need to subclass anything, just use a function. - Other libraries and inspirations - -------------------------------- + **Errors are simple exceptions.** + : A validator can just `raise Invalid(msg)` and expect the user to get + useful messages. + + **Schemas are basic Python data structures.** + : Should your data be a dictionary of integer keys to strings? + `{int: str}` does what you expect. List of integers, floats or + strings? `[int, float, str]`. + + **Designed from the ground up for validating more than just forms.** + : Nested data structures are treated in the same way as any other + type. Need a list of dictionaries? `[{}]` + + **Consistency.** + : Types in the schema are checked as types. Values are compared as + values. Callables are called to validate. Simple. + + ## Other libraries and inspirations Voluptuous is heavily inspired by - `Validino `__, and to a lesser - extent, `jsonvalidator `__ and - `json\_schema `__. + [Validino](http://code.google.com/p/validino/), and to a lesser extent, + [jsonvalidator](http://code.google.com/p/jsonvalidator/) and + [json\_schema](http://blog.sendapatch.se/category/json_schema.html). + + [pytest-voluptuous](https://github.com/F-Secure/pytest-voluptuous) is a + [pytest](https://github.com/pytest-dev/pytest) plugin that helps in + using voluptuous validators in `assert`s. I greatly prefer the light-weight style promoted by these libraries to the complexity of libraries like FormEncode. - .. |Build Status| image:: https://travis-ci.org/alecthomas/voluptuous.png - :target: https://travis-ci.org/alecthomas/voluptuous - .. |Coverage Status| image:: https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master - :target: https://coveralls.io/github/alecthomas/voluptuous?branch=master - .. |Gitter chat| image:: https://badges.gitter.im/alecthomas.png - :target: https://gitter.im/alecthomas/Lobby - Platform: any Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers @@ -660,7 +739,6 @@ Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.1 -Classifier: Programming Language :: Python :: 3.2 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Description-Content-Type: text/markdown diff --git a/third_party/python/voluptuous/README.md b/third_party/python/voluptuous/README.md index d2341b66c7e5..46e2288f4bea 100644 --- a/third_party/python/voluptuous/README.md +++ b/third_party/python/voluptuous/README.md @@ -26,11 +26,11 @@ To file a bug, create a [new issue](https://github.com/alecthomas/voluptuous/iss ## Documentation -The documentation is provided [here] (http://alecthomas.github.io/voluptuous/). +The documentation is provided [here](http://alecthomas.github.io/voluptuous/). ## Changelog -See [CHANGELOG.md](CHANGELOG.md). +See [CHANGELOG.md](https://github.com/alecthomas/voluptuous/blob/master/CHANGELOG.md). ## Show me an example @@ -38,7 +38,7 @@ Twitter's [user search API](https://dev.twitter.com/rest/reference/get/users/sea query URLs like: ``` -$ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' +$ curl 'https://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' ``` To validate this we might use a schema like: @@ -234,7 +234,7 @@ contain anything, specify it as `list`: ... raise AssertionError('MultipleInvalid not raised') ... except MultipleInvalid as e: ... exc = e ->>> str(exc) == "not a valid value" +>>> str(exc) == "not a valid value @ data[1]" True >>> schema([]) [] @@ -246,6 +246,59 @@ True ``` +### Sets and frozensets + +Sets and frozensets are treated as a set of valid values. Each element +in the schema set is compared to each value in the input data: + +```pycon +>>> schema = Schema({42}) +>>> schema({42}) == {42} +True +>>> try: +... schema({43}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema = Schema({int}) +>>> schema({1, 2, 3}) == {1, 2, 3} +True +>>> schema = Schema({int, str}) +>>> schema({1, 2, 'abc'}) == {1, 2, 'abc'} +True +>>> schema = Schema(frozenset([int])) +>>> try: +... schema({3}) +... raise AssertionError('Invalid not raised') +... except Invalid as e: +... exc = e +>>> str(exc) == 'expected a frozenset' +True + +``` + +However, an empty set (`set()`) is treated as is. If you want to specify a set +that can contain anything, specify it as `set`: + +```pycon +>>> schema = Schema(set()) +>>> try: +... schema({1}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema(set()) == set() +True +>>> schema = Schema(set) +>>> schema({1, 2}) == {1, 2} +True + +``` + ### Validation functions Validators are simple callables that raise an `Invalid` exception when @@ -368,28 +421,6 @@ token `extra` as a key: ``` -However, an empty dict (`{}`) is treated as is. If you want to specify a list that can -contain anything, specify it as `dict`: - -```pycon ->>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this ->>> try: -... schema({'extra': 1}) -... raise AssertionError('MultipleInvalid not raised') -... except MultipleInvalid as e: -... exc = e ->>> str(exc) == "not a valid value" -True ->>> schema({}) -{} ->>> schema = Schema(dict) # do this instead ->>> schema({}) -{} ->>> schema({'extra': 1}) -{'extra': 1} - -``` - #### Required dictionary keys By default, keys in the schema are not required to be in the data: @@ -465,18 +496,15 @@ True ``` -### Recursive schema +### Recursive / nested schema -There is no syntax to have a recursive schema. The best way to do it is to have a wrapper like this: +You can use `voluptuous.Self` to define a nested schema: ```pycon ->>> from voluptuous import Schema, Any ->>> def s2(v): -... return s1(v) -... ->>> s1 = Schema({"key": Any(s2, "value")}) ->>> s1({"key": {"key": "value"}}) -{'key': {'key': 'value'}} +>>> from voluptuous import Schema, Self +>>> recursive = Schema({"more": Self, "value": int}) +>>> recursive({"more": {"value": 42}, "value": 41}) == {'more': {'value': 42}, 'value': 41} +True ``` @@ -609,6 +637,48 @@ to the second element in the schema, and succeed: ``` +## Multi-field validation + +Validation rules that involve multiple fields can be implemented as +custom validators. It's recommended to use `All()` to do a two-pass +validation - the first pass checking the basic structure of the data, +and only after that, the second pass applying your cross-field +validator: + +```python +def passwords_must_match(passwords): + if passwords['password'] != passwords['password_again']: + raise Invalid('passwords must match') + return passwords + +s=Schema(All( + # First "pass" for field types + {'password':str, 'password_again':str}, + # Follow up the first "pass" with your multi-field rules + passwords_must_match +)) + +# valid +s({'password':'123', 'password_again':'123'}) + +# raises MultipleInvalid: passwords must match +s({'password':'123', 'password_again':'and now for something completely different'}) + +``` + +With this structure, your multi-field validator will run with +pre-validated data from the first "pass" and so will not have to do +its own type checking on its inputs. + +The flipside is that if the first "pass" of validation fails, your +cross-field validator will not run: + +``` +# raises Invalid because password_again is not a string +# passwords_must_match() will not run because first-pass validation already failed +s({'password':'123', 'password_again': 1337}) +``` + ## Running tests. Voluptuous is using nosetests: @@ -645,5 +715,9 @@ Voluptuous is heavily inspired by [jsonvalidator](http://code.google.com/p/jsonvalidator/) and [json\_schema](http://blog.sendapatch.se/category/json_schema.html). +[pytest-voluptuous](https://github.com/F-Secure/pytest-voluptuous) is a +[pytest](https://github.com/pytest-dev/pytest) plugin that helps in +using voluptuous validators in `assert`s. + I greatly prefer the light-weight style promoted by these libraries to the complexity of libraries like FormEncode. diff --git a/third_party/python/voluptuous/README.rst b/third_party/python/voluptuous/README.rst deleted file mode 100644 index 3bf3e2e5d5ce..000000000000 --- a/third_party/python/voluptuous/README.rst +++ /dev/null @@ -1,644 +0,0 @@ -Voluptuous is a Python data validation library -============================================== - -|Build Status| |Coverage Status| |Gitter chat| - -Voluptuous, *despite* the name, is a Python data validation library. It -is primarily intended for validating data coming into Python as JSON, -YAML, etc. - -It has three goals: - -1. Simplicity. -2. Support for complex data structures. -3. Provide useful error messages. - -Contact -------- - -Voluptuous now has a mailing list! Send a mail to -` `__ to -subscribe. Instructions will follow. - -You can also contact me directly via `email `__ -or `Twitter `__. - -To file a bug, create a `new -issue `__ on GitHub -with a short example of how to replicate the issue. - -Documentation -------------- - -The documentation is provided [here] -(http://alecthomas.github.io/voluptuous/). - -Changelog ---------- - -See `CHANGELOG.md `__. - -Show me an example ------------------- - -Twitter's `user search -API `__ accepts -query URLs like: - -:: - - $ curl 'http://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' - -To validate this we might use a schema like: - -.. code:: pycon - - >>> from voluptuous import Schema - >>> schema = Schema({ - ... 'q': str, - ... 'per_page': int, - ... 'page': int, - ... }) - -This schema very succinctly and roughly describes the data required by -the API, and will work fine. But it has a few problems. Firstly, it -doesn't fully express the constraints of the API. According to the API, -``per_page`` should be restricted to at most 20, defaulting to 5, for -example. To describe the semantics of the API more accurately, our -schema will need to be more thoroughly defined: - -.. code:: pycon - - >>> from voluptuous import Required, All, Length, Range - >>> schema = Schema({ - ... Required('q'): All(str, Length(min=1)), - ... Required('per_page', default=5): All(int, Range(min=1, max=20)), - ... 'page': All(int, Range(min=0)), - ... }) - -This schema fully enforces the interface defined in Twitter's -documentation, and goes a little further for completeness. - -"q" is required: - -.. code:: pycon - - >>> from voluptuous import MultipleInvalid, Invalid - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data['q']" - True - -...must be a string: - -.. code:: pycon - - >>> try: - ... schema({'q': 123}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected str for dictionary value @ data['q']" - True - -...and must be at least one character in length: - -.. code:: pycon - - >>> try: - ... schema({'q': ''}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" - True - >>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} - True - -"per\_page" is a positive integer no greater than 20: - -.. code:: pycon - - >>> try: - ... schema({'q': '#topic', 'per_page': 900}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" - True - >>> try: - ... schema({'q': '#topic', 'per_page': -10}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" - True - -"page" is an integer >= 0: - -.. code:: pycon - - >>> try: - ... schema({'q': '#topic', 'per_page': 'one'}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "expected int for dictionary value @ data['per_page']" - >>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} - True - -Defining schemas ----------------- - -Schemas are nested data structures consisting of dictionaries, lists, -scalars and *validators*. Each node in the input schema is pattern -matched against corresponding nodes in the input data. - -Literals -~~~~~~~~ - -Literals in the schema are matched using normal equality checks: - -.. code:: pycon - - >>> schema = Schema(1) - >>> schema(1) - 1 - >>> schema = Schema('a string') - >>> schema('a string') - 'a string' - -Types -~~~~~ - -Types in the schema are matched by checking if the corresponding value -is an instance of the type: - -.. code:: pycon - - >>> schema = Schema(int) - >>> schema(1) - 1 - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected int" - True - -URL's -~~~~~ - -URL's in the schema are matched by using ``urlparse`` library. - -.. code:: pycon - - >>> from voluptuous import Url - >>> schema = Schema(Url()) - >>> schema('http://w3.org') - 'http://w3.org' - >>> try: - ... schema('one') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "expected a URL" - True - -Lists -~~~~~ - -Lists in the schema are treated as a set of valid values. Each element -in the schema list is compared to each value in the input data: - -.. code:: pycon - - >>> schema = Schema([1, 'a', 'string']) - >>> schema([1]) - [1] - >>> schema([1, 1, 1]) - [1, 1, 1] - >>> schema(['a', 1, 'string', 1, 'string']) - ['a', 1, 'string', 1, 'string'] - -However, an empty list (``[]``) is treated as is. If you want to specify -a list that can contain anything, specify it as ``list``: - -.. code:: pycon - - >>> schema = Schema([]) - >>> try: - ... schema([1]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema([]) - [] - >>> schema = Schema(list) - >>> schema([]) - [] - >>> schema([1, 2]) - [1, 2] - -Validation functions -~~~~~~~~~~~~~~~~~~~~ - -Validators are simple callables that raise an ``Invalid`` exception when -they encounter invalid data. The criteria for determining validity is -entirely up to the implementation; it may check that a value is a valid -username with ``pwd.getpwnam()``, it may check that a value is of a -specific type, and so on. - -The simplest kind of validator is a Python function that raises -ValueError when its argument is invalid. Conveniently, many builtin -Python functions have this property. Here's an example of a date -validator: - -.. code:: pycon - - >>> from datetime import datetime - >>> def Date(fmt='%Y-%m-%d'): - ... return lambda v: datetime.strptime(v, fmt) - -.. code:: pycon - - >>> schema = Schema(Date()) - >>> schema('2013-03-03') - datetime.datetime(2013, 3, 3, 0, 0) - >>> try: - ... schema('2013-03') - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - -In addition to simply determining if a value is valid, validators may -mutate the value into a valid form. An example of this is the -``Coerce(type)`` function, which returns a function that coerces its -argument to the given type: - -.. code:: python - - def Coerce(type, msg=None): - """Coerce a value to a type. - - If the type constructor throws a ValueError, the value will be marked as - Invalid. - """ - def f(v): - try: - return type(v) - except ValueError: - raise Invalid(msg or ('expected %s' % type.__name__)) - return f - -This example also shows a common idiom where an optional human-readable -message can be provided. This can vastly improve the usefulness of the -resulting error messages. - -Dictionaries -~~~~~~~~~~~~ - -Each key-value pair in a schema dictionary is validated against each -key-value pair in the corresponding data dictionary: - -.. code:: pycon - - >>> schema = Schema({1: 'one', 2: 'two'}) - >>> schema({1: 'one'}) - {1: 'one'} - -Extra dictionary keys -^^^^^^^^^^^^^^^^^^^^^ - -By default any additional keys in the data, not in the schema will -trigger exceptions: - -.. code:: pycon - - >>> schema = Schema({2: 3}) - >>> try: - ... schema({1: 2, 2: 3}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[1]" - True - -This behaviour can be altered on a per-schema basis. To allow additional -keys use ``Schema(..., extra=ALLOW_EXTRA)``: - -.. code:: pycon - - >>> from voluptuous import ALLOW_EXTRA - >>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) - >>> schema({1: 2, 2: 3}) - {1: 2, 2: 3} - -To remove additional keys use ``Schema(..., extra=REMOVE_EXTRA)``: - -.. code:: pycon - - >>> from voluptuous import REMOVE_EXTRA - >>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) - >>> schema({1: 2, 2: 3}) - {2: 3} - -It can also be overridden per-dictionary by using the catch-all marker -token ``extra`` as a key: - -.. code:: pycon - - >>> from voluptuous import Extra - >>> schema = Schema({1: {Extra: object}}) - >>> schema({1: {'foo': 'bar'}}) - {1: {'foo': 'bar'}} - -However, an empty dict (``{}``) is treated as is. If you want to specify -a list that can contain anything, specify it as ``dict``: - -.. code:: pycon - - >>> schema = Schema({}, extra=ALLOW_EXTRA) # don't do this - >>> try: - ... schema({'extra': 1}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value" - True - >>> schema({}) - {} - >>> schema = Schema(dict) # do this instead - >>> schema({}) - {} - >>> schema({'extra': 1}) - {'extra': 1} - -Required dictionary keys -^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, keys in the schema are not required to be in the data: - -.. code:: pycon - - >>> schema = Schema({1: 2, 3: 4}) - >>> schema({3: 4}) - {3: 4} - -Similarly to how extra\_ keys work, this behaviour can be overridden -per-schema: - -.. code:: pycon - - >>> schema = Schema({1: 2, 3: 4}, required=True) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - -And per-key, with the marker token ``Required(key)``: - -.. code:: pycon - - >>> schema = Schema({Required(1): 2, 3: 4}) - >>> try: - ... schema({3: 4}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - -Optional dictionary keys -^^^^^^^^^^^^^^^^^^^^^^^^ - -If a schema has ``required=True``, keys may be individually marked as -optional using the marker token ``Optional(key)``: - -.. code:: pycon - - >>> from voluptuous import Optional - >>> schema = Schema({1: 2, Optional(3): 4}, required=True) - >>> try: - ... schema({}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "required key not provided @ data[1]" - True - >>> schema({1: 2}) - {1: 2} - >>> try: - ... schema({1: 2, 4: 5}) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "extra keys not allowed @ data[4]" - True - -.. code:: pycon - - >>> schema({1: 2, 3: 4}) - {1: 2, 3: 4} - -Recursive schema -~~~~~~~~~~~~~~~~ - -There is no syntax to have a recursive schema. The best way to do it is -to have a wrapper like this: - -.. code:: pycon - - >>> from voluptuous import Schema, Any - >>> def s2(v): - ... return s1(v) - ... - >>> s1 = Schema({"key": Any(s2, "value")}) - >>> s1({"key": {"key": "value"}}) - {'key': {'key': 'value'}} - -Extending an existing Schema -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Often it comes handy to have a base ``Schema`` that is extended with -more requirements. In that case you can use ``Schema.extend`` to create -a new ``Schema``: - -.. code:: pycon - - >>> from voluptuous import Schema - >>> person = Schema({'name': str}) - >>> person_with_age = person.extend({'age': int}) - >>> sorted(list(person_with_age.schema.keys())) - ['age', 'name'] - -The original ``Schema`` remains unchanged. - -Objects -~~~~~~~ - -Each key-value pair in a schema dictionary is validated against each -attribute-value pair in the corresponding object: - -.. code:: pycon - - >>> from voluptuous import Object - >>> class Structure(object): - ... def __init__(self, q=None): - ... self.q = q - ... def __repr__(self): - ... return ''.format(self) - ... - >>> schema = Schema(Object({'q': 'one'}, cls=Structure)) - >>> schema(Structure(q='one')) - - -Allow None values -~~~~~~~~~~~~~~~~~ - -To allow value to be None as well, use Any: - -.. code:: pycon - - >>> from voluptuous import Any - - >>> schema = Schema(Any(None, int)) - >>> schema(None) - >>> schema(5) - 5 - -Error reporting ---------------- - -Validators must throw an ``Invalid`` exception if invalid data is passed -to them. All other exceptions are treated as errors in the validator and -will not be caught. - -Each ``Invalid`` exception has an associated ``path`` attribute -representing the path in the data structure to our currently validating -value, as well as an ``error_message`` attribute that contains the -message of the original exception. This is especially useful when you -want to catch ``Invalid`` exceptions and give some feedback to the user, -for instance in the context of an HTTP API. - -.. code:: pycon - - >>> def validate_email(email): - ... """Validate email.""" - ... if not "@" in email: - ... raise Invalid("This email is invalid.") - ... return email - >>> schema = Schema({"email": validate_email}) - >>> exc = None - >>> try: - ... schema({"email": "whatever"}) - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) - "This email is invalid. for dictionary value @ data['email']" - >>> exc.path - ['email'] - >>> exc.msg - 'This email is invalid.' - >>> exc.error_message - 'This email is invalid.' - -The ``path`` attribute is used during error reporting, but also during -matching to determine whether an error should be reported to the user or -if the next match should be attempted. This is determined by comparing -the depth of the path where the check is, to the depth of the path where -the error occurred. If the error is more than one level deeper, it is -reported. - -The upshot of this is that *matching is depth-first and fail-fast*. - -To illustrate this, here is an example schema: - -.. code:: pycon - - >>> schema = Schema([[2, 3], 6]) - -Each value in the top-level list is matched depth-first in-order. Given -input data of ``[[6]]``, the inner list will match the first element of -the schema, but the literal ``6`` will not match any of the elements of -that list. This error will be reported back to the user immediately. No -backtracking is attempted: - -.. code:: pycon - - >>> try: - ... schema([[6]]) - ... raise AssertionError('MultipleInvalid not raised') - ... except MultipleInvalid as e: - ... exc = e - >>> str(exc) == "not a valid value @ data[0][0]" - True - -If we pass the data ``[6]``, the ``6`` is not a list type and so will -not recurse into the first element of the schema. Matching will continue -on to the second element in the schema, and succeed: - -.. code:: pycon - - >>> schema([6]) - [6] - -Running tests. --------------- - -Voluptuous is using nosetests: - -:: - - $ nosetests - -Why use Voluptuous over another validation library? ---------------------------------------------------- - -**Validators are simple callables** - No need to subclass anything, just use a function. -**Errors are simple exceptions.** - A validator can just ``raise Invalid(msg)`` and expect the user to - get useful messages. -**Schemas are basic Python data structures.** - Should your data be a dictionary of integer keys to strings? - ``{int: str}`` does what you expect. List of integers, floats or - strings? ``[int, float, str]``. -**Designed from the ground up for validating more than just forms.** - Nested data structures are treated in the same way as any other - type. Need a list of dictionaries? ``[{}]`` -**Consistency.** - Types in the schema are checked as types. Values are compared as - values. Callables are called to validate. Simple. - -Other libraries and inspirations --------------------------------- - -Voluptuous is heavily inspired by -`Validino `__, and to a lesser -extent, `jsonvalidator `__ and -`json\_schema `__. - -I greatly prefer the light-weight style promoted by these libraries to -the complexity of libraries like FormEncode. - -.. |Build Status| image:: https://travis-ci.org/alecthomas/voluptuous.png - :target: https://travis-ci.org/alecthomas/voluptuous -.. |Coverage Status| image:: https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master - :target: https://coveralls.io/github/alecthomas/voluptuous?branch=master -.. |Gitter chat| image:: https://badges.gitter.im/alecthomas.png - :target: https://gitter.im/alecthomas/Lobby diff --git a/third_party/python/voluptuous/setup.py b/third_party/python/voluptuous/setup.py index 03a1ddd4105e..6408f8ba962d 100644 --- a/third_party/python/voluptuous/setup.py +++ b/third_party/python/voluptuous/setup.py @@ -1,26 +1,16 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup import sys +import io import os import atexit sys.path.insert(0, '.') version = __import__('voluptuous').__version__ -try: - import pypandoc - long_description = pypandoc.convert('README.md', 'rst') - with open('README.rst', 'w') as f: - f.write(long_description) - atexit.register(lambda: os.unlink('README.rst')) -except (ImportError, OSError): - print('WARNING: Could not locate pandoc, using Markdown long_description.') - with open('README.md') as f: - long_description = f.read() -description = long_description.splitlines()[0].strip() +with io.open('README.md', encoding='utf-8') as f: + long_description = f.read() + description = long_description.splitlines()[0].strip() setup( @@ -30,6 +20,7 @@ setup( version=version, description=description, long_description=long_description, + long_description_content_type='text/markdown', license='BSD', platforms=['any'], packages=['voluptuous'], @@ -43,9 +34,7 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ] ) diff --git a/third_party/python/voluptuous/voluptuous/__init__.py b/third_party/python/voluptuous/voluptuous/__init__.py index a3d0567a4f0d..10236d5a6670 100644 --- a/third_party/python/voluptuous/voluptuous/__init__.py +++ b/third_party/python/voluptuous/voluptuous/__init__.py @@ -1,15 +1,9 @@ # flake8: noqa -try: - from schema_builder import * - from validators import * - from util import * - from error import * -except ImportError: - from .schema_builder import * - from .validators import * - from .util import * - from .error import * +from voluptuous.schema_builder import * +from voluptuous.validators import * +from voluptuous.util import * +from voluptuous.error import * -__version__ = '0.10.5' -__author__ = 'tusharmakkar08' +__version__ = '0.11.5' +__author__ = 'alecthomas' diff --git a/third_party/python/voluptuous/voluptuous/error.py b/third_party/python/voluptuous/voluptuous/error.py index aa87500a7684..86c4e0a35933 100644 --- a/third_party/python/voluptuous/voluptuous/error.py +++ b/third_party/python/voluptuous/voluptuous/error.py @@ -187,3 +187,13 @@ class NotInInvalid(Invalid): class ExactSequenceInvalid(Invalid): pass + + +class NotEnoughValid(Invalid): + """The value did not pass enough validations.""" + pass + + +class TooManyValid(Invalid): + """The value passed more than expected validations.""" + pass diff --git a/third_party/python/voluptuous/voluptuous/schema_builder.py b/third_party/python/voluptuous/voluptuous/schema_builder.py index 8b4981456fa1..8d7a81a3e3e5 100644 --- a/third_party/python/voluptuous/voluptuous/schema_builder.py +++ b/third_party/python/voluptuous/voluptuous/schema_builder.py @@ -6,11 +6,7 @@ import sys from contextlib import contextmanager import itertools - -try: - import error as er -except ImportError: - from . import error as er +from voluptuous import error as er if sys.version_info >= (3,): long = int @@ -126,6 +122,10 @@ class Undefined(object): UNDEFINED = Undefined() +def Self(): + raise er.SchemaError('"Self" should never be called') + + def default_factory(value): if value is UNDEFINED or callable(value): return value @@ -201,11 +201,57 @@ class Schema(object): self.extra = int(extra) # ensure the value is an integer self._compiled = self._compile(schema) + @classmethod + def infer(cls, data, **kwargs): + """Create a Schema from concrete data (e.g. an API response). + + For example, this will take a dict like: + + { + 'foo': 1, + 'bar': { + 'a': True, + 'b': False + }, + 'baz': ['purple', 'monkey', 'dishwasher'] + } + + And return a Schema: + + { + 'foo': int, + 'bar': { + 'a': bool, + 'b': bool + }, + 'baz': [str] + } + + Note: only very basic inference is supported. + """ + def value_to_schema_type(value): + if isinstance(value, dict): + if len(value) == 0: + return dict + return {k: value_to_schema_type(v) + for k, v in iteritems(value)} + if isinstance(value, list): + if len(value) == 0: + return list + else: + return [value_to_schema_type(v) + for v in value] + return type(value) + + return cls(value_to_schema_type(data), **kwargs) + def __eq__(self, other): - if str(other) == str(self.schema): - # Because repr is combination mixture of object and schema - return True - return False + if not isinstance(other, Schema): + return False + return other.schema == self.schema + + def __ne__(self, other): + return not (self == other) def __str__(self): return str(self.schema) @@ -228,16 +274,22 @@ class Schema(object): def _compile(self, schema): if schema is Extra: return lambda _, v: v + if schema is Self: + return lambda p, v: self._compiled(p, v) + elif hasattr(schema, "__voluptuous_compile__"): + return schema.__voluptuous_compile__(self) if isinstance(schema, Object): return self._compile_object(schema) - if isinstance(schema, collections.Mapping) and len(schema): + if isinstance(schema, collections.Mapping): return self._compile_dict(schema) - elif isinstance(schema, list) and len(schema): + elif isinstance(schema, list): return self._compile_list(schema) elif isinstance(schema, tuple): return self._compile_tuple(schema) + elif isinstance(schema, (frozenset, set)): + return self._compile_set(schema) type_ = type(schema) - if type_ is type: + if inspect.isclass(schema): type_ = schema if type_ in (bool, bytes, int, long, str, unicode, float, complex, object, list, dict, type(None)) or callable(schema): @@ -284,11 +336,25 @@ class Schema(object): def validate_mapping(path, iterable, out): required_keys = all_required_keys.copy() - # keeps track of all default keys that haven't been filled - default_keys = all_default_keys.copy() + + # Build a map of all provided key-value pairs. + # The type(out) is used to retain ordering in case a ordered + # map type is provided as input. + key_value_map = type(out)() + for key, value in iterable: + key_value_map[key] = value + + # Insert default values for non-existing keys. + for key in all_default_keys: + if not isinstance(key.default, Undefined) and \ + key.schema not in key_value_map: + # A default value has been specified for this missing + # key, insert it. + key_value_map[key.schema] = key.default() + error = None errors = [] - for key, value in iterable: + for key, value in key_value_map.items(): key_path = path + [key] remove_key = False @@ -338,12 +404,10 @@ class Schema(object): required_keys.discard(skey) break - # Key and value okay, mark any Required() fields as found. + # Key and value okay, mark as found in case it was + # a Required() field. required_keys.discard(skey) - # No need for a default if it was filled - default_keys.discard(skey) - break else: if remove_key: @@ -355,13 +419,6 @@ class Schema(object): errors.append(er.Invalid('extra keys not allowed', key_path)) # else REMOVE_EXTRA: ignore the key so it's removed from output - # set defaults for any that can have defaults - for key in default_keys: - if not isinstance(key.default, Undefined): # if the user provides a default with the node - out[key.schema] = key.default() - if key in required_keys: - required_keys.discard(key) - # for any required keys left that weren't found and don't have defaults: for key in required_keys: msg = key.msg if hasattr(key, 'msg') and key.msg else 'required key not provided' @@ -412,7 +469,7 @@ class Schema(object): A dictionary schema will only validate a dictionary: - >>> validate = Schema({'prop': str}) + >>> validate = Schema({}) >>> with raises(er.MultipleInvalid, 'expected a dictionary'): ... validate([]) @@ -427,6 +484,7 @@ class Schema(object): >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['two']"): ... validate({'two': 'three'}) + Validation function, in this case the "int" type: >>> validate = Schema({'one': 'two', 'three': 'four', int: str}) @@ -436,17 +494,10 @@ class Schema(object): >>> validate({10: 'twenty'}) {10: 'twenty'} - An empty dictionary is matched as value: - - >>> validate = Schema({}) - >>> with raises(er.MultipleInvalid, 'not a valid value'): - ... validate([]) - By default, a "type" in the schema (in this case "int") will be used purely to validate that the corresponding value is of that type. It will not Coerce the value: - >>> validate = Schema({'one': 'two', 'three': 'four', int: str}) >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['10']"): ... validate({'10': 'twenty'}) @@ -561,6 +612,10 @@ class Schema(object): # Empty seq schema, allow any data. if not schema: + if data: + raise er.MultipleInvalid([ + er.ValueInvalid('not a valid value', [value]) for value in data + ]) return data out = [] @@ -622,6 +677,46 @@ class Schema(object): """ return self._compile_sequence(schema, list) + def _compile_set(self, schema): + """Validate a set. + + A set is an unordered collection of unique elements. + + >>> validator = Schema({int}) + >>> validator(set([42])) == set([42]) + True + >>> with raises(er.Invalid, 'expected a set'): + ... validator(42) + >>> with raises(er.MultipleInvalid, 'invalid value in set'): + ... validator(set(['a'])) + """ + type_ = type(schema) + type_name = type_.__name__ + + def validate_set(path, data): + if not isinstance(data, type_): + raise er.Invalid('expected a %s' % type_name, path) + + _compiled = [self._compile(s) for s in schema] + errors = [] + for value in data: + for validate in _compiled: + try: + validate(path, value) + break + except er.Invalid: + pass + else: + invalid = er.Invalid('invalid value in %s' % type_name, path) + errors.append(invalid) + + if errors: + raise er.MultipleInvalid(errors) + + return data + + return validate_set + def extend(self, schema, required=None, extra=None): """Create a new `Schema` by merging this and the provided `schema`. @@ -700,7 +795,7 @@ def _compile_scalar(schema): >>> with raises(er.Invalid, 'not a valid value'): ... _compile_scalar(lambda v: float(v))([], 'a') """ - if isinstance(schema, type): + if inspect.isclass(schema): def validate_instance(path, data): if isinstance(data, schema): return data @@ -803,7 +898,6 @@ def _iterate_object(obj): for key in slots: if key != '__dict__': yield (key, getattr(obj, key)) - raise StopIteration() class Msg(object): @@ -879,10 +973,11 @@ class VirtualPathComponent(str): class Marker(object): """Mark nodes for special treatment.""" - def __init__(self, schema_, msg=None): + def __init__(self, schema_, msg=None, description=None): self.schema = schema_ self._schema = Schema(schema_) self.msg = msg + self.description = description def __call__(self, v): try: @@ -899,7 +994,9 @@ class Marker(object): return repr(self.schema) def __lt__(self, other): - return self.schema < other.schema + if isinstance(other, Marker): + return self.schema < other.schema + return self.schema < other def __hash__(self): return hash(self.schema) @@ -934,8 +1031,9 @@ class Optional(Marker): {'key2': 'value'} """ - def __init__(self, schema, msg=None, default=UNDEFINED): - super(Optional, self).__init__(schema, msg=msg) + def __init__(self, schema, msg=None, default=UNDEFINED, description=None): + super(Optional, self).__init__(schema, msg=msg, + description=description) self.default = default_factory(default) @@ -975,8 +1073,9 @@ class Exclusive(Optional): ... 'social': {'social_network': 'barfoo', 'token': 'tEMp'}}) """ - def __init__(self, schema, group_of_exclusion, msg=None): - super(Exclusive, self).__init__(schema, msg=msg) + def __init__(self, schema, group_of_exclusion, msg=None, description=None): + super(Exclusive, self).__init__(schema, msg=msg, + description=description) self.group_of_exclusion = group_of_exclusion @@ -1042,8 +1141,9 @@ class Required(Marker): {'key': []} """ - def __init__(self, schema, msg=None, default=UNDEFINED): - super(Required, self).__init__(schema, msg=msg) + def __init__(self, schema, msg=None, default=UNDEFINED, description=None): + super(Required, self).__init__(schema, msg=msg, + description=description) self.default = default_factory(default) @@ -1072,6 +1172,7 @@ class Remove(Marker): def __hash__(self): return object.__hash__(self) + def message(default=None, cls=None): """Convenience decorator to allow functions to provide a message. @@ -1174,7 +1275,8 @@ def validate(*a, **kw): returns = schema_arguments[RETURNS_KEY] del schema_arguments[RETURNS_KEY] - input_schema = Schema(schema_arguments) if len(schema_arguments) != 0 else lambda x: x + input_schema = (Schema(schema_arguments, extra=ALLOW_EXTRA) + if len(schema_arguments) != 0 else lambda x: x) output_schema = Schema(returns) if returns_defined else lambda x: x @wraps(func) diff --git a/third_party/python/voluptuous/voluptuous/tests/tests.md b/third_party/python/voluptuous/voluptuous/tests/tests.md index 18f6fbafa7e2..5ba97ab64b54 100644 --- a/third_party/python/voluptuous/voluptuous/tests/tests.md +++ b/third_party/python/voluptuous/voluptuous/tests/tests.md @@ -266,3 +266,8 @@ Ensure that subclasses of Invalid of are raised as is. ... exc = e >>> exc.errors[0].__class__.__name__ 'SpecialInvalid' + +Ensure that Optional('Classification') < 'Name' will return True instead of throwing an AttributeError + + >>> Optional('Classification') < 'Name' + True diff --git a/third_party/python/voluptuous/voluptuous/tests/tests.py b/third_party/python/voluptuous/voluptuous/tests/tests.py index 045ad122a104..fa44fbf770af 100644 --- a/third_party/python/voluptuous/voluptuous/tests/tests.py +++ b/third_party/python/voluptuous/voluptuous/tests/tests.py @@ -1,14 +1,17 @@ import copy import collections +import os import sys -from nose.tools import assert_equal, assert_raises, assert_true + +from nose.tools import assert_equal, assert_false, assert_raises, assert_true from voluptuous import ( - Schema, Required, Optional, Extra, Invalid, In, Remove, Literal, - Url, MultipleInvalid, LiteralInvalid, NotIn, Match, Email, + Schema, Required, Exclusive, Optional, Extra, Invalid, In, Remove, Literal, + Url, MultipleInvalid, LiteralInvalid, TypeInvalid, NotIn, Match, Email, Replace, Range, Coerce, All, Any, Length, FqdnUrl, ALLOW_EXTRA, PREVENT_EXTRA, validate, ExactSequence, Equal, Unordered, Number, Maybe, Datetime, Date, - Contains, Marker) + Contains, Marker, IsDir, IsFile, PathExists, SomeOf, TooManyValid, Self, + raises) from voluptuous.humanize import humanize_error from voluptuous.util import u @@ -153,6 +156,39 @@ def test_literal(): assert False, "Did not raise Invalid" +def test_class(): + class C1(object): + pass + + schema = Schema(C1) + schema(C1()) + + try: + schema(None) + except MultipleInvalid as e: + assert_equal(str(e), "expected C1") + assert_equal(len(e.errors), 1) + assert_equal(type(e.errors[0]), TypeInvalid) + else: + assert False, "Did not raise Invalid" + + # In Python 2, this will be an old-style class (classobj instance) + class C2: + pass + + schema = Schema(C2) + schema(C2()) + + try: + schema(None) + except MultipleInvalid as e: + assert_equal(str(e), "expected C2") + assert_equal(len(e.errors), 1) + assert_equal(type(e.errors[0]), TypeInvalid) + else: + assert False, "Did not raise Invalid" + + def test_email_validation(): """ test with valid email """ schema = Schema({"email": Email()}) @@ -375,6 +411,69 @@ def test_subschema_extension(): assert_equal(extended.schema, {'a': {'b': str, 'c': float, 'e': int}, 'd': str}) +def test_equality(): + assert_equal(Schema('foo'), Schema('foo')) + + assert_equal(Schema(['foo', 'bar', 'baz']), + Schema(['foo', 'bar', 'baz'])) + + # Ensure two Schemas w/ two equivalent dicts initialized in a different + # order are considered equal. + dict_a = {} + dict_a['foo'] = 1 + dict_a['bar'] = 2 + dict_a['baz'] = 3 + + dict_b = {} + dict_b['baz'] = 3 + dict_b['bar'] = 2 + dict_b['foo'] = 1 + + assert_equal(Schema(dict_a), Schema(dict_b)) + + +def test_equality_negative(): + """Verify that Schema objects are not equal to string representations""" + assert_false(Schema('foo') == 'foo') + + assert_false(Schema(['foo', 'bar']) == "['foo', 'bar']") + assert_false(Schema(['foo', 'bar']) == Schema("['foo', 'bar']")) + + assert_false(Schema({'foo': 1, 'bar': 2}) == "{'foo': 1, 'bar': 2}") + assert_false(Schema({'foo': 1, 'bar': 2}) == Schema("{'foo': 1, 'bar': 2}")) + + +def test_inequality(): + assert_true(Schema('foo') != 'foo') + + assert_true(Schema(['foo', 'bar']) != "['foo', 'bar']") + assert_true(Schema(['foo', 'bar']) != Schema("['foo', 'bar']")) + + assert_true(Schema({'foo': 1, 'bar': 2}) != "{'foo': 1, 'bar': 2}") + assert_true(Schema({'foo': 1, 'bar': 2}) != Schema("{'foo': 1, 'bar': 2}")) + + +def test_inequality_negative(): + assert_false(Schema('foo') != Schema('foo')) + + assert_false(Schema(['foo', 'bar', 'baz']) != + Schema(['foo', 'bar', 'baz'])) + + # Ensure two Schemas w/ two equivalent dicts initialized in a different + # order are considered equal. + dict_a = {} + dict_a['foo'] = 1 + dict_a['bar'] = 2 + dict_a['baz'] = 3 + + dict_b = {} + dict_b['baz'] = 3 + dict_b['bar'] = 2 + dict_b['foo'] = 1 + + assert_false(Schema(dict_a) != Schema(dict_b)) + + def test_repr(): """Verify that __repr__ returns valid Python expressions""" match = Match('a pattern', msg='message') @@ -393,7 +492,7 @@ def test_repr(): ) assert_equal(repr(coerce_), "Coerce(int, msg='moo')") assert_equal(repr(all_), "All('10', Coerce(int, msg=None), msg='all msg')") - assert_equal(repr(maybe_int), "Maybe(%s)" % str(int)) + assert_equal(repr(maybe_int), "Any(None, %s, msg=None)" % str(int)) def test_list_validation_messages(): @@ -514,14 +613,16 @@ def test_unordered(): def test_maybe(): - assert_raises(TypeError, Maybe, lambda x: x) - s = Schema(Maybe(int)) assert s(1) == 1 assert s(None) is None - assert_raises(Invalid, s, 'foo') + s = Schema(Maybe({str: Coerce(int)})) + assert s({'foo': '100'}) == {'foo': 100} + assert s(None) is None + assert_raises(Invalid, s, {'foo': 'bar'}) + def test_empty_list_as_exact(): s = Schema([]) @@ -529,40 +630,6 @@ def test_empty_list_as_exact(): s([]) -def test_empty_dict_as_exact(): - # {} always evaluates as {} - s = Schema({}) - assert_raises(Invalid, s, {'extra': 1}) - s = Schema({}, extra=ALLOW_EXTRA) # this should not be used - assert_raises(Invalid, s, {'extra': 1}) - - # {...} evaluates as Schema({...}) - s = Schema({'foo': int}) - assert_raises(Invalid, s, {'foo': 1, 'extra': 1}) - s = Schema({'foo': int}, extra=ALLOW_EXTRA) - s({'foo': 1, 'extra': 1}) - - # dict matches {} or {...} - s = Schema(dict) - s({'extra': 1}) - s({}) - s = Schema(dict, extra=PREVENT_EXTRA) - s({'extra': 1}) - s({}) - - # nested {} evaluate as {} - s = Schema({ - 'inner': {} - }, extra=ALLOW_EXTRA) - assert_raises(Invalid, s, {'inner': {'extra': 1}}) - s({}) - s = Schema({ - 'inner': Schema({}, extra=ALLOW_EXTRA) - }) - assert_raises(Invalid, s, {'inner': {'extra': 1}}) - s({}) - - def test_schema_decorator_match_with_args(): @validate(int) def fn(arg): @@ -643,6 +710,38 @@ def test_schema_decorator_return_only_unmatch(): assert_raises(Invalid, fn, 1) +def test_schema_decorator_partial_match_called_with_args(): + @validate(arg1=int) + def fn(arg1, arg2): + return arg1 + + fn(1, "foo") + + +def test_schema_decorator_partial_unmatch_called_with_args(): + @validate(arg1=int) + def fn(arg1, arg2): + return arg1 + + assert_raises(Invalid, fn, "bar", "foo") + + +def test_schema_decorator_partial_match_called_with_kwargs(): + @validate(arg2=int) + def fn(arg1, arg2): + return arg1 + + fn(arg1="foo", arg2=1) + + +def test_schema_decorator_partial_unmatch_called_with_kwargs(): + @validate(arg2=int) + def fn(arg1, arg2): + return arg1 + + assert_raises(Invalid, fn, arg1=1, arg2="foo") + + def test_unicode_as_key(): if sys.version_info >= (3,): text_type = str @@ -762,10 +861,15 @@ def test_datetime(): def test_date(): schema = Schema({"date": Date()}) schema({"date": "2016-10-24"}) - assert_raises(MultipleInvalid, schema, {"date": "2016-10-2"}) assert_raises(MultipleInvalid, schema, {"date": "2016-10-24Z"}) +def test_date_custom_format(): + schema = Schema({"date": Date("%Y%m%d")}) + schema({"date": "20161024"}) + assert_raises(MultipleInvalid, schema, {"date": "2016-10-24"}) + + def test_ordered_dict(): if not hasattr(collections, 'OrderedDict'): # collections.OrderedDict was added in Python2.7; only run if present @@ -793,6 +897,79 @@ def test_marker_hashable(): assert_equal(definition.get('j'), None) +def test_schema_infer(): + schema = Schema.infer({ + 'str': 'foo', + 'bool': True, + 'int': 42, + 'float': 3.14 + }) + assert_equal(schema, Schema({ + Required('str'): str, + Required('bool'): bool, + Required('int'): int, + Required('float'): float + })) + + +def test_schema_infer_dict(): + schema = Schema.infer({ + 'a': { + 'b': { + 'c': 'foo' + } + } + }) + + assert_equal(schema, Schema({ + Required('a'): { + Required('b'): { + Required('c'): str + } + } + })) + + +def test_schema_infer_list(): + schema = Schema.infer({ + 'list': ['foo', True, 42, 3.14] + }) + + assert_equal(schema, Schema({ + Required('list'): [str, bool, int, float] + })) + + +def test_schema_infer_scalar(): + assert_equal(Schema.infer('foo'), Schema(str)) + assert_equal(Schema.infer(True), Schema(bool)) + assert_equal(Schema.infer(42), Schema(int)) + assert_equal(Schema.infer(3.14), Schema(float)) + assert_equal(Schema.infer({}), Schema(dict)) + assert_equal(Schema.infer([]), Schema(list)) + + +def test_schema_infer_accepts_kwargs(): + schema = Schema.infer({ + 'str': 'foo', + 'bool': True + }, required=False, extra=True) + + # Subset of schema should be acceptable thanks to required=False. + schema({'bool': False}) + + # Keys that are in schema should still match required types. + try: + schema({'str': 42}) + except Invalid: + pass + else: + assert False, 'Did not raise Invalid for Number' + + # Extra fields should be acceptable thanks to extra=True. + schema({'str': 'bar', 'int': 42}) + + def test_validation_performance(): """ This test comes to make sure the validation complexity of dictionaries is done in a linear time. @@ -816,7 +993,7 @@ def test_validation_performance(): for i in range(num_of_keys): schema_dict[CounterMarker(str(i))] = str data[str(i)] = str(i) - data_extra_keys[str(i*2)] = str(i) # half of the keys are present, and half aren't + data_extra_keys[str(i * 2)] = str(i) # half of the keys are present, and half aren't schema = Schema(schema_dict, extra=ALLOW_EXTRA) @@ -828,3 +1005,261 @@ def test_validation_performance(): schema(data_extra_keys) assert counter[0] <= num_of_keys, "Validation complexity is not linear! %s > %s" % (counter[0], num_of_keys) + + +def test_IsDir(): + schema = Schema(IsDir()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.dirname(os.path.abspath(__file__))) + + +def test_IsFile(): + schema = Schema(IsFile()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.abspath(__file__)) + + +def test_PathExists(): + schema = Schema(PathExists()) + assert_raises(MultipleInvalid, schema, 3) + schema(os.path.abspath(__file__)) + + +def test_description(): + marker = Marker(Schema(str), description='Hello') + assert marker.description == 'Hello' + + optional = Optional('key', description='Hello') + assert optional.description == 'Hello' + + exclusive = Exclusive('alpha', 'angles', description='Hello') + assert exclusive.description == 'Hello' + + required = Required('key', description='Hello') + assert required.description == 'Hello' + + +def test_SomeOf_min_validation(): + validator = All(Length(min=8), SomeOf( + min_valid=3, + validators=[Match(r'.*[A-Z]', 'no uppercase letters'), + Match(r'.*[a-z]', 'no lowercase letters'), + Match(r'.*[0-9]', 'no numbers'), + Match(r'.*[$@$!%*#?&^:;/<,>|{}()\-\'._+=]', 'no symbols')])) + + validator('ffe532A1!') + with raises(MultipleInvalid, 'length of value must be at least 8'): + validator('a') + + with raises(MultipleInvalid, 'no uppercase letters, no lowercase letters'): + validator('wqs2!#s111') + + with raises(MultipleInvalid, 'no lowercase letters, no symbols'): + validator('3A34SDEF5') + + +def test_SomeOf_max_validation(): + validator = SomeOf( + max_valid=2, + validators=[Match(r'.*[A-Z]', 'no uppercase letters'), + Match(r'.*[a-z]', 'no lowercase letters'), + Match(r'.*[0-9]', 'no numbers')], + msg='max validation test failed') + + validator('Aa') + with raises(TooManyValid, 'max validation test failed'): + validator('Aa1') + + +def test_self_validation(): + schema = Schema({"number": int, + "follow": Self}) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + + +def test_any_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Optional('q'): int, + Required('q2'): Any(int, msg='toto') + }) + try: + s({'q': 'str', 'q2': 'tata'}) + except MultipleInvalid as exc: + assert ( + (exc.errors[0].path == ['q'] and exc.errors[1].path == ['q2']) or + (exc.errors[1].path == ['q'] and exc.errors[0].path == ['q2']) + ) + else: + assert False, "Did not raise AnyInvalid" + + +def test_all_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Optional('q'): int, + Required('q2'): All([str, Length(min=10)], msg='toto'), + }) + try: + s({'q': 'str', 'q2': 12}) + except MultipleInvalid as exc: + assert ( + (exc.errors[0].path == ['q'] and exc.errors[1].path == ['q2']) or + (exc.errors[1].path == ['q'] and exc.errors[0].path == ['q2']) + ) + else: + assert False, "Did not raise AllInvalid" + + +def test_match_error_has_path(): + """https://github.com/alecthomas/voluptuous/issues/347""" + s = Schema({ + Required('q2'): Match("a"), + }) + try: + s({'q2': 12}) + except MultipleInvalid as exc: + assert exc.errors[0].path == ['q2'] + else: + assert False, "Did not raise MatchInvalid" + + +def test_self_any(): + schema = Schema({"number": int, + "follow": Any(Self, "stop")}) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + schema({"follow": {"follow": {"number": 123456, "follow": "stop"}}}) + + +def test_self_all(): + schema = Schema({"number": int, + "follow": All(Self, + Schema({"extra_number": int}, + extra=ALLOW_EXTRA))}, + extra=ALLOW_EXTRA) + try: + schema({"number": "abc"}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + try: + schema({"follow": {"number": '123456.712'}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + schema({"follow": {"number": 123456}}) + schema({"follow": {"follow": {"number": 123456}}}) + schema({"follow": {"number": 123456, "extra_number": 123}}) + try: + schema({"follow": {"number": 123456, "extra_number": "123"}}) + except MultipleInvalid: + pass + else: + assert False, "Did not raise Invalid" + + +def test_SomeOf_on_bounds_assertion(): + with raises(AssertionError, 'when using "SomeOf" you should specify at least one of min_valid and max_valid'): + SomeOf(validators=[]) + + +def test_comparing_voluptuous_object_to_str(): + assert_true(Optional('Classification') < 'Name') + + +def test_set_of_integers(): + schema = Schema({int}) + with raises(Invalid, 'expected a set'): + schema(42) + with raises(Invalid, 'expected a set'): + schema(frozenset([42])) + + schema(set()) + schema(set([42])) + schema(set([42, 43, 44])) + try: + schema(set(['abc'])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in set") + else: + assert False, "Did not raise Invalid" + + +def test_frozenset_of_integers(): + schema = Schema(frozenset([int])) + with raises(Invalid, 'expected a frozenset'): + schema(42) + with raises(Invalid, 'expected a frozenset'): + schema(set([42])) + + schema(frozenset()) + schema(frozenset([42])) + schema(frozenset([42, 43, 44])) + try: + schema(frozenset(['abc'])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in frozenset") + else: + assert False, "Did not raise Invalid" + + +def test_set_of_integers_and_strings(): + schema = Schema({int, str}) + with raises(Invalid, 'expected a set'): + schema(42) + + schema(set()) + schema(set([42])) + schema(set(['abc'])) + schema(set([42, 'abc'])) + try: + schema(set([None])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in set") + else: + assert False, "Did not raise Invalid" + + +def test_frozenset_of_integers_and_strings(): + schema = Schema(frozenset([int, str])) + with raises(Invalid, 'expected a frozenset'): + schema(42) + + schema(frozenset()) + schema(frozenset([42])) + schema(frozenset(['abc'])) + schema(frozenset([42, 'abc'])) + try: + schema(frozenset([None])) + except MultipleInvalid as e: + assert_equal(str(e), "invalid value in frozenset") + else: + assert False, "Did not raise Invalid" diff --git a/third_party/python/voluptuous/voluptuous/util.py b/third_party/python/voluptuous/voluptuous/util.py index 3a09297f2a7e..434c360c7e95 100644 --- a/third_party/python/voluptuous/voluptuous/util.py +++ b/third_party/python/voluptuous/voluptuous/util.py @@ -1,13 +1,8 @@ import sys -try: - from error import LiteralInvalid, TypeInvalid, Invalid - from schema_builder import Schema, default_factory, raises - import validators -except ImportError: - from .error import LiteralInvalid, TypeInvalid, Invalid - from .schema_builder import Schema, default_factory, raises - from . import validators +from voluptuous.error import LiteralInvalid, TypeInvalid, Invalid +from voluptuous.schema_builder import Schema, default_factory, raises +from voluptuous import validators __author__ = 'tusharmakkar08' diff --git a/third_party/python/voluptuous/voluptuous/validators.py b/third_party/python/voluptuous/voluptuous/validators.py index 2318dd6e1143..d5e3ed598051 100644 --- a/third_party/python/voluptuous/voluptuous/validators.py +++ b/third_party/python/voluptuous/voluptuous/validators.py @@ -5,22 +5,16 @@ import sys from functools import wraps from decimal import Decimal, InvalidOperation -try: - from schema_builder import Schema, raises, message - from error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, AnyInvalid, - AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, RangeInvalid, - PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, DateInvalid, InInvalid, - TypeInvalid, NotInInvalid, ContainsInvalid) -except ImportError: - from .schema_builder import Schema, raises, message - from .error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, AnyInvalid, - AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, RangeInvalid, - PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, DateInvalid, InInvalid, - TypeInvalid, NotInInvalid, ContainsInvalid) - +from voluptuous.schema_builder import Schema, raises, message +from voluptuous.error import (MultipleInvalid, CoerceInvalid, TrueInvalid, FalseInvalid, BooleanInvalid, Invalid, + AnyInvalid, AllInvalid, MatchInvalid, UrlInvalid, EmailInvalid, FileInvalid, DirInvalid, + RangeInvalid, PathInvalid, ExactSequenceInvalid, LengthInvalid, DatetimeInvalid, + DateInvalid, InInvalid, TypeInvalid, NotInInvalid, ContainsInvalid, NotEnoughValid, + TooManyValid) if sys.version_info >= (3,): import urllib.parse as urlparse + basestring = str else: import urlparse @@ -99,7 +93,7 @@ class Coerce(object): def __call__(self, v): try: return self.type(v) - except (ValueError, TypeError): + except (ValueError, TypeError, InvalidOperation): msg = self.msg or ('expected %s' % self.type_name) raise CoerceInvalid(msg) @@ -187,7 +181,40 @@ def Boolean(v): return bool(v) -class Any(object): +class _WithSubValidators(object): + """Base class for validators that use sub-validators. + + Special class to use as a parent class for validators using sub-validators. + This class provides the `__voluptuous_compile__` method so the + sub-validators are compiled by the parent `Schema`. + """ + + def __init__(self, *validators, **kwargs): + self.validators = validators + self.msg = kwargs.pop('msg', None) + + def __voluptuous_compile__(self, schema): + self._compiled = [ + schema._compile(v) + for v in self.validators + ] + return self._run + + def _run(self, path, value): + return self._exec(self._compiled, value, path) + + def __call__(self, v): + return self._exec((Schema(val) for val in self.validators), v) + + def __repr__(self): + return '%s(%s, msg=%r)' % ( + self.__class__.__name__, + ", ".join(repr(v) for v in self.validators), + self.msg + ) + + +class Any(_WithSubValidators): """Use the first validated value. :param msg: Message to deliver to user if validation fails. @@ -212,33 +239,30 @@ class Any(object): ... validate(4) """ - def __init__(self, *validators, **kwargs): - self.validators = validators - self.msg = kwargs.pop('msg', None) - self._schemas = [Schema(val, **kwargs) for val in validators] - - def __call__(self, v): + def _exec(self, funcs, v, path=None): error = None - for schema in self._schemas: + for func in funcs: try: - return schema(v) + if path is None: + return func(v) + else: + return func(path, v) except Invalid as e: if error is None or len(e.path) > len(error.path): error = e else: if error: - raise error if self.msg is None else AnyInvalid(self.msg) - raise AnyInvalid(self.msg or 'no valid value found') - - def __repr__(self): - return 'Any([%s])' % (", ".join(repr(v) for v in self.validators)) + raise error if self.msg is None else AnyInvalid( + self.msg, path=path) + raise AnyInvalid(self.msg or 'no valid value found', + path=path) # Convenience alias Or = Any -class All(object): +class All(_WithSubValidators): """Value must pass all validators. The output of each validator is passed as input to the next. @@ -251,25 +275,17 @@ class All(object): 10 """ - def __init__(self, *validators, **kwargs): - self.validators = validators - self.msg = kwargs.pop('msg', None) - self._schemas = [Schema(val, **kwargs) for val in validators] - - def __call__(self, v): + def _exec(self, funcs, v, path=None): try: - for schema in self._schemas: - v = schema(v) + for func in funcs: + if path is None: + v = func(v) + else: + v = func(path, v) except Invalid as e: - raise e if self.msg is None else AllInvalid(self.msg) + raise e if self.msg is None else AllInvalid(self.msg, path=path) return v - def __repr__(self): - return 'All(%s, msg=%r)' % ( - ", ".join(repr(v) for v in self.validators), - self.msg - ) - # Convenience alias And = All @@ -419,9 +435,13 @@ def IsFile(v): >>> with raises(FileInvalid, 'Not a file'): ... IsFile()(None) """ - if v: - return os.path.isfile(v) - else: + try: + if v: + v = str(v) + return os.path.isfile(v) + else: + raise FileInvalid('Not a file') + except TypeError: raise FileInvalid('Not a file') @@ -435,9 +455,13 @@ def IsDir(v): >>> with raises(DirInvalid, 'Not a directory'): ... IsDir()(None) """ - if v: - return os.path.isdir(v) - else: + try: + if v: + v = str(v) + return os.path.isdir(v) + else: + raise DirInvalid("Not a directory") + except TypeError: raise DirInvalid("Not a directory") @@ -453,16 +477,21 @@ def PathExists(v): >>> with raises(PathInvalid, 'Not a Path'): ... PathExists()(None) """ - if v: - return os.path.exists(v) - else: + try: + if v: + v = str(v) + return os.path.exists(v) + else: + raise PathInvalid("Not a Path") + except TypeError: raise PathInvalid("Not a Path") -class Maybe(object): - """Validate that the object is of a given type or is None. +def Maybe(validator): + """Validate that the object matches given validator or is None. - :raises Invalid: if the value is not of the type declared and is not None + :raises Invalid: if the value does not match the given validator and is not + None >>> s = Schema(Maybe(int)) >>> s(10) @@ -471,21 +500,7 @@ class Maybe(object): ... s("string") """ - def __init__(self, kind, msg=None): - if not isinstance(kind, type): - raise TypeError("kind has to be a type") - - self.kind = kind - self.msg = msg - - def __call__(self, v): - if v is not None and not isinstance(v, self.kind): - raise Invalid(self.msg or "%s must be None or of type %s" % (v, self.kind)) - - return v - - def __repr__(self): - return 'Maybe(%s)' % str(self.kind) + return Any(None, validator) class Range(object): @@ -621,15 +636,10 @@ class Date(Datetime): """Validate that the value matches the date format.""" DEFAULT_FORMAT = '%Y-%m-%d' - FORMAT_DESCRIPTION = 'yyyy-mm-dd' def __call__(self, v): try: datetime.datetime.strptime(v, self.format) - if len(v) != len(self.FORMAT_DESCRIPTION): - raise DateInvalid( - self.msg or 'value has invalid length' - ' expected length %d (%s)' % (len(self.FORMAT_DESCRIPTION), self.FORMAT_DESCRIPTION)) except (TypeError, ValueError): raise DateInvalid( self.msg or 'value does not match' @@ -704,7 +714,7 @@ class Contains(object): return v def __repr__(self): - return 'Contains(%s)' % (self.item, ) + return 'Contains(%s)' % (self.item,) class ExactSequence(object): @@ -866,10 +876,8 @@ class Unordered(object): el = missing[0] raise Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format(el[0], el[1])) elif missing: - raise MultipleInvalid([ - Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format(el[0], el[1])) - for el in missing - ]) + raise MultipleInvalid([Invalid(self.msg or 'Element #{} ({}) is not valid against any validator'.format( + el[0], el[1])) for el in missing]) return v def __repr__(self): @@ -904,15 +912,16 @@ class Number(object): """ precision, scale, decimal_num = self._get_precision_scale(v) - if self.precision is not None and self.scale is not None and\ - precision != self.precision and scale != self.scale: - raise Invalid(self.msg or "Precision must be equal to %s, and Scale must be equal to %s" %(self.precision, self.scale)) + if self.precision is not None and self.scale is not None and precision != self.precision\ + and scale != self.scale: + raise Invalid(self.msg or "Precision must be equal to %s, and Scale must be equal to %s" % (self.precision, + self.scale)) else: if self.precision is not None and precision != self.precision: - raise Invalid(self.msg or "Precision must be equal to %s"%self.precision) + raise Invalid(self.msg or "Precision must be equal to %s" % self.precision) - if self.scale is not None and scale != self.scale : - raise Invalid(self.msg or "Scale must be equal to %s"%self.scale) + if self.scale is not None and scale != self.scale: + raise Invalid(self.msg or "Scale must be equal to %s" % self.scale) if self.yield_decimal: return decimal_num @@ -933,3 +942,63 @@ class Number(object): raise Invalid(self.msg or 'Value must be a number enclosed with string') return (len(decimal_num.as_tuple().digits), -(decimal_num.as_tuple().exponent), decimal_num) + + +class SomeOf(_WithSubValidators): + """Value must pass at least some validations, determined by the given parameter. + Optionally, number of passed validations can be capped. + + The output of each validator is passed as input to the next. + + :param min_valid: Minimum number of valid schemas. + :param validators: a list of schemas or validators to match input against + :param max_valid: Maximum number of valid schemas. + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-Schema constructors. + + :raises NotEnoughValid: if the minimum number of validations isn't met + :raises TooManyValid: if the more validations than the given amount is met + + >>> validate = Schema(SomeOf(min_valid=2, validators=[Range(1, 5), Any(float, int), 6.6])) + >>> validate(6.6) + 6.6 + >>> validate(3) + 3 + >>> with raises(MultipleInvalid, 'value must be at most 5, not a valid value'): + ... validate(6.2) + """ + + def __init__(self, validators, min_valid=None, max_valid=None, **kwargs): + assert min_valid is not None or max_valid is not None, \ + 'when using "%s" you should specify at least one of min_valid and max_valid' % (type(self).__name__,) + self.min_valid = min_valid or 0 + self.max_valid = max_valid or len(validators) + super(SomeOf, self).__init__(*validators, **kwargs) + + def _exec(self, funcs, v, path=None): + errors = [] + funcs = list(funcs) + for func in funcs: + try: + if path is None: + v = func(v) + else: + v = func(path, v) + except Invalid as e: + errors.append(e) + + passed_count = len(funcs) - len(errors) + if self.min_valid <= passed_count <= self.max_valid: + return v + + msg = self.msg + if not msg: + msg = ', '.join(map(str, errors)) + + if passed_count > self.max_valid: + raise TooManyValid(msg) + raise NotEnoughValid(msg) + + def __repr__(self): + return 'SomeOf(min_valid=%s, validators=[%s], max_valid=%s, msg=%r)' % ( + self.min_valid, ", ".join(repr(v) for v in self.validators), self.max_valid, self.msg)