1
from datetime import datetime
2
from operator import itemgetter
4
from dateutil.tz import tzutc
5
from dateutil.parser import parse
7
from txaws.server.exception import APIError
10
class SchemaError(APIError):
11
"""Raised when failing to extract or bundle L{Parameter}s."""
13
def __init__(self, message):
14
code = self.__class__.__name__[:-len("Error")]
15
super(SchemaError, self).__init__(400, code=code, message=message)
18
class MissingParameterError(SchemaError):
19
"""Raised when a parameter is missing.
21
@param name: The name of the missing parameter.
24
def __init__(self, name, kind=None):
25
message = "The request must contain the parameter %s" % name
27
message += " (%s)" % (kind,)
28
super(MissingParameterError, self).__init__(message)
31
class InconsistentParameterError(SchemaError):
32
def __init__(self, name):
33
message = "Parameter %s is used inconsistently" % (name,)
34
super(InconsistentParameterError, self).__init__(message)
37
class InvalidParameterValueError(SchemaError):
38
"""Raised when the value of a parameter is invalid."""
41
class InvalidParameterCombinationError(SchemaError):
43
Raised when there is more than one parameter with the same name,
44
when this isn't explicitly allowed for.
46
@param name: The name of the missing parameter.
49
def __init__(self, name):
50
message = "The parameter '%s' may only be specified once." % name
51
super(InvalidParameterCombinationError, self).__init__(message)
54
class UnknownParameterError(SchemaError):
55
"""Raised when a parameter to extract is unknown."""
57
def __init__(self, name):
58
message = "The parameter %s is not recognized" % name
59
super(UnknownParameterError, self).__init__(message)
62
class UnknownParametersError(Exception):
64
Raised when extra unknown fields are passed to L{Structure.parse}.
66
@ivar result: The already coerced result representing the known parameters.
67
@ivar unknown: The unknown parameters.
69
def __init__(self, result, unknown):
71
self.unknown = unknown
72
message = "The parameters %s are not recognized" % (unknown,)
73
super(UnknownParametersError, self).__init__(message)
76
class Parameter(object):
77
"""A single parameter in an HTTP request.
79
@param name: A name for the key of the parameter, as specified
80
in a request. For example, a single parameter would be specified
81
simply as 'GroupName'. If more than one group name was accepted,
82
it would be specified as 'GroupName.n'. A more complex example
83
is 'IpPermissions.n.Groups.m.GroupName'.
84
@param optional: If C{True} the parameter may not be present.
85
@param default: A default value for the parameter, if not present.
86
@param min: Minimum value for a parameter.
87
@param max: Maximum value for a parameter.
88
@param allow_none: Whether the parameter may be C{None}.
89
@param validator: A callable to validate the parameter, returning a bool.
92
supports_multiple = False
95
def __init__(self, name=None, optional=False, default=None,
96
min=None, max=None, allow_none=False, validator=None,
99
self.optional = optional
100
self.default = default
103
self.allow_none = allow_none
104
self.validator = validator
107
def coerce(self, value):
108
"""Coerce a single value according to this parameter's settings.
110
@param value: A L{str}, or L{None}. If L{None} is passed - meaning no
111
value is avalable at all, not even the empty string - and this
112
parameter is optional, L{self.default} will be returned.
120
if not self.allow_none:
121
raise MissingParameterError(self.name, kind=self.kind)
124
self._check_range(value)
125
parsed = self.parse(value)
126
if self.validator and not self.validator(parsed):
127
raise ValueError(value)
131
value = value.decode("utf-8")
132
message = "Invalid %s value %s" % (self.kind, value)
133
except UnicodeDecodeError:
134
message = "Invalid %s value" % self.kind
135
raise InvalidParameterValueError(message)
137
def _check_range(self, value):
138
"""Check that the given C{value} is in the expected range."""
139
if self.min is None and self.max is None:
142
measure = self.measure(value)
143
prefix = "Value (%s) for parameter %s is invalid. %s"
145
if self.min is not None and measure < self.min:
146
message = prefix % (value, self.name,
147
self.lower_than_min_template % self.min)
148
raise InvalidParameterValueError(message)
150
if self.max is not None and measure > self.max:
151
message = prefix % (value, self.name,
152
self.greater_than_max_template % self.max)
153
raise InvalidParameterValueError(message)
155
def parse(self, value):
157
Parse a single parameter value coverting it to the appropriate type.
159
raise NotImplementedError()
161
def format(self, value):
163
Format a single parameter value in a way suitable for an HTTP request.
165
raise NotImplementedError()
167
def measure(self, value):
169
Return an C{int} providing a measure for C{value}, used for C{range}.
171
raise NotImplementedError()
174
class Unicode(Parameter):
175
"""A parameter that must be a C{unicode}."""
179
lower_than_min_template = "Length must be at least %s."
180
greater_than_max_template = "Length exceeds maximum of %s."
182
def parse(self, value):
183
return value.decode("utf-8")
185
def format(self, value):
186
return value.encode("utf-8")
188
def measure(self, value):
192
class UnicodeLine(Unicode):
193
"""A parameter that must be a C{unicode} string without newlines."""
195
def coerce(self, value):
196
super(UnicodeLine, self).coerce(value)
198
raise InvalidParameterValueError("Can't contain newlines.")
201
class RawStr(Parameter):
202
"""A parameter that must be a C{str}."""
206
def parse(self, value):
209
def format(self, value):
213
class Integer(Parameter):
214
"""A parameter that must be a positive C{int}."""
218
lower_than_min_template = "Value must be at least %s."
219
greater_than_max_template = "Value exceeds maximum of %s."
221
def __init__(self, name=None, optional=False, default=None,
222
min=0, max=None, allow_none=False, validator=None,
224
super(Integer, self).__init__(name, optional, default, min, max,
225
allow_none, validator, doc=doc)
227
def parse(self, value):
230
def format(self, value):
233
def measure(self, value):
237
class Bool(Parameter):
238
"""A parameter that must be a C{bool}."""
242
def parse(self, value):
249
def format(self, value):
256
class Enum(Parameter):
257
"""A parameter with enumerated values.
259
@param name: The name of the parameter, as specified in a request.
260
@param optional: If C{True} the parameter may not be present.
261
@param default: A default value for the parameter, if not present.
262
@param mapping: A mapping of accepted values to the values that
263
will be returned by C{parse}.
268
def __init__(self, name=None, mapping=None, optional=False, default=None,
270
super(Enum, self).__init__(name, optional=optional, default=default,
273
raise TypeError("Must provide mapping")
274
self.mapping = mapping
275
self.reverse = dict((value, key) for key, value in mapping.iteritems())
277
def parse(self, value):
279
return self.mapping[value]
283
def format(self, value):
284
return self.reverse[value]
287
class Date(Parameter):
288
"""A parameter that must be a valid ISO 8601 formatted date."""
292
def parse(self, value):
293
return parse(value).replace(tzinfo=tzutc())
295
def format(self, value):
296
# Convert value to UTC.
297
tt = value.utctimetuple()
298
utc_value = datetime(
299
tt.tm_year, tt.tm_mon, tt.tm_mday, tt.tm_hour, tt.tm_min,
301
return datetime.strftime(utc_value, "%Y-%m-%dT%H:%M:%SZ")
304
class List(Parameter):
306
A homogenous list of instances of a parameterized type.
308
There is a strange behavior that lists can have any starting index and any
309
gaps are ignored. Conventionally they are 1-based, and so indexes proceed
310
like 1, 2, 3... However, any non-negative index can be used and the
311
ordering will be used to determine the true index. So::
313
{5: 'a', 7: 'b', 9: 'c'}
321
supports_multiple = True
323
def __init__(self, name=None, item=None, optional=False, default=None,
326
@param item: A L{Parameter} instance which will be used to parse and
327
format the values in the list.
330
raise TypeError("Must provide item")
331
super(List, self).__init__(name, optional=optional, default=default,
333
if item.name is None:
339
def parse(self, value):
341
Convert a dictionary of {relative index: value} to a list of parsed
345
if not isinstance(value, dict):
346
# We interpret non-list inputs as a list of one element, for
347
# compatibility with certain EC2 APIs.
348
return [self.item.coerce(value)]
349
for index in value.keys():
351
indices.append(int(index))
353
raise UnknownParameterError(index)
354
result = [None] * len(value)
355
for index_index, index in enumerate(sorted(indices)):
356
v = value[str(index)]
358
raise UnknownParameterError(index)
359
result[index_index] = self.item.coerce(v)
362
def format(self, value):
364
Convert a list like::
370
{"1": "a", "2": "b", "3": "c"}
372
C{value} may also be an L{Arguments} instance, mapping indices to
373
values. Who knows why.
375
if isinstance(value, Arguments):
376
return dict((str(i), self.item.format(v)) for i, v in value)
377
return dict((str(i + 1), self.item.format(v))
378
for i, v in enumerate(value))
381
class Structure(Parameter):
383
A structure with named fields of parameterized types.
387
supports_multiple = True
389
def __init__(self, name=None, fields=None, optional=False, default=None,
392
@param fields: A mapping of field name to field L{Parameter} instance.
395
raise TypeError("Must provide fields")
396
super(Structure, self).__init__(name, optional=optional,
397
default=default, doc=doc)
398
_namify_arguments(fields)
401
def parse(self, value):
403
Convert a dictionary of raw values to a dictionary of processed values.
407
for k, v in value.iteritems():
409
if (isinstance(v, dict)
410
and not self.fields[k].supports_multiple):
412
# We support "foo.1" as "foo" as long as there is only
413
# one "foo.#" parameter provided.... -_-
416
raise InvalidParameterCombinationError(k)
417
result[k] = self.fields[k].coerce(v)
420
for k, v in self.fields.iteritems():
422
result[k] = v.coerce(None)
424
raise UnknownParametersError(result, rest)
427
def format(self, value):
429
Convert a dictionary of processed values to a dictionary of raw values.
431
if not isinstance(value, Arguments):
432
value = value.iteritems()
433
return dict((k, self.fields[k].format(v)) for k, v in value)
436
class Arguments(object):
437
"""Arguments parsed from a request."""
439
def __init__(self, tree):
440
"""Initialize a new L{Arguments} instance.
442
@param tree: The C{dict}-based structure of the L{Argument} instance
445
for key, value in tree.iteritems():
446
self.__dict__[key] = self._wrap(value)
449
return "Arguments(%s)" % (self.__dict__,)
454
"""Returns an iterator yielding C{(name, value)} tuples."""
455
return self.__dict__.iteritems()
457
def __getitem__(self, index):
458
"""Return the argument value with the given L{index}."""
459
return self.__dict__[index]
462
"""Return the number of arguments."""
463
return len(self.__dict__)
465
def __contains__(self, key):
466
"""Return whether an argument with the given name is present."""
467
return key in self.__dict__
469
def _wrap(self, value):
470
"""Wrap the given L{tree} with L{Arguments} as necessary.
472
@param tree: A {dict}, containing L{dict}s and/or leaf values, nested
475
if isinstance(value, dict):
476
if any(isinstance(name, int) for name in value.keys()):
477
if not all(isinstance(name, int) for name in value.keys()):
478
raise RuntimeError("Integer and non-integer keys: %r"
480
items = sorted(value.iteritems(), key=itemgetter(0))
481
return [self._wrap(val) for _, val in items]
483
return Arguments(value)
484
elif isinstance(value, list):
485
return [self._wrap(x) for x in value]
490
def _namify_arguments(mapping):
492
Ensure that a mapping of names to parameters has the parameters set to the
496
for name, parameter in mapping.iteritems():
497
parameter.name = name
498
result.append(parameter)
502
class Schema(object):
504
The schema that the arguments of an HTTP request must be compliant with.
507
def __init__(self, *_parameters, **kwargs):
508
"""Initialize a new L{Schema} instance.
510
Any number of L{Parameter} instances can be passed. The parameter names
511
are used in L{Schema.extract} and L{Schema.bundle}. For example::
513
schema = Schema(name="SetName", parameters=[Unicode("Name")])
515
means that the result of L{Schema.extract} would have a C{Name}
516
attribute. Similarly, L{Schema.bundle} would look for a C{Name}
519
A more complex example::
523
parameters=[List("Names", Unicode())])
525
means that the result of L{Schema.extract} would have a C{Names}
526
attribute, which would itself contain a list of names. Similarly,
527
L{Schema.bundle} would look for a C{Names} attribute.
529
Currently all parameters other than C{parameters} have no effect; they
530
are merely exposed as attributes of instances of Schema, and are able
531
to be overridden in L{extend}.
533
@param name: (keyword) The name of the API call that this schema
534
represents. Accessible via the C{name} attribute.
535
@param parameters: (keyword) The parameters of the API, as a list of
536
named L{Parameter} instances.
537
@param doc: (keyword) The documentation of this API Call. Accessible
538
via the C{doc} attribute.
539
@param result: (keyword) A description of the result of this API
540
call. Accessible via the C{result} attribute.
541
@param errors: (keyword) A sequence of exception classes that the API
542
can potentially raise. Accessible as a L{set} via the C{errors}
545
self.name = kwargs.pop('name', None)
546
self.doc = kwargs.pop('doc', None)
547
self.result = kwargs.pop('result', None)
548
self.errors = set(kwargs.pop('errors', []))
549
if 'parameters' in kwargs:
550
if len(_parameters) > 0:
551
raise TypeError("parameters= must only be passed "
552
"without positional arguments")
553
self._parameters = kwargs['parameters']
555
self._parameters = self._convert_old_schema(_parameters)
557
def get_parameters(self):
559
Get the list of parameters this schema supports.
561
return self._parameters[:]
563
def extract(self, params):
564
"""Extract parameters from a raw C{dict} according to this schema.
566
@param params: The raw parameters to parse.
567
@return: A tuple of an L{Arguments} object holding the extracted
568
arguments and any unparsed arguments.
570
structure = Structure(fields=dict([(p.name, p)
571
for p in self._parameters]))
573
tree = structure.coerce(self._convert_flat_to_nest(params))
575
except UnknownParametersError, error:
577
rest = self._convert_nest_to_flat(error.unknown)
578
return Arguments(tree), rest
580
def bundle(self, *arguments, **extra):
581
"""Bundle the given arguments in a C{dict} with EC2-style format.
583
@param arguments: L{Arguments} instances to bundle. Keys in
584
later objects will override those in earlier objects.
585
@param extra: Any number of additional parameters. These will override
586
similarly named arguments in L{arguments}.
590
for argument in arguments:
591
params.update(argument)
595
for name, value in params.iteritems():
598
segments = name.split('.')
600
parameter = self.get_parameter(first)
601
if parameter is None:
602
raise RuntimeError("Parameter '%s' not in schema" % name)
607
result[name] = parameter.format(value)
609
return self._convert_nest_to_flat(result)
611
def get_parameter(self, name):
613
Get the parameter on this schema with the given C{name}.
615
for parameter in self._parameters:
616
if parameter.name == name:
619
def _convert_flat_to_nest(self, params):
621
Convert a structure in the form of::
623
{'foo.1.bar': 'value',
624
'foo.2.baz': 'value'}
628
{'foo': {'1': {'bar': 'value'},
629
'2': {'baz': 'value'}}}
631
This is intended for use both during parsing of HTTP arguments like
632
'foo.1.bar=value' and when dealing with schema declarations that look
635
This is the inverse of L{_convert_nest_to_flat}.
638
for k, v in params.iteritems():
640
segments = k.split('.')
641
for index, item in enumerate(segments):
642
if index == len(segments) - 1:
646
if not isinstance(last, dict):
647
raise InconsistentParameterError(k)
648
if type(last.get(item)) is dict and type(newd) is not dict:
649
raise InconsistentParameterError(k)
650
last = last.setdefault(item, newd)
653
def _convert_nest_to_flat(self, params, _result=None, _prefix=None):
655
Convert a data structure that looks like::
657
{"foo": {"bar": "baz", "shimmy": "sham"}}
662
"foo.shimmy": "sham"}
664
This is the inverse of L{_convert_flat_to_nest}.
668
for k, v in params.iteritems():
672
path = _prefix + '.' + k
673
if isinstance(v, dict):
674
self._convert_nest_to_flat(v, _result=_result, _prefix=path)
679
def extend(self, *schema_items, **kwargs):
681
Add any number of schema items to a new schema.
683
Takes the same arguments as the constructor, and returns a new
686
If parameters, result, or errors is specified, they will be merged with
687
the existing parameters, result, or errors.
692
'parameters': self._parameters[:],
693
'result': self.result.copy() if self.result else {},
694
'errors': self.errors.copy() if self.errors else set()}
695
if 'parameters' in kwargs:
696
new_params = kwargs.pop('parameters')
697
new_kwargs['parameters'].extend(new_params)
698
new_kwargs['result'].update(kwargs.pop('result', {}))
699
new_kwargs['errors'].update(kwargs.pop('errors', set()))
700
new_kwargs.update(kwargs)
703
parameters = self._convert_old_schema(schema_items)
704
new_kwargs['parameters'].extend(parameters)
705
return Schema(**new_kwargs)
707
def _convert_old_schema(self, parameters):
709
Convert an ugly old schema, using dotted names, to the hot new schema,
710
using List and Structure.
712
The old schema assumes that every other dot implies an array. So a list
715
[Integer("foo.bar.baz.quux"), Integer("foo.bar.shimmy")]
722
fields={"baz": List(item=Integer()),
723
"shimmy": Integer()}))]
725
By design, the old schema syntax ignored the names "bar" and "quux".
727
# 'merged' here is an associative list that maps parameter names to
728
# Parameter instances, OR sub-associative lists which represent nested
729
# lists and structures.
733
# [("foo", Integer("foo"))]
735
# [Integer("foo.bar")]
736
# (which represents a list of integers called "foo" with a meaningless
737
# index name of "bar") becomes
738
# [("foo", [("bar", Integer("foo.bar"))])].
740
for parameter in parameters:
741
segments = parameter.name.split('.')
742
_merge_associative_list(merged, segments, parameter)
743
result = [self._inner_convert_old_schema(node, 1) for node in merged]
746
def _inner_convert_old_schema(self, node, depth):
748
Internal recursion helper for L{_convert_old_schema}.
750
@param node: A node in the associative list tree as described in
751
_convert_old_schema. A two tuple of (name, parameter).
752
@param depth: The depth that the node is at. This is important to know
753
if we're currently processing a list or a structure. ("foo.N" is a
754
list called "foo", "foo.N.fieldname" describes a field in a list of
757
name, parameter_description = node
758
if not isinstance(parameter_description, list):
759
# This is a leaf, i.e., an actual L{Parameter} instance.
760
return parameter_description
762
# we're processing a structure.
764
for node in parameter_description:
765
fields[node[0]] = self._inner_convert_old_schema(
767
return Structure(name, fields=fields)
769
# we're processing a list.
770
if not isinstance(parameter_description, list):
771
raise TypeError("node %r must be an associative list"
772
% (parameter_description,))
773
if not len(parameter_description) == 1:
775
"Multiple different index names specified: %r"
776
% ([item[0] for item in parameter_description],))
777
subnode = parameter_description[0]
778
item = self._inner_convert_old_schema(subnode, depth + 1)
779
return List(name=name, item=item, optional=item.optional)
782
def _merge_associative_list(alist, path, value):
784
Merge a value into an associative list at the given path, maintaining
785
insertion order. Examples will explain it::
788
>>> _merge_associative_list(alist, ["foo", "bar"], "barvalue")
789
>>> _merge_associative_list(alist, ["foo", "baz"], "bazvalue")
790
>>> alist == [("foo", [("bar", "barvalue"), ("baz", "bazvalue")])]
792
@param alist: An associative list of names to values.
793
@param path: A path through sub-alists which we ultimately want to point to
795
@param value: The value to set.
796
@return: None. This operation mutates the associative list in place.
798
for key in path[:-1]:
805
alist.append((key, subalist))
807
alist.append((path[-1], value))