-
-
Notifications
You must be signed in to change notification settings - Fork 744
/
common.py
1300 lines (1056 loc) · 48.7 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
"""
eve.methods.common
~~~~~~~~~~~~~~~~~~
Utility functions for API methods implementations.
:copyright: (c) 2017 by Nicola Iarocci.
:license: BSD, see LICENSE for more details.
"""
import base64
try:
from collections import Counter
except:
from backport_collections import Counter
import simplejson as json
import time
from bson.dbref import DBRef
from bson.errors import InvalidId
from copy import copy
from datetime import datetime
from eve.utils import auto_fields
from eve.utils import config
from eve.utils import debug_error_message
from eve.utils import document_etag
from eve.utils import parse_request
from eve.versioning import get_data_version_relation_document
from eve.versioning import resolve_document_version
from flask import Response
from flask import abort
from flask import current_app as app
from flask import g
from flask import request
from functools import wraps
from werkzeug.datastructures import MultiDict, CombinedMultiDict
def get_document(resource, concurrency_check, original=None, **lookup):
""" Retrieves and return a single document. Since this function is used by
the editing methods (PUT, PATCH, DELETE), we make sure that the client
request references the current representation of the document before
returning it. However, this concurrency control may be turned off by
internal functions. If resource enables soft delete, soft deleted documents
will be returned, and must be handled by callers.
:param resource: the name of the resource to which the document belongs to.
:param concurrency_check: boolean check for concurrency control
:param original: in case the document was already retrieved before
:param **lookup: document lookup query
.. versionchanged:: 0.6
Return soft deleted documents.
.. versionchanged:: 0.5
Concurrency control optional for internal functions.
ETAG are now stored with the document (#369).
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.5
Pass current resource to ``parse_request``, allowing for proper
processing of new configuration settings: `filters`, `sorting`, `paging`.
"""
req = parse_request(resource)
if config.DOMAIN[resource]['soft_delete']:
# get_document should always fetch soft deleted documents from the db
# callers must handle soft deleted documents
req.show_deleted = True
if original:
document = original
else:
document = app.data.find_one(resource, req, **lookup)
if document:
e_if_m = config.ENFORCE_IF_MATCH
if_m = config.IF_MATCH
if not req.if_match and e_if_m and if_m and concurrency_check:
# we don't allow editing unless the client provides an etag
# for the document or explicitly decides to allow editing by either
# disabling the ``concurrency_check`` or ``IF_MATCH`` or
# ``ENFORCE_IF_MATCH`` fields.
abort(428, description='To edit a document '
'its etag must be provided using the If-Match header')
# ensure the retrieved document has LAST_UPDATED and DATE_CREATED,
# eventually with same default values as in GET.
document[config.LAST_UPDATED] = last_updated(document)
document[config.DATE_CREATED] = date_created(document)
if req.if_match and concurrency_check:
ignore_fields = config.DOMAIN[resource]['etag_ignore_fields']
etag = document.get(config.ETAG, document_etag(document,
ignore_fields=ignore_fields))
if req.if_match != etag:
# client and server etags must match, or we don't allow editing
# (ensures that client's version of the document is up to date)
abort(412, description='Client and server etags don\'t match')
return document
def parse(value, resource):
""" Safely evaluates a string containing a Python expression. We are
receiving json and returning a dict.
:param value: the string to be evaluated.
:param resource: name of the involved resource.
.. versionchanged:: 0.1.1
Serialize data-specific values as needed.
.. versionchanged:: 0.1.0
Support for PUT method.
.. versionchanged:: 0.0.5
Support for 'application/json' Content-Type.
.. versionchanged:: 0.0.4
When parsing POST requests, eventual default values are injected in
parsed documents.
"""
try:
# assume it's not decoded to json yet (request Content-Type = form)
document = json.loads(value)
except:
# already a json
document = value
# if needed, get field values serialized by the data diver being used.
# If any error occurs, assume validation will take care of it (i.e. a badly
# formatted objectid).
try:
document = serialize(document, resource)
except:
pass
return document
def payload():
""" Performs sanity checks or decoding depending on the Content-Type,
then returns the request payload as a dict. If request Content-Type is
unsupported, aborts with a 400 (Bad Request).
.. versionchanged:: 0.7
Allow 'multipart/form-data' form fields to be JSON encoded, once the
MULTIPART_FORM_FIELDS_AS_JSON setting was been set.
.. versionchanged:: 0.3
Allow 'multipart/form-data' content type.
.. versionchanged:: 0.1.1
Payload returned as a standard python dict regardless of request content
type.
.. versionchanged:: 0.0.9
More informative error messages.
request.get_json() replaces the now deprecated request.json
.. versionchanged:: 0.0.7
Native Flask request.json preferred over json.loads.
.. versionadded: 0.0.5
"""
content_type = request.headers.get('Content-Type', '').split(';')[0]
if content_type in config.JSON_REQUEST_CONTENT_TYPES:
return request.get_json(force=True)
elif content_type == 'application/x-www-form-urlencoded':
return multidict_to_dict(request.form) if len(request.form) else \
abort(400, description='No form-urlencoded data supplied')
elif content_type == 'multipart/form-data':
# as multipart is also used for file uploads, we let an empty
# request.form go through as long as there are also files in the
# request.
if len(request.form) or len(request.files):
# merge form fields and request files, so we get a single payload
# to be validated against the resource schema.
formItems = MultiDict(request.form)
if config.MULTIPART_FORM_FIELDS_AS_JSON:
for key, lst in formItems.lists():
new_lst = []
for value in lst:
try:
new_lst.append(json.loads(value))
except ValueError:
new_lst.append(json.loads('"{0}"'.format(value)))
formItems.setlist(key, new_lst)
payload = CombinedMultiDict([formItems, request.files])
return multidict_to_dict(payload)
else:
abort(400, description='No multipart/form-data supplied')
else:
abort(400, description='Unknown or no Content-Type header supplied')
def multidict_to_dict(multidict):
""" Convert a MultiDict containing form data into a regular dict. If the
config setting AUTO_COLLAPSE_MULTI_KEYS is True, multiple values with the
same key get entered as a list. If it is False, the first entry is picked.
"""
if config.AUTO_COLLAPSE_MULTI_KEYS:
d = dict(multidict.lists())
for key, value in d.items():
if len(value) == 1:
d[key] = value[0]
return d
else:
return multidict.to_dict()
class RateLimit(object):
""" Implements the Rate-Limiting logic using Redis as a backend.
:param key_prefix: the key used to uniquely identify a client.
:param limit: requests limit, per period.
:param period: limit validity period
:param send_x_headers: True if response headers are supposed to include
special 'X-RateLimit' headers
.. versionadded:: 0.0.7
"""
# Maybe has something complicated problems.
def __init__(self, key, limit, period, send_x_headers=True):
self.reset = int(time.time()) + period
self.key = key
self.limit = limit
self.period = period
self.send_x_headers = send_x_headers
p = app.redis.pipeline()
p.incr(self.key)
p.expireat(self.key, self.reset)
self.current = p.execute()[0]
remaining = property(lambda x: x.limit - x.current)
over_limit = property(lambda x: x.current > x.limit)
def get_rate_limit():
""" If available, returns a RateLimit instance which is valid for the
current request-response.
.. versionadded:: 0.0.7
"""
return getattr(g, '_rate_limit', None)
def ratelimit():
""" Enables support for Rate-Limits on API methods
The key is constructed by default from the remote address or the
authorization.username if authentication is being used. On
a authentication-only API, this will impose a ratelimit even on
non-authenticated users, reducing exposure to DDoS attacks.
Before the function is executed it increments the rate limit with the help
of the RateLimit class and stores an instance on g as g._rate_limit. Also
if the client is indeed over limit, we return a 429, see
http://tools.ietf.org/html/draft-nottingham-http-new-status-04#section-4
.. versionadded:: 0.0.7
"""
def decorator(f):
@wraps(f)
def rate_limited(*args, **kwargs):
method_limit = app.config.get('RATE_LIMIT_' + request.method)
if method_limit and app.redis:
limit = method_limit[0]
period = method_limit[1]
# If authorization is being used the key is 'username'.
# Else, fallback to client IP.
key = 'rate-limit/%s' % (request.authorization.username
if request.authorization else
request.remote_addr)
rlimit = RateLimit(key, limit, period, True)
if rlimit.over_limit:
return Response('Rate limit exceeded', 429)
# store the rate limit for further processing by
# send_response
g._rate_limit = rlimit
else:
g._rate_limit = None
return f(*args, **kwargs)
return rate_limited
return decorator
def last_updated(document):
""" Fixes document's LAST_UPDATED field value. Flask-PyMongo returns
timezone-aware values while stdlib datetime values are timezone-naive.
Comparisons between the two would fail.
If LAST_UPDATE is missing we assume that it has been created outside of the
API context and inject a default value, to allow for proper computing of
Last-Modified header tag. By design all documents return a LAST_UPDATED
(and we don't want to break existing clients).
:param document: the document to be processed.
.. versionchanged:: 0.1.0
Moved to common.py and renamed as public, so it can also be used by edit
methods (via get_document()).
.. versionadded:: 0.0.5
"""
if config.LAST_UPDATED in document:
return document[config.LAST_UPDATED].replace(tzinfo=None)
else:
return epoch()
def date_created(document):
""" If DATE_CREATED is missing we assume that it has been created outside
of the API context and inject a default value. By design all documents
return a DATE_CREATED (and we dont' want to break existing clients).
:param document: the document to be processed.
.. versionchanged:: 0.1.0
Moved to common.py and renamed as public, so it can also be used by edit
methods (via get_document()).
.. versionadded:: 0.0.5
"""
return document[config.DATE_CREATED] if config.DATE_CREATED in document \
else epoch()
def epoch():
""" A datetime.min alternative which won't crash on us.
.. versionchanged:: 0.1.0
Moved to common.py and renamed as public, so it can also be used by edit
methods (via get_document()).
.. versionadded:: 0.0.5
"""
return datetime(1970, 1, 1)
def serialize(document, resource=None, schema=None, fields=None):
""" Recursively handles field values that require data-aware serialization.
Relies on the app.data.serializers dictionary.
.. versionchanged:: 0.7
Add support for normalizing anyof-like rules inside lists. See #876.
.. versionchanged:: 0.6
Add support for normalizing dotted fields.
.. versionchanged:: 0.5.4
Fix serialization of lists of lists. See # 614.
.. versionchanged:: 0.5.3
Don't block on custom serialization errors so the whole document can
be processed. See #568.
.. versionchanged:: 0.5.2
Fix serialization of keyschemas with objectids. See #525.
.. versionchanged:: 0.3
Fix serialization of sub-documents. See #244.
.. versionadded:: 0.1.1
"""
normalize_dotted_fields(document)
if app.data.serializers:
if resource:
schema = config.DOMAIN[resource]['schema']
if not fields:
fields = document.keys()
for field in fields:
if document[field] is None:
continue
if field in schema:
field_schema = schema[field]
field_type = field_schema.get('type')
for x_of in ['allof', 'anyof', 'oneof', 'noneof']:
for optschema in field_schema.get(x_of, []):
optschema = dict(field_schema, **optschema)
optschema.pop(x_of, None)
serialize(document, schema={field: optschema})
x_of_type = '{0}_type'.format(x_of)
for opttype in field_schema.get(x_of_type, []):
optschema = dict(field_schema, type=opttype)
optschema.pop(x_of_type, None)
serialize(document, schema={field: optschema})
if config.AUTO_CREATE_LISTS and field_type == 'list':
# Convert single values to lists
if not isinstance(document[field], list):
document[field] = [document[field]]
if 'schema' in field_schema:
field_schema = field_schema['schema']
if 'dict' in (field_type, field_schema.get('type')):
# either a dict or a list of dicts
embedded = [document[field]] if field_type == 'dict' \
else document[field]
for subdocument in embedded:
if type(subdocument) is not dict:
# value is not a dict - continue serialization
# error will be reported by validation if
# appropriate
continue
elif 'schema' in field_schema:
serialize(subdocument,
schema=field_schema['schema'])
else:
serialize(subdocument, schema=field_schema)
elif field_schema.get('type') == 'list':
# a list of lists
sublist_schema = field_schema.get('schema')
item_type = sublist_schema.get('type')
for sublist in document[field]:
for i, v in enumerate(sublist):
if item_type == 'dict':
serialize(sublist[i],
schema=sublist_schema['schema'])
elif item_type in app.data.serializers:
sublist[i] = serialize_value(item_type, v)
elif field_schema.get('type') is None:
# a list of items determined by *of rules
for x_of in ['allof', 'anyof', 'oneof', 'noneof']:
for optschema in field_schema.get(x_of, []):
serialize(document,
schema={field: {'type': field_type,
'schema': optschema}})
x_of_type = '{0}_type'.format(x_of)
for opttype in field_schema.get(x_of_type, []):
serialize(document,
schema={field: {'type': field_type,
'schema': {'type': opttype}}})
else:
# a list of one type, arbitrary length
field_type = field_schema.get('type')
if field_type in app.data.serializers:
for i, v in enumerate(document[field]):
document[field][i] = \
serialize_value(field_type, v)
elif 'items' in field_schema:
# a list of multiple types, fixed length
for i, (s, v) in enumerate(zip(field_schema['items'],
document[field])):
field_type = s.get('type')
if field_type in app.data.serializers:
document[field][i] = \
serialize_value(field_type, document[field][i])
elif 'valueschema' in field_schema:
# a valueschema
field_type = field_schema['valueschema']['type']
if field_type == 'objectid':
target = document[field]
for field in target:
target[field] = \
serialize_value(field_type, target[field])
elif field_type == 'dict':
for subdocument in document[field].values():
serialize(
subdocument,
schema=field_schema['valueschema']['schema'])
elif field_type in app.data.serializers:
# a simple field
document[field] = \
serialize_value(field_type, document[field])
return document
def serialize_value(field_type, value):
"""Serialize value of a given type. Relies on the app.data.serializers
dictionary.
"""
try:
return app.data.serializers[field_type](value)
except (KeyError, ValueError, TypeError, InvalidId):
# value can't be cast or no serializer defined, return as is and
# validation will later report back the issue.
return value
def normalize_dotted_fields(document):
""" Normalizes eventual dotted fields so validation can be performed
seamlessly. For example this document:
{"location.city": "a nested city"}
would be normalized to:
{"location": {"city": "a nested city"}}
Being recursive, normalizing of sub-documents is also supported. For
example:
{"location": {"city": "a city", "sub.address": "a subaddress"}}
would be normalized to:
{"location": {"city": "a city", "sub": {"address": "a subaddress}}}
.. versionchanged:: 0.7
Fix normalization of nested inputs (#738).
.. versionadded:: 0.6
"""
if isinstance(document, list):
prev = document
for i in prev:
normalize_dotted_fields(i)
elif isinstance(document, dict):
for field in list(document):
if '.' in field:
parts = field.split('.')
prev = document
for part in parts[:-1]:
if part not in prev:
prev[part] = {}
prev = prev[part]
if isinstance(document[field], (dict, list)):
normalize_dotted_fields(document[field])
prev[parts[-1]] = document[field]
document.pop(field)
elif isinstance(document[field], (dict, list)):
normalize_dotted_fields(document[field])
def build_response_document(
document, resource, embedded_fields, latest_doc=None):
""" Prepares a document for response including generation of ETag and
metadata fields.
:param document: the document to embed other documents into.
:param resource: the resource name.
:param embedded_fields: the list of fields we are allowed to embed.
:param document: the latest version of document.
.. versionchanged:: 0.5
Only compute ETAG if necessary (#369).
Add version support (#475).
.. versionadded:: 0.4
"""
resource_def = config.DOMAIN[resource]
# need to update the document field since the etag must be computed on the
# same document representation that might have been used in the collection
# 'get' method
document[config.DATE_CREATED] = date_created(document)
document[config.LAST_UPDATED] = last_updated(document)
# Up to v0.4 etags were not stored with the documents.
if config.IF_MATCH and config.ETAG not in document:
ignore_fields = resource_def['etag_ignore_fields']
document[config.ETAG] = document_etag(document,
ignore_fields=ignore_fields)
# hateoas links
if resource_def['hateoas'] and resource_def['id_field'] in document:
version = None
if resource_def['versioning'] is True \
and request.args.get(config.VERSION_PARAM):
version = document[config.VERSION]
self_dict = {'self': document_link(resource,
document[resource_def['id_field']],
version)}
if config.LINKS not in document:
document[config.LINKS] = self_dict
elif 'self' not in document[config.LINKS]:
document[config.LINKS].update(self_dict)
# add version numbers
resolve_document_version(document, resource, 'GET', latest_doc)
# resolve media
resolve_media_files(document, resource)
# resolve soft delete
if resource_def['soft_delete'] is True:
if document.get(config.DELETED) is None:
document[config.DELETED] = False
elif document[config.DELETED] is True:
# Soft deleted documents are sent without expansion of embedded
# documents. Return before resolving them.
return
# resolve embedded documents
resolve_embedded_documents(document, resource, embedded_fields)
def field_definition(resource, chained_fields):
""" Resolves query string to resource with dot notation like
'people.address.city' and returns corresponding field definition
of the resource
:param resource: the resource name whose field to be accepted.
:param chained_fields: query string to retrieve field definition
.. versionadded 0.5
"""
definition = config.DOMAIN[resource]
subfields = chained_fields.split('.')
for field in subfields:
if field not in definition.get('schema', {}):
if 'data_relation' in definition:
sub_resource = definition['data_relation']['resource']
definition = config.DOMAIN[sub_resource]
if field not in definition['schema']:
return
definition = definition['schema'][field]
field_type = definition.get('type')
if field_type == 'list':
definition = definition['schema']
elif field_type == 'objectid':
pass
return definition
def resolve_embedded_fields(resource, req):
""" Returns a list of validated embedded fields from the incoming request
or from the resource definition is the request does not specify.
:param resource: the resource name.
:param req: and instace of :class:`eve.utils.ParsedRequest`.
.. versionchanged:: 0.5
Enables subdocuments embedding. #389.
.. versionadded:: 0.4
"""
embedded_fields = []
non_embedded_fields = []
if req.embedded:
# Parse the embedded clause, we are expecting
# something like: '{"user":1}'
try:
client_embedding = json.loads(req.embedded)
except ValueError:
abort(400, description='Unable to parse `embedded` clause')
# Build the list of fields where embedding is being requested
try:
embedded_fields = [k for k, v in client_embedding.items()
if v == 1]
non_embedded_fields = [k for k, v in client_embedding.items()
if v == 0]
except AttributeError:
# We got something other than a dict
abort(400, description='Unable to parse `embedded` clause')
embedded_fields = list(
(set(config.DOMAIN[resource]['embedded_fields']) |
set(embedded_fields)) - set(non_embedded_fields))
# For each field, is the field allowed to be embedded?
# Pick out fields that have a `data_relation` where `embeddable=True`
enabled_embedded_fields = []
for field in sorted(embedded_fields, key=lambda a: a.count('.')):
# Reject bogus field names
field_def = field_definition(resource, field)
if field_def:
if field_def.get('type') == 'list':
field_def = field_def['schema']
if 'data_relation' in field_def and \
field_def['data_relation'].get('embeddable'):
# or could raise 400 here
enabled_embedded_fields.append(field)
return enabled_embedded_fields
def embedded_document(references, data_relation, field_name):
""" Returns a document to be embedded by reference using data_relation
taking into account document versions
:param reference: reference to the document to be embedded.
:param data_relation: the relation schema definition.
:param field_name: field name used in abort message only
.. versionadded:: 0.5
"""
embedded_docs = []
output_is_list = True
if not isinstance(references, list):
output_is_list = False
references = [references]
# Retrieve and serialize the requested document
if 'version' in data_relation and data_relation['version'] is True:
# For the version flow, I keep the as-is logic (flow is too complex to make it bulk)
for reference in references:
# grab the specific version
embedded_doc = get_data_version_relation_document(
data_relation, reference)
# grab the latest version
latest_embedded_doc = get_data_version_relation_document(
data_relation, reference, latest=True)
# make sure we got the documents
if embedded_doc is None or latest_embedded_doc is None:
# your database is not consistent!!! that is bad
# TODO: we should notify the developers with a log.
abort(404, description=debug_error_message(
"Unable to locate embedded documents for '%s'" %
field_name
))
build_response_document(embedded_doc, data_relation['resource'],
[], latest_embedded_doc)
embedded_docs.append(embedded_doc)
else:
id_value_to_sort, list_of_id_field_name, subresources_query = generate_query_and_sorting_criteria(data_relation,
references)
for subresource in subresources_query:
list_embedded_doc = list(app.data.find(subresource,
None,
subresources_query[subresource]))
if not list_embedded_doc:
embedded_docs.extend([None] *
len(subresources_query[subresource]["$or"]))
else:
for embedded_doc in list_embedded_doc:
resolve_media_files(embedded_doc, subresource)
embedded_docs.extend(list_embedded_doc)
# After having retrieved my data, I have to be sure that the sorting of the
# list is the same in input as in output (this is to support embedding of
# sub-documents - only in case the storage is not done via DBref)
if embedded_docs:
embedded_docs = sort_db_response(embedded_docs, id_value_to_sort, list_of_id_field_name)
if output_is_list:
return embedded_docs
elif embedded_docs:
return embedded_docs[0]
else:
return None
def sort_db_response(embedded_docs, id_value_to_sort, list_of_id_field_name):
""" Sorts the documents fetched from the database
:param embedded_docs: the documents fetch from the database.
:param id_value_to_sort: id_value sort criteria.
:param list_of_id_field_name: list of name of fields
:return embedded_docs: the list of documents sorted as per input
"""
id_field_name_occurrences = Counter(list_of_id_field_name)
temp_embedded_docs = []
old_occurrence = 0
for id_field_name in set(list_of_id_field_name):
current_occurrence = old_occurrence + int(id_field_name_occurrences[id_field_name])
temp_embedded_docs.extend(
sort_per_resource(embedded_docs[old_occurrence:current_occurrence],
id_value_to_sort,
id_field_name))
old_occurrence = current_occurrence
return temp_embedded_docs
def sort_per_resource(embedded_docs, id_value_to_sort, id_field_name):
""" Sorts the documents fetched from the database per single resource
:param embedded_docs: the documents fetch from the database.
:param id_value_to_sort: id_value sort criteria.
:param list_of_id_field_name: list of name of fields
:return embedded_docs: the list of documents sorted as per input
"""
# Removing None
number_of_none = embedded_docs.count(None)
if number_of_none:
embedded_docs = [x for x in embedded_docs if x is not None]
id2dict = dict((d[id_field_name], d) for d in embedded_docs)
temporary_embedded_docs = []
if number_of_none:
for id_value_ in id_value_to_sort:
if id_value_ in id2dict:
temporary_embedded_docs.append(id2dict[id_value_])
else:
temporary_embedded_docs.append(None)
return embedded_docs
def generate_query_and_sorting_criteria(data_relation, references):
""" Generate query and sorting critiria
:param data_relation: data relation for the resource.
:param references: DBRef or id to use to embed the document.
:returns id_value_to_sort: list of ids to use in the sort
list_of_id_field_name: list of field name (important only for DBRef)
subresources_query: the list of query to perform per resource
(in case is not DBRef, it will be only one query)
"""
query = {"$or": []}
subresources_query = {}
old_subresource = ""
id_value_to_sort = []
# id_field name should be the same for
# all the elements in the list
list_of_id_field_name = []
for counter, reference in enumerate(references):
# if reference is DBRef take the referenced collection as subresource
# NOTE: using DBRef, I can define several resource for each link
subresource = reference.collection if isinstance(reference, DBRef) \
else data_relation['resource']
if old_subresource and old_subresource != subresource:
add_query_to_list(query, subresource, subresources_query)
# NOTE: in case it is a DBRef link, the id_field_name is always the _id
# regardless the Eve set-up
id_field_name = "_id" if isinstance(reference, DBRef) \
else config.DOMAIN[subresource]['id_field']
id_field_value = reference.id \
if isinstance(reference, DBRef) else reference
query["$or"].append({id_field_name: id_field_value})
id_value_to_sort.append(id_field_value)
list_of_id_field_name.append(id_field_name)
if counter == len(references) - 1:
add_query_to_list(query, subresource, subresources_query)
return id_value_to_sort, list_of_id_field_name, subresources_query
def add_query_to_list(query, subresource, subresource_query):
subresource_query.update({subresource: copy(query)})
query.clear()
query["$or"] = []
def subdocuments(fields_chain, resource, document):
""" Traverses the given document and yields subdocuments which
correspond to the given fields_chain
:param fields_chain: list of nested field names.
:param resource: the resource name.
:param document: document to be traversed
.. versionadded:: 0.5
"""
if len(fields_chain) == 0:
yield document
elif isinstance(document, dict) and fields_chain[0] in document:
subdocument = document[fields_chain[0]]
docs = subdocument if isinstance(subdocument, list) else [subdocument]
try:
resource = field_definition(
resource, fields_chain[0])['data_relation']['resource']
except KeyError:
resource = resource
for doc in docs:
for result in subdocuments(fields_chain[1:], resource, doc):
yield result
else:
yield document
def resolve_embedded_documents(document, resource, embedded_fields):
""" Loops through the documents, adding embedded representations
of any fields that are (1) defined eligible for embedding in the
DOMAIN and (2) requested to be embedded in the current `req`.
Currently we support embedding of documents by references located
in any subdocuments. For example, query embedded={"user.friends":1}
will return a document with "user" and all his "friends" embedded,
but only if "user" is a subdocument.
We do not support multiple layers embeddings.
:param document: the document to embed other documents into.
:param resource: the resource name.
:param embedded_fields: the list of fields we are allowed to embed.
.. versionchanged:: 0.5
Support for embedding documents located in subdocuments.
Allocated two functions embedded_document and subdocuments.
.. versionchanged:: 0.4
Moved parsing of embedded fields to _resolve_embedded_fields.
Support for document versioning.
.. versionchanged:: 0.2
Support for 'embedded_fields'.
.. versionchanged:: 0.1.1
'collection' key has been renamed to 'resource' (data_relation).
.. versionadded:: 0.1.0
"""
# NOTE(Gonéri): We resolve the embedded documents at the end.
for field in sorted(embedded_fields, key=lambda a: a.count('.')):
data_relation = field_definition(resource, field)['data_relation']
getter = lambda ref: embedded_document(ref, data_relation, field) # noqa
fields_chain = field.split('.')
last_field = fields_chain[-1]
for subdocument in subdocuments(fields_chain[:-1], resource, document):
if last_field not in subdocument:
continue
subdocument[last_field] = getter(subdocument[last_field])
def resolve_media_files(document, resource):
""" Embed media files into the response document.
:param document: the document eventually containing the media files.
:param resource: the resource being consumed by the request.
.. versionadded:: 0.4
"""
for field in resource_media_fields(document, resource):
if isinstance(document[field], list):
resolved_list = []
for file_id in document[field]:
resolved_list.append(resolve_one_media(file_id, resource))
document[field] = resolved_list
else:
document[field] = resolve_one_media(document[field], resource)
def resolve_one_media(file_id, resource):
""" Get response for one media file """
_file = app.media.get(file_id, resource)
if _file:
# otherwise we have a valid file and should send extended response
# start with the basic file object
if config.RETURN_MEDIA_AS_BASE64_STRING:
ret_file = base64.encodestring(_file.read())
elif config.RETURN_MEDIA_AS_URL:
prefix = config.MEDIA_BASE_URL if config.MEDIA_BASE_URL \
is not None else app.api_prefix
ret_file = '%s/%s/%s' % (prefix, config.MEDIA_ENDPOINT,
file_id)
else:
ret_file = None
if config.EXTENDED_MEDIA_INFO:
ret = {
'file': ret_file,
}
# check if we should return any special fields
for attribute in config.EXTENDED_MEDIA_INFO:
if hasattr(_file, attribute):
# add extended field if found in the file object
ret.update({
attribute: getattr(_file, attribute)
})
else:
# tried to select an invalid attribute
abort(500, description=debug_error_message(
'Invalid extended media attribute requested'
))
return ret
else:
return ret_file
else:
return None
def marshal_write_response(document, resource):
""" Limit response document to minimize bandwidth when client supports it.
:param document: the response document.
:param resource: the resource being consumed by the request.
.. versionchanged: 0.5
Avoid exposing 'auth_field' if it is not intended to be public.
.. versionadded:: 0.4
"""
resource_def = app.config['DOMAIN'][resource]
if app.config['BANDWIDTH_SAVER'] is True:
# only return the automatic fields and special extra fields
fields = auto_fields(resource) + resource_def['extra_response_fields']
document = dict((k, v) for (k, v) in document.items() if k in fields)
else:
# avoid exposing the auth_field if it is not included in the
# resource schema.
auth_field = resource_def.get('auth_field')
if auth_field and auth_field not in resource_def['schema']:
try: