From 4e9ee2363fee59f0ececdaed39aa5bf883214b61 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Mon, 22 Mar 2021 14:50:16 +0100 Subject: [PATCH] [Backport #7057] ResourceBase for metadata-only resources --- geonode/api/api.py | 13 +- geonode/api/resourcebase_api.py | 23 +++ geonode/api/tests.py | 146 +++++++++++++++++- geonode/base/api/serializers.py | 3 +- geonode/base/api/tests.py | 25 +-- .../0057_resourcebase_metadata_only.py | 18 +++ geonode/base/models.py | 5 + geonode/base/populate_test_data.py | 16 +- geonode/locale/it/LC_MESSAGES/django.po | 3 + geonode/maps/tests.py | 2 +- geonode/security/tests.py | 21 +++ geonode/security/utils.py | 2 +- 12 files changed, 253 insertions(+), 24 deletions(-) create mode 100644 geonode/base/migrations/0057_resourcebase_metadata_only.py diff --git a/geonode/api/api.py b/geonode/api/api.py index 18204aa57e2..c20aa5a82ba 100644 --- a/geonode/api/api.py +++ b/geonode/api/api.py @@ -282,7 +282,7 @@ def dehydrate_layers_count(self, bundle): request = bundle.request obj_with_perms = get_objects_for_user(request.user, 'base.view_resourcebase').filter(polymorphic_ctype__model='layer') - filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')) + filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False) if not settings.SKIP_PERMS_FILTER: filter_set = get_visible_resources( @@ -432,7 +432,7 @@ def dehydrate(self, bundle): request = bundle.request counts = _get_resource_counts( request, - resourcebase_filter_kwargs={'group': bundle.obj} + resourcebase_filter_kwargs={'group': bundle.obj, 'metadata_only': False} ) bundle.data.update(resource_counts=counts) @@ -506,17 +506,20 @@ def dehydrate_email(self, bundle): def dehydrate_layers_count(self, bundle): obj_with_perms = get_objects_for_user(bundle.request.user, 'base.view_resourcebase').filter(polymorphic_ctype__model='layer') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count() + return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ + .distinct().count() def dehydrate_maps_count(self, bundle): obj_with_perms = get_objects_for_user(bundle.request.user, 'base.view_resourcebase').filter(polymorphic_ctype__model='map') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count() + return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ + .distinct().count() def dehydrate_documents_count(self, bundle): obj_with_perms = get_objects_for_user(bundle.request.user, 'base.view_resourcebase').filter(polymorphic_ctype__model='document') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count() + return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ + .distinct().count() def dehydrate_avatar_100(self, bundle): return avatar_url(bundle.obj, 240) diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 09d47d4ab7c..3e72c846386 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -154,6 +154,7 @@ class CommonModelApi(ModelResource): 'is_approved', 'is_published', 'dirty_state', + 'metadata_only' ] def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): @@ -722,6 +723,13 @@ class LayerResource(CommonModelApi): null=True, use_in='detail') + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop('metadata_only', False) + orm_filters = super(LayerResource, self).build_filters(_filters) + orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + return orm_filters + def format_objects(self, objects): """ Formats the object. @@ -865,6 +873,7 @@ class Meta(CommonMetaApi): 'id': ALL, 'name': ALL, 'alternate': ALL, + 'metadata_only': ALL }) @@ -872,6 +881,13 @@ class MapResource(CommonModelApi): """Maps API""" + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop('metadata_only', False) + orm_filters = super(MapResource, self).build_filters(_filters) + orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + return orm_filters + def format_objects(self, objects): """ Formats the objects and provides reference to list of layers in map @@ -1013,6 +1029,13 @@ class DocumentResource(CommonModelApi): """Documents API""" + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop('metadata_only', False) + orm_filters = super(DocumentResource, self).build_filters(_filters) + orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + return orm_filters + def format_objects(self, objects): """ Formats the objects and provides reference to list of layers in map diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 6d5c63217c5..588f0bc0da8 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -17,6 +17,8 @@ # along with this program. If not, see . # ######################################################################### +from geonode.maps.models import Map +from geonode.documents.models import Document from unittest.mock import patch from django.conf import settings @@ -541,7 +543,7 @@ class ThesaurusKeywordResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSuppor def setUp(self): super(ThesaurusKeywordResourceTests, self).setUp() - + all_public() self.list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) def test_api_will_return_a_valid_json_response(self): @@ -610,3 +612,145 @@ def test_will_return_default_keyword_label_for_not_existing_lang(self, lang): actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]] self.assertValidJSONResponse(resp) self.assertListEqual(expected_labels, actual_labels) + + +class LayerResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json' + ] + + def setUp(self): + super(LayerResourceTests, self).setUp() + self.user = get_user_model().objects.get(username="admin") + self.list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) + all_public() + self.token = get_or_create_token(self.user) + self.auth_header = 'Bearer {}'.format(self.token) + + def test_the_api_should_return_all_layers_with_metadata_false(self): + resp = self.api_client.get(self.list_url, authentication=self.auth_header) + self.assertValidJSONResponse(resp) + self.assertEqual(8, resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_layers_with_metadata_true(self): + url = f"{self.list_url}?metadata_only=True" + resp = self.api_client.get(url, authentication=self.auth_header) + self.assertValidJSONResponse(resp) + self.assertEqual(1, resp.json()["meta"]["total_count"]) + + +class DocumentResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json' + ] + + def setUp(self): + super(DocumentResourceTests, self).setUp() + all_public() + self.user = get_user_model().objects.get(username="admin") + self.list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'documents'}) + + def test_the_api_should_return_all_documents_with_metadata_false(self): + resp = self.api_client.get(self.list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 9) + + def test_the_api_should_return_all_documents_with_metadata_true(self): + url = f"{self.list_url}?metadata_only=True" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 1) + + +class MapResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json' + ] + + def setUp(self): + super(MapResourceTests, self).setUp() + all_public() + self.user = get_user_model().objects.get(username="admin") + self.list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'maps'}) + + def test_the_api_should_return_all_maps_with_metadata_false(self): + resp = self.api_client.get(self.list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 9) + + def test_the_api_should_return_all_maps_with_metadata_true(self): + url = f"{self.list_url}?metadata_only=True" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 1) + + +class TopicCategoryResourceTest(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json' + ] + + def setUp(self): + super(TopicCategoryResourceTest, self).setUp() + self.user = get_user_model().objects.get(username="admin") + self.list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'categories'}) + + def test_the_api_should_return_all_maps_with_metadata_false(self): + url = f"{self.list_url}?type=map" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + actual = sum([x['count'] for x in resp.json()['objects']]) + self.assertEqual(9, actual) + + def test_the_api_should_return_all_maps_with_metadata_true(self): + x = Map.objects.get(title='map metadata true') + x.metadata_only = False + x.save() + url = f"{self.list_url}?type=map" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + # by adding a new layer, the total should increase + actual = sum([x['count'] for x in resp.json()['objects']]) + self.assertEqual(10, actual) + + def test_the_api_should_return_all_document_with_metadata_false(self): + url = f"{self.list_url}?type=document" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + actual = sum([x['count'] for x in resp.json()['objects']]) + self.assertEqual(9, actual) + + def test_the_api_should_return_all_document_with_metadata_true(self): + x = Document.objects.get(title='doc metadata true') + x.metadata_only = False + x.save() + url = f"{self.list_url}?type=document" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + # by adding a new layer, the total should increase + actual = sum([x['count'] for x in resp.json()['objects']]) + self.assertEqual(10, actual) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index e098e2de509..d9ef235ee50 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -258,6 +258,7 @@ def __init__(self, *args, **kwargs): self.fields['raw_constraints_other'] = serializers.CharField(read_only=True) self.fields['raw_supplemental_information'] = serializers.CharField(read_only=True) self.fields['raw_data_quality_statement'] = serializers.CharField(read_only=True) + self.fields['metadata_only'] = serializers.BooleanField() self.fields['embed_url'] = EmbedUrlField() self.fields['thumbnail_url'] = ThumbnailUrlField() @@ -289,7 +290,7 @@ class Meta: 'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved', 'detail_url', 'embed_url', 'created', 'last_updated', 'raw_abstract', 'raw_purpose', 'raw_constraints_other', - 'raw_supplemental_information', 'raw_data_quality_statement' + 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only' # TODO # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry, # metadata_uploaded, metadata_uploaded_preserve, metadata_xml, diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index cfcb8d1d9f7..2a779c28160 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -533,14 +533,17 @@ def test_embed_urls(self): for resource in resources: url = reverse('base-resources-detail', kwargs={'pk': resource.pk}) response = self.client.get(url, format='json') - self.assertEqual(response.status_code, 200) - self.assertEqual(int(response.data['resource']['pk']), int(resource.pk)) - embed_url = response.data['resource']['embed_url'] - self.assertIsNotNone(embed_url) - - instance = resource.get_real_instance() - if hasattr(instance, 'embed_url'): - if instance.embed_url != NotImplemented: - self.assertEqual(instance.embed_url, embed_url) - else: - self.assertEqual("", embed_url) + if resource.title.endswith('metadata true'): + self.assertEqual(response.status_code, 404) + else: + self.assertEqual(response.status_code, 200) + self.assertEqual(int(response.data['resource']['pk']), int(resource.pk)) + embed_url = response.data['resource']['embed_url'] + self.assertIsNotNone(embed_url) + + instance = resource.get_real_instance() + if hasattr(instance, 'embed_url'): + if instance.embed_url != NotImplemented: + self.assertEqual(instance.embed_url, embed_url) + else: + self.assertEqual("", embed_url) diff --git a/geonode/base/migrations/0057_resourcebase_metadata_only.py b/geonode/base/migrations/0057_resourcebase_metadata_only.py new file mode 100644 index 00000000000..901a231d9c6 --- /dev/null +++ b/geonode/base/migrations/0057_resourcebase_metadata_only.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-03-10 11:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0056_resourcebase_ll_bbox_polygon'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='metadata_only', + field=models.BooleanField(default=False, help_text='If true, will be excluded from search', verbose_name='Metadata'), + ), + ] \ No newline at end of file diff --git a/geonode/base/models.py b/geonode/base/models.py index 6da61c20eeb..b2a84a75abb 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -890,6 +890,11 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): blank=True, null=True) + metadata_only = models.BooleanField( + _("Metadata"), + default=False, + help_text=_('If true, will be excluded from search')) + __is_approved = False __is_published = False diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index e5ee5cac7d5..c0382e9d79a 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -69,6 +69,7 @@ def create_fixtures(): biota = TopicCategory.objects.get(identifier='biota') location = TopicCategory.objects.get(identifier='location') elevation = TopicCategory.objects.get(identifier='elevation') + farming = TopicCategory.objects.get(identifier='farming') world_extent = [-180, 180, -90, 90] map_data = [ @@ -81,6 +82,7 @@ def create_fixtures(): ('morx', 'common thing double', ('populartag',), [0, 10, 0, 10], elevation), ('titledupe something else ', 'whatever common', ('populartag',), [0, 10, 0, 10], elevation), ('something titledupe else ', 'bar common', ('populartag',), [0, 50, 0, 50], elevation), + ('map metadata true', 'map metadata true', ('populartag',), [0, 22, 0, 22], farming), ] user_data = [ @@ -129,6 +131,8 @@ def callable(): 0, 10, 0, 10], next_date(), ('populartag',), biota), ('common morx', 'lorem ipsum', 'fleem', 'geonode:fleem', [ 0, 50, 0, 50], next_date(), ('populartag',), biota), + ('layer metadata true', 'lorem ipsum', 'fleem', 'geonode:metadatatrue', [ + 0, 22, 0, 22], next_date(), ('populartag',), farming) ] document_data = [('lorem ipsum', 'common lorem ipsum', ('populartag',), world_extent, biota), @@ -139,7 +143,8 @@ def callable(): ('quux', 'common double thing', ('populartag',), [0, 5, 0, 5], location), ('morx', 'common thing double', ('populartag',), [0, 10, 0, 10], elevation), ('titledupe something else ', 'whatever common', ('populartag',), [0, 10, 0, 10], elevation), - ('something titledupe else ', 'bar common', ('populartag',), [0, 50, 0, 50], elevation)] + ('something titledupe else ', 'bar common', ('populartag',), [0, 50, 0, 50], elevation), + ('doc metadata true', 'doc metadata true', ('populartag',), [0, 22, 0, 22], farming)] return map_data, user_data, people_data, layer_data, document_data @@ -200,7 +205,8 @@ def create_models(type=None, integration=False): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', - category=category + category=category, + metadata_only=title == 'map metadata true' ) m.save() m.set_default_permissions() @@ -221,7 +227,8 @@ def create_models(type=None, integration=False): ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', category=category, - doc_file=f + doc_file=f, + metadata_only=title == 'doc metadata true' ) m.save() m.set_default_permissions() @@ -249,7 +256,8 @@ def create_models(type=None, integration=False): temporal_extent_end=end, date=start, storeType=storeType, - category=category + category=category, + metadata_only=title == 'layer metadata true' ) layer.save() layer.set_default_permissions() diff --git a/geonode/locale/it/LC_MESSAGES/django.po b/geonode/locale/it/LC_MESSAGES/django.po index bde1fc7237a..1caf802a8c5 100644 --- a/geonode/locale/it/LC_MESSAGES/django.po +++ b/geonode/locale/it/LC_MESSAGES/django.po @@ -6293,3 +6293,6 @@ msgstr "Livello di permessi non valido." msgid "Permission Denied" msgstr "Permesso negato" + +msgid "If true, will be excluded from search" +msgstr "Se Vero, l'elemento verrà escluso dalla ricerca" diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index bff902fe68f..26cceac1c72 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -205,7 +205,7 @@ def test_map_save(self, thumbnail_mock): self.client.logout() # We have now 10 maps and 8 layers - self.assertEqual(Map.objects.all().count(), 10) + self.assertEqual(Map.objects.all().count(), 11) map_obj = Map.objects.get(id=map_id) self.assertEqual(map_obj.title, "Title") self.assertEqual(map_obj.abstract, "Abstract") diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 575908aaa87..e3780ccf3b9 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -59,6 +59,7 @@ from geonode.layers.populate_layers_data import create_layer_data from .utils import ( + get_visible_resources, purge_geofence_all, get_users_with_perms, get_geofence_rules, @@ -1666,3 +1667,23 @@ def test_sync_resources_with_guardian_delay_true(self): clean_layer = Layer.objects.get(pk=self._l.id) # Check dirty state self.assertFalse(clean_layer.dirty_state) + + +class TestGetVisibleResources(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + def setUp(self): + super(TestGetVisibleResources, self).setUp() + self.user = get_user_model().objects.get(username="admin") + + def test_get_visible_resources_should_return_resource_with_metadata_only_false(self): + layers = Layer.objects.all() + actual = get_visible_resources(queryset=layers, user=self.user) + self.assertEqual(8, len(actual)) + + def test_get_visible_resources_should_return_updated_resource_with_metadata_only_false(self): + # Updating the layer with metadata only True to verify that the filter works + x = Layer.objects.get(title='layer metadata true') + x.metadata_only = False + x.save() + layers = Layer.objects.all() + actual = get_visible_resources(queryset=layers, user=self.user) + self.assertEqual(9, len(actual)) diff --git a/geonode/security/utils.py b/geonode/security/utils.py index 4ed4fdc6d6e..c6f3d625408 100644 --- a/geonode/security/utils.py +++ b/geonode/security/utils.py @@ -69,7 +69,7 @@ def get_visible_resources(queryset, except Exception: pass - filter_set = queryset + filter_set = queryset.filter(metadata_only=False) if not is_admin: if admin_approval_required: