diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e0e54a53e7..79ebd63c91b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,8 +88,9 @@ jobs: working_directory: ./ no_output_timeout: 10m - run: - name: Run pep8 checks + name: Run code quality checks command: | + docker-compose -f docker-compose-test.yml exec django bash -c 'black --check geonode' docker-compose -f docker-compose-test.yml exec django bash -c 'flake8 geonode' docker-compose -f docker-compose-test.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8' working_directory: ./ diff --git a/.env_dev b/.env_dev index dba52f04cb3..9acead0ba21 100644 --- a/.env_dev +++ b/.env_dev @@ -93,7 +93,7 @@ GEOSERVER_LOCATION=http://localhost:8080/geoserver/ GEOSERVER_ADMIN_USER=admin GEOSERVER_ADMIN_PASSWORD=geoserver -OGC_REQUEST_TIMEOUT=5 +OGC_REQUEST_TIMEOUT=60 OGC_REQUEST_MAX_RETRIES=0 OGC_REQUEST_BACKOFF_FACTOR=0.3 OGC_REQUEST_POOL_MAXSIZE=10 diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index cd28c673bb6..29ad6d52f26 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,16 +1,27 @@ name: Backport on: - pull_request: + pull_request_target: types: - closed - labeled jobs: backport: - runs-on: ubuntu-18.04 name: Backport + runs-on: ubuntu-latest + # Only react to merged PRs for security reasons. + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. + if: > + github.event.pull_request.merged + && ( + github.event.action == 'closed' + || ( + github.event.action == 'labeled' + && contains(github.event.label.name, 'backport') + ) + ) steps: - name: Backporting - uses: tibdex/backport@v1.0.0 + uses: tibdex/backport@v2 with: - github_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 7925a436a44..db5ca65a4b3 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Workshop](https://docs.geonode.org/en/master/usage/index.html). Install ------- - The latest official release is 4.0.0! + The latest official release is 4.0.2! GeoNode can be setup in different ways, flavors and plattforms. If you´re planning to do development or install for production please visit diff --git a/dev_config.yml b/dev_config.yml index cbcb23b126c..d3cdb0eafc9 100644 --- a/dev_config.yml +++ b/dev_config.yml @@ -1,6 +1,6 @@ --- -GEOSERVER_URL: "https://artifacts.geonode.org/geoserver/2.20.5/geoserver.war" -DATA_DIR_URL: "https://artifacts.geonode.org/geoserver/2.20.5/geonode-geoserver-ext-web-app-data.zip" +GEOSERVER_URL: "https://artifacts.geonode.org/geoserver/2.20.6/geoserver.war" +DATA_DIR_URL: "https://artifacts.geonode.org/geoserver/2.20.6/geonode-geoserver-ext-web-app-data.zip" JETTY_RUNNER_URL: "https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.31.v20200723/jetty-runner-9.4.31.v20200723.jar" WINDOWS: py2exe: "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml index 44026a274c1..c3414316458 100644 --- a/docker-compose-geoserver-server.yml +++ b/docker-compose-geoserver-server.yml @@ -2,7 +2,7 @@ version: '2.2' services: data-dir-conf: - image: geonode/geoserver_data:2.20.5 + image: geonode/geoserver_data:2.20.6 restart: on-failure container_name: gsconf4${COMPOSE_PROJECT_NAME} labels: @@ -13,7 +13,7 @@ services: - geoserver-data-dir:/geoserver_data/data geoserver: - image: geonode/geoserver:2.20.5 + image: geonode/geoserver:2.20.6 restart: unless-stopped container_name: geoserver4${COMPOSE_PROJECT_NAME} stdin_open: true diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 1137d5e00d0..b5e6ccde4f6 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -89,7 +89,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.20.5 + image: geonode/geoserver:2.20.6 container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8080/geoserver/rest/workspaces/geonode.html" @@ -113,7 +113,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.20.5 + image: geonode/geoserver_data:2.20.6 container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 72397a7e2e2..0cf35c972c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.20.5 + image: geonode/geoserver:2.20.6 container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8080/geoserver/rest/workspaces/geonode.html" @@ -113,7 +113,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.20.5 + image: geonode/geoserver_data:2.20.6 container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/geonode/__init__.py b/geonode/__init__.py index 9ccfdb6f96e..e221b120dab 100644 --- a/geonode/__init__.py +++ b/geonode/__init__.py @@ -19,7 +19,7 @@ import os -__version__ = (4, 1, 0, 'dev', 0) +__version__ = (4, 1, 0, "dev", 0) default_app_config = "geonode.apps.AppConfig" @@ -27,16 +27,19 @@ def get_version(): import geonode.version + return geonode.version.get_version(__version__) def main(global_settings, **settings): from django.core.wsgi import get_wsgi_application - os.environ.setdefault('DJANGO_SETTINGS_MODULE', settings.get('django_settings')) + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings.get("django_settings")) app = get_wsgi_application() return app class GeoNodeException(Exception): """Base class for exceptions in this module.""" + pass diff --git a/geonode/api/api.py b/geonode/api/api.py index 55bb2a51bb9..0663176dec9 100644 --- a/geonode/api/api.py +++ b/geonode/api/api.py @@ -35,8 +35,13 @@ from geonode import geoserver from geonode.api.paginator import CrossSiteXHRPaginator -from geonode.api.authorization import GeoNodeStyleAuthorization, ApiLockdownAuthorization, \ - GroupAuthorization, GroupProfileAuthorization, GeoNodePeopleAuthorization +from geonode.api.authorization import ( + GeoNodeStyleAuthorization, + ApiLockdownAuthorization, + GroupAuthorization, + GroupProfileAuthorization, + GeoNodePeopleAuthorization, +) from guardian.shortcuts import get_objects_for_user from tastypie.bundle import Bundle @@ -61,12 +66,7 @@ from geonode.utils import check_ogc_backend from geonode.security.utils import get_visible_resources -FILTER_TYPES = { - 'dataset': Dataset, - 'map': Map, - 'document': Document, - 'geoapp': GeoApp -} +FILTER_TYPES = {"dataset": Dataset, "map": Map, "document": Document, "geoapp": GeoApp} class CountJSONSerializer(Serializer): @@ -76,33 +76,29 @@ def get_resources_counts(self, options): if settings.SKIP_PERMS_FILTER: resources = ResourceBase.objects.all() else: - resources = get_objects_for_user( - options['user'], - 'base.view_resourcebase' - ) + resources = get_objects_for_user(options["user"], "base.view_resourcebase") resources = get_visible_resources( resources, - options['user'], + options["user"], admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) subtypes = [] if resources and resources.exists(): - if options['title_filter']: - resources = resources.filter(title__icontains=options['title_filter']) - if options['type_filter']: - _type_filter = options['type_filter'] + if options["title_filter"]: + resources = resources.filter(title__icontains=options["title_filter"]) + if options["type_filter"]: + _type_filter = options["type_filter"] for label, app in apps.app_configs.items(): - if hasattr(app, 'type') and app.type == 'GEONODE_APP': - if hasattr(app, 'default_model'): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): _model = apps.get_model(label, app.default_model) if issubclass(_model, _type_filter): - subtypes.append( - resources.filter( - polymorphic_ctype__model=_model.__name__.lower())) + subtypes.append(resources.filter(polymorphic_ctype__model=_model.__name__.lower())) if not isinstance(_type_filter, str): _type_filter = _type_filter.__name__.lower() @@ -111,36 +107,34 @@ def get_resources_counts(self, options): counts = list() if subtypes: for subtype in subtypes: - counts.extend( - list(subtype.values(options['count_type']).annotate(count=Count(options['count_type']))) - ) + counts.extend(list(subtype.values(options["count_type"]).annotate(count=Count(options["count_type"])))) else: - counts = list(resources.values(options['count_type']).annotate(count=Count(options['count_type']))) + counts = list(resources.values(options["count_type"]).annotate(count=Count(options["count_type"]))) _counts = {} for c in counts: - if c and c['count'] and options['count_type']: - if not _counts.get(c[options['count_type']], None): - _counts.update({c[options['count_type']]: c['count']}) + if c and c["count"] and options["count_type"]: + if not _counts.get(c[options["count_type"]], None): + _counts.update({c[options["count_type"]]: c["count"]}) else: - _counts[c[options['count_type']]] += c['count'] + _counts[c[options["count_type"]]] += c["count"] return _counts def to_json(self, data, options=None): options = options or {} data = self.to_simple(data, options) counts = self.get_resources_counts(options) - if 'objects' in data: - for item in data['objects']: - item['count'] = counts.get(item['id'], 0) + if "objects" in data: + for item in data["objects"]: + item["count"] = counts.get(item["id"], 0) # Add in the current time. - data['requested_time'] = time.time() + data["requested_time"] = time.time() return json.dumps(data, cls=DjangoJSONEncoder, sort_keys=True) class TypeFilteredResource(ModelResource): - """ Common resource used to apply faceting to categories, keywords, and + """Common resource used to apply faceting to categories, keywords, and regions based on the type passed as query parameter in the form type:dataset/map/document""" @@ -154,21 +148,21 @@ def build_filters(self, filters=None, ignore_bad_filters=False): orm_filters = super().build_filters(filters) - if 'type' in filters and filters['type'] in FILTER_TYPES.keys(): - self.type_filter = FILTER_TYPES[filters['type']] + if "type" in filters and filters["type"] in FILTER_TYPES.keys(): + self.type_filter = FILTER_TYPES[filters["type"]] else: self.type_filter = None - if 'title__icontains' in filters: - self.title_filter = filters['title__icontains'] + if "title__icontains" in filters: + self.title_filter = filters["title__icontains"] return orm_filters def serialize(self, request, data, format, options=None): if options is None: options = {} - options['title_filter'] = getattr(self, 'title_filter', None) - options['type_filter'] = getattr(self, 'type_filter', None) - options['user'] = request.user + options["title_filter"] = getattr(self, "title_filter", None) + options["type_filter"] = getattr(self, "type_filter", None) + options["user"] = request.user return super().serialize(request, data, format, options) @@ -179,16 +173,16 @@ class TagResource(TypeFilteredResource): def serialize(self, request, data, format, options=None): if options is None: options = {} - options['count_type'] = 'keywords' + options["count_type"] = "keywords" return super().serialize(request, data, format, options) class Meta: - queryset = HierarchicalKeyword.objects.all().order_by('name') - resource_name = 'keywords' - allowed_methods = ['get'] + queryset = HierarchicalKeyword.objects.all().order_by("name") + resource_name = "keywords" + allowed_methods = ["get"] filtering = { - 'slug': ALL, + "slug": ALL, } serializer = CountJSONSerializer() authorization = ApiLockdownAuthorization() @@ -203,19 +197,19 @@ class ThesaurusKeywordResource(TypeFilteredResource): def build_filters(self, filters={}, ignore_bad_filters=False): """adds filtering by current language""" _filters = filters.copy() - id = _filters.pop('id', None) + id = _filters.pop("id", None) orm_filters = super().build_filters(_filters) if id is not None: - orm_filters['id__in'] = id + orm_filters["id__in"] = id - if 'thesaurus' in _filters: - orm_filters['thesaurus__identifier'] = _filters['thesaurus'] + if "thesaurus" in _filters: + orm_filters["thesaurus__identifier"] = _filters["thesaurus"] return orm_filters def serialize(self, request, data, format, options={}): - options['count_type'] = 'tkeywords__id' + options["count_type"] = "tkeywords__id" return super().serialize(request, data, format, options) @@ -230,28 +224,25 @@ def dehydrate_thesaurus_identifier(self, bundle): def dehydrate(self, bundle): lang = get_language() - label = ThesaurusKeywordLabel.objects.filter(keyword=bundle.data['id']).filter(lang=lang) + label = ThesaurusKeywordLabel.objects.filter(keyword=bundle.data["id"]).filter(lang=lang) if label.exists(): - bundle.data['label_id'] = label.get().id - bundle.data['label'] = label.get().label - bundle.data['alt_label'] = label.get().label + bundle.data["label_id"] = label.get().id + bundle.data["label"] = label.get().label + bundle.data["alt_label"] = label.get().label else: - bundle.data['label'] = bundle.data['alt_label'] + bundle.data["label"] = bundle.data["alt_label"] return bundle class Meta: - queryset = ThesaurusKeyword.objects \ - .all() \ - .order_by('alt_label') \ - .select_related('thesaurus') + queryset = ThesaurusKeyword.objects.all().order_by("alt_label").select_related("thesaurus") - resource_name = 'thesaurus/keywords' - allowed_methods = ['get'] + resource_name = "thesaurus/keywords" + allowed_methods = ["get"] filtering = { - 'id': ALL, - 'alt_label': ALL, - 'thesaurus': ALL, + "id": ALL, + "alt_label": ALL, + "thesaurus": ALL, } serializer = CountJSONSerializer() authorization = ApiLockdownAuthorization() @@ -263,17 +254,17 @@ class RegionResource(TypeFilteredResource): def serialize(self, request, data, format, options=None): if options is None: options = {} - options['count_type'] = 'regions' + options["count_type"] = "regions" return super().serialize(request, data, format, options) class Meta: - queryset = Region.objects.all().order_by('name') - resource_name = 'regions' - allowed_methods = ['get'] + queryset = Region.objects.all().order_by("name") + resource_name = "regions" + allowed_methods = ["get"] filtering = { - 'name': ALL, - 'code': ALL, + "name": ALL, + "code": ALL, } if settings.API_INCLUDE_REGIONS_COUNT: serializer = CountJSONSerializer() @@ -282,13 +273,15 @@ class Meta: class TopicCategoryResource(TypeFilteredResource): """Category api""" + layers_count = fields.IntegerField(default=0) def dehydrate_datasets_count(self, bundle): request = bundle.request - obj_with_perms = get_objects_for_user(request.user, - 'base.view_resourcebase').filter(polymorphic_ctype__model='dataset') - filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False) + obj_with_perms = get_objects_for_user(request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="dataset" + ) + filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")).filter(metadata_only=False) if not settings.SKIP_PERMS_FILTER: filter_set = get_visible_resources( @@ -296,23 +289,24 @@ def dehydrate_datasets_count(self, bundle): request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) return filter_set.distinct().count() def serialize(self, request, data, format, options=None): if options is None: options = {} - options['count_type'] = 'category' + options["count_type"] = "category" return super().serialize(request, data, format, options) class Meta: queryset = TopicCategory.objects.all() - resource_name = 'categories' - allowed_methods = ['get'] + resource_name = "categories" + allowed_methods = ["get"] filtering = { - 'identifier': ALL, + "identifier": ALL, } serializer = CountJSONSerializer() authorization = ApiLockdownAuthorization() @@ -325,17 +319,14 @@ class GroupCategoryResource(TypeFilteredResource): class Meta: queryset = GroupCategory.objects.all() - allowed_methods = ['get'] + allowed_methods = ["get"] include_resource_uri = False - filtering = {'slug': ALL, - 'name': ALL} - ordering = ['name'] + filtering = {"slug": ALL, "name": ALL} + ordering = ["name"] authorization = ApiLockdownAuthorization() def apply_filters(self, request, applicable_filters): - filtered = super().apply_filters( - request, - applicable_filters) + filtered = super().apply_filters(request, applicable_filters) return filtered def dehydrate_detail_url(self, bundle): @@ -346,34 +337,24 @@ def dehydrate_member_count(self, bundle): user = request.user filtered = bundle.obj.groups.all() if not user.is_authenticated or user.is_anonymous: - filtered = filtered.exclude(access='private') + filtered = filtered.exclude(access="private") elif not user.is_superuser: - categories_ids = user.group_list_all().values('categories') - filtered = filtered.filter( - Q(id__in=categories_ids) | - ~Q(access='private') - ) + categories_ids = user.group_list_all().values("categories") + filtered = filtered.filter(Q(id__in=categories_ids) | ~Q(access="private")) return filtered.count() def dehydrate(self, bundle): """Provide additional resource counts""" request = bundle.request counts = _get_resource_counts( - request, - resourcebase_filter_kwargs={ - 'group__groupprofile__categories': bundle.obj - } + request, resourcebase_filter_kwargs={"group__groupprofile__categories": bundle.obj} ) bundle.data.update(resource_counts=counts) return bundle class GroupProfileResource(ModelResource): - categories = fields.ToManyField( - GroupCategoryResource, - 'categories', - full=True - ) + categories = fields.ToManyField(GroupCategoryResource, "categories", full=True) member_count = fields.CharField() manager_count = fields.CharField() logo_url = fields.CharField() @@ -381,14 +362,14 @@ class GroupProfileResource(ModelResource): class Meta: queryset = GroupProfile.objects.all() - resource_name = 'group_profile' - allowed_methods = ['get'] + resource_name = "group_profile" + allowed_methods = ["get"] filtering = { - 'title': ALL, - 'slug': ALL, - 'categories': ALL_WITH_RELATIONS, + "title": ALL, + "slug": ALL, + "categories": ALL_WITH_RELATIONS, } - ordering = ['title', 'last_modified'] + ordering = ["title", "last_modified"] authorization = GroupProfileAuthorization() def dehydrate_member_count(self, bundle): @@ -402,7 +383,7 @@ def dehydrate_manager_count(self, bundle): def dehydrate_detail_url(self, bundle): """Return relative URL to the geonode UI's page on the group""" if bundle.obj.slug: - return reverse('group_detail', args=[bundle.obj.slug]) + return reverse("group_detail", args=[bundle.obj.slug]) else: return None @@ -411,34 +392,25 @@ def dehydrate_logo_url(self, bundle): class GroupResource(ModelResource): - group_profile = fields.ToOneField( - GroupProfileResource, - 'groupprofile', - full=True, - null=True, - blank=True - ) + group_profile = fields.ToOneField(GroupProfileResource, "groupprofile", full=True, null=True, blank=True) resource_counts = fields.CharField() class Meta: queryset = Group.objects.exclude(groupprofile=None) - resource_name = 'groups' - allowed_methods = ['get'] + resource_name = "groups" + allowed_methods = ["get"] filtering = { - 'name': ALL, - 'title': ALL, - 'group_profile': ALL_WITH_RELATIONS, + "name": ALL, + "title": ALL, + "group_profile": ALL_WITH_RELATIONS, } - ordering = ['name', 'last_modified'] + ordering = ["name", "last_modified"] authorization = GroupAuthorization() def dehydrate(self, bundle): """Provide additional resource counts""" request = bundle.request - counts = _get_resource_counts( - request, - resourcebase_filter_kwargs={'group': bundle.obj, 'metadata_only': False} - ) + counts = _get_resource_counts(request, resourcebase_filter_kwargs={"group": bundle.obj, "metadata_only": False}) bundle.data.update(resource_counts=counts) return bundle @@ -455,9 +427,10 @@ def get_object_list(self, request): class ProfileResource(TypeFilteredResource): """Profile api""" + avatar_100 = fields.CharField(null=True) profile_detail_url = fields.CharField() - email = fields.CharField(default='') + email = fields.CharField(default="") layers_count = fields.IntegerField(default=0) maps_count = fields.IntegerField(default=0) documents_count = fields.IntegerField(default=0) @@ -471,31 +444,27 @@ def build_filters(self, filters=None, ignore_bad_filters=False): orm_filters = super().build_filters(filters) - if 'group' in filters: - orm_filters['group'] = filters['group'] + if "group" in filters: + orm_filters["group"] = filters["group"] - if 'name__icontains' in filters: - orm_filters['username__icontains'] = filters['name__icontains'] + if "name__icontains" in filters: + orm_filters["username__icontains"] = filters["name__icontains"] return orm_filters def apply_filters(self, request, applicable_filters): """filter by group if applicable by group functionality""" - group = applicable_filters.pop('group', None) - name = applicable_filters.pop('name__icontains', None) + group = applicable_filters.pop("group", None) + name = applicable_filters.pop("name__icontains", None) - semi_filtered = super().apply_filters( - request, - applicable_filters) + semi_filtered = super().apply_filters(request, applicable_filters) if group is not None: - semi_filtered = semi_filtered.filter( - groupmember__group__slug=group) + semi_filtered = semi_filtered.filter(groupmember__group__slug=group) if name is not None: - semi_filtered = semi_filtered.filter( - profile__first_name__icontains=name) + semi_filtered = semi_filtered.filter(profile__first_name__icontains=name) if request.user and not group and not request.user.is_superuser: semi_filtered = semi_filtered & get_available_users(request.user) @@ -503,29 +472,44 @@ def apply_filters(self, request, applicable_filters): return semi_filtered def dehydrate_email(self, bundle): - email = '' + email = "" if bundle.request.user.is_superuser: email = bundle.obj.email return email def dehydrate_datasets_count(self, bundle): - obj_with_perms = get_objects_for_user(bundle.request.user, - 'base.view_resourcebase').filter(polymorphic_ctype__model='dataset') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ - .distinct().count() + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="dataset" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) def dehydrate_maps_count(self, bundle): - obj_with_perms = get_objects_for_user(bundle.request.user, - 'base.view_resourcebase').filter(polymorphic_ctype__model='map') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ - .distinct().count() + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="map" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) def dehydrate_documents_count(self, bundle): - obj_with_perms = get_objects_for_user(bundle.request.user, - 'base.view_resourcebase').filter(polymorphic_ctype__model='document') - return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\ - .distinct().count() + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="document" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) def dehydrate_avatar_100(self, bundle): return avatar_url(bundle.obj, 240) @@ -538,11 +522,9 @@ def dehydrate_current_user(self, bundle): def dehydrate_activity_stream_url(self, bundle): return reverse( - 'actstream_actor', - kwargs={ - 'content_type_id': ContentType.objects.get_for_model( - bundle.obj).pk, - 'object_id': bundle.obj.pk}) + "actstream_actor", + kwargs={"content_type_id": ContentType.objects.get_for_model(bundle.obj).pk, "object_id": bundle.obj.pk}, + ) def dehydrate(self, bundle): """ @@ -552,25 +534,26 @@ def dehydrate(self, bundle): is_admin = bundle.request.user.is_staff or bundle.request.user.is_superuser if not (is_owner or is_admin): bundle.data = dict( - id=bundle.data.get('id', ''), - username=bundle.data.get('username', ''), - first_name=bundle.data.get('first_name', ''), - last_name=bundle.data.get('last_name', ''), - avatar_100=bundle.data.get('avatar_100', ''), - profile_detail_url=bundle.data.get('profile_detail_url', ''), - documents_count=bundle.data.get('documents_count', 0), - maps_count=bundle.data.get('maps_count', 0), - layers_count=bundle.data.get('layers_count', 0), + id=bundle.data.get("id", ""), + username=bundle.data.get("username", ""), + first_name=bundle.data.get("first_name", ""), + last_name=bundle.data.get("last_name", ""), + avatar_100=bundle.data.get("avatar_100", ""), + profile_detail_url=bundle.data.get("profile_detail_url", ""), + documents_count=bundle.data.get("documents_count", 0), + maps_count=bundle.data.get("maps_count", 0), + layers_count=bundle.data.get("layers_count", 0), ) return bundle def prepend_urls(self): if settings.HAYSTACK_SEARCH: return [ - url(r"^(?P{})/search{}$".format( - self._meta.resource_name, trailing_slash() + url( + r"^(?P{})/search{}$".format(self._meta.resource_name, trailing_slash()), + self.wrap_view("get_search"), + name="api_get_search", ), - self.wrap_view('get_search'), name="api_get_search"), ] else: return [] @@ -578,20 +561,19 @@ def prepend_urls(self): def serialize(self, request, data, format, options=None): if options is None: options = {} - options['count_type'] = 'owner' + options["count_type"] = "owner" return super().serialize(request, data, format, options) class Meta: - queryset = get_user_model().objects.exclude(Q(username='AnonymousUser') | Q(is_active=False)) - resource_name = 'profiles' - allowed_methods = ['get'] - ordering = ['username', 'date_joined'] - excludes = ['is_staff', 'password', 'is_superuser', - 'is_active', 'last_login'] + queryset = get_user_model().objects.exclude(Q(username="AnonymousUser") | Q(is_active=False)) + resource_name = "profiles" + allowed_methods = ["get"] + ordering = ["username", "date_joined"] + excludes = ["is_staff", "password", "is_superuser", "is_active", "last_login"] filtering = { - 'username': ALL, + "username": ALL, } serializer = CountJSONSerializer() authorization = GeoNodePeopleAuthorization() @@ -599,13 +581,14 @@ class Meta: class OwnersResource(TypeFilteredResource): """Owners api, lighter and faster version of the profiles api""" + full_name = fields.CharField(null=True) def dehydrate_full_name(self, bundle): return bundle.obj.get_full_name() or bundle.obj.username def dehydrate_email(self, bundle): - email = '' + email = "" if bundle.request.user.is_superuser: email = bundle.obj.email return email @@ -623,20 +606,19 @@ def dehydrate(self, bundle): def serialize(self, request, data, format, options=None): if options is None: options = {} - options['count_type'] = 'owner' + options["count_type"] = "owner" return super().serialize(request, data, format, options) class Meta: - queryset = get_user_model().objects.exclude(username='AnonymousUser') - resource_name = 'owners' - allowed_methods = ['get'] - ordering = ['username', 'date_joined'] - excludes = ['is_staff', 'password', 'is_superuser', - 'is_active', 'last_login'] + queryset = get_user_model().objects.exclude(username="AnonymousUser") + resource_name = "owners" + allowed_methods = ["get"] + ordering = ["username", "date_joined"] + excludes = ["is_staff", "password", "is_superuser", "is_active", "last_login"] filtering = { - 'username': ALL, + "username": ALL, } serializer = CountJSONSerializer() authorization = ApiLockdownAuthorization() @@ -644,47 +626,36 @@ class Meta: class GeoserverStyleResource(ModelResource): """Styles API for Geoserver backend.""" - body = fields.CharField( - attribute='sld_body', - use_in='detail') - name = fields.CharField(attribute='name') - title = fields.CharField(attribute='sld_title') + + body = fields.CharField(attribute="sld_body", use_in="detail") + name = fields.CharField(attribute="name") + title = fields.CharField(attribute="sld_title") # dataset_default_style is polymorphic, so it will have many to many # relation layer = fields.ManyToManyField( - 'geonode.api.resourcebase_api.LayerResource', - attribute='dataset_default_style', - null=True) - version = fields.CharField( - attribute='sld_version', - null=True, - blank=True) - style_url = fields.CharField(attribute='sld_url') - workspace = fields.CharField(attribute='workspace', null=True) - type = fields.CharField(attribute='type') + "geonode.api.resourcebase_api.LayerResource", attribute="dataset_default_style", null=True + ) + version = fields.CharField(attribute="sld_version", null=True, blank=True) + style_url = fields.CharField(attribute="sld_url") + workspace = fields.CharField(attribute="workspace", null=True) + type = fields.CharField(attribute="type") class Meta: paginator_class = CrossSiteXHRPaginator queryset = Style.objects.all() - resource_name = 'styles' - detail_uri_name = 'id' + resource_name = "styles" + detail_uri_name = "id" authorization = GeoNodeStyleAuthorization() - allowed_methods = ['get'] - filtering = { - 'id': ALL, - 'title': ALL, - 'name': ALL, - 'layer': ALL_WITH_RELATIONS - } + allowed_methods = ["get"] + filtering = {"id": ALL, "title": ALL, "name": ALL, "layer": ALL_WITH_RELATIONS} def build_filters(self, filters=None, **kwargs): """Apply custom filters for layer.""" - filters = super().build_filters( - filters, **kwargs) + filters = super().build_filters(filters, **kwargs) # Convert dataset__ filters into dataset_styles__dataset__ updated_filters = {} for key, value in filters.items(): - key = key.replace('dataset__', 'dataset_default_style__') + key = key.replace("dataset__", "dataset_default_style__") updated_filters[key] = value return updated_filters @@ -695,7 +666,7 @@ def populate_object(self, style): :type style: Style :return: """ - style.type = 'sld' + style.type = "sld" return style def build_bundle(self, obj=None, data=None, request=None, **kwargs): @@ -707,16 +678,14 @@ def build_bundle(self, obj=None, data=None, request=None, **kwargs): elif obj: obj = self.populate_object(obj) - return Bundle( - obj=obj, - data=data, - request=request, - **kwargs) + return Bundle(obj=obj, data=data, request=request, **kwargs) if check_ogc_backend(geoserver.BACKEND_PACKAGE): + class StyleResource(GeoserverStyleResource): """Wrapper for Generic Style Resource""" + pass @@ -744,26 +713,21 @@ def _get_resource_counts(request, resourcebase_filter_kwargs): request=request, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) values = resources.values( - 'polymorphic_ctype__model', - 'is_approved', - 'is_published', + "polymorphic_ctype__model", + "is_approved", + "is_published", ) - qs = values.annotate(counts=Count('polymorphic_ctype__model')) - types = [ - 'dataset', - 'document', - 'map', - 'geoapp', - 'all' - ] + qs = values.annotate(counts=Count("polymorphic_ctype__model")) + types = ["dataset", "document", "map", "geoapp", "all"] subtypes = [] for label, app in apps.app_configs.items(): - if hasattr(app, 'type') and app.type == 'GEONODE_APP': - if hasattr(app, 'default_model'): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): _model = apps.get_model(label, app.default_model) if issubclass(_model, GeoApp): types.append(_model.__name__.lower()) @@ -771,24 +735,24 @@ def _get_resource_counts(request, resourcebase_filter_kwargs): counts = {} for type_ in types: counts[type_] = { - 'total': 0, - 'visible': 0, - 'published': 0, - 'approved': 0, + "total": 0, + "visible": 0, + "published": 0, + "approved": 0, } for record in qs: - resource_type = record['polymorphic_ctype__model'] + resource_type = record["polymorphic_ctype__model"] if resource_type in subtypes: - resource_type = 'geoapp' - is_visible = all((record['is_approved'], record['is_published'])) - counts['all']['total'] += record['counts'] - counts['all']['visible'] += record['counts'] if is_visible else 0 - counts['all']['published'] += record['counts'] if record['is_published'] else 0 - counts['all']['approved'] += record['counts'] if record['is_approved'] else 0 + resource_type = "geoapp" + is_visible = all((record["is_approved"], record["is_published"])) + counts["all"]["total"] += record["counts"] + counts["all"]["visible"] += record["counts"] if is_visible else 0 + counts["all"]["published"] += record["counts"] if record["is_published"] else 0 + counts["all"]["approved"] += record["counts"] if record["is_approved"] else 0 section = counts.get(resource_type) if section is not None: - section['total'] += record['counts'] - section['visible'] += record['counts'] if is_visible else 0 - section['published'] += record['counts'] if record['is_published'] else 0 - section['approved'] += record['counts'] if record['is_approved'] else 0 + section["total"] += record["counts"] + section["visible"] += record["counts"] if is_visible else 0 + section["published"] += record["counts"] if record["is_published"] else 0 + section["approved"] += record["counts"] if record["is_approved"] else 0 return counts diff --git a/geonode/api/authentication.py b/geonode/api/authentication.py index f8f971f9b21..287e3c93737 100644 --- a/geonode/api/authentication.py +++ b/geonode/api/authentication.py @@ -25,12 +25,10 @@ class OAuthAuthentication(Authentication): - def extract_auth_header(self, request): auth_header = None try: - auth_header = request.META.get( - 'HTTP_AUTHORIZATION', request.META.get('HTTP_AUTHORIZATION2')) + auth_header = request.META.get("HTTP_AUTHORIZATION", request.META.get("HTTP_AUTHORIZATION2")) except KeyError: pass return auth_header @@ -47,7 +45,7 @@ def token_is_valid(self, token): def is_authenticated(self, request, **kwargs): user = AnonymousUser() authenticated = False - if 'HTTP_AUTHORIZATION' in request.META: + if "HTTP_AUTHORIZATION" in request.META: auth_header = self.extract_auth_header(request) if auth_header: access_token = get_token_from_auth_header(auth_header) diff --git a/geonode/api/authorization.py b/geonode/api/authorization.py index 92fdd1b4fbf..376d53c8e8b 100644 --- a/geonode/api/authorization.py +++ b/geonode/api/authorization.py @@ -38,47 +38,37 @@ class GeoNodeAuthorization(DjangoAuthorization): def read_list(self, object_list, bundle): permitted_ids = [] try: - permitted_ids = get_objects_for_user( - bundle.request.user, - 'base.view_resourcebase').values('id') + permitted_ids = get_objects_for_user(bundle.request.user, "base.view_resourcebase").values("id") except Exception: pass return object_list.filter(id__in=permitted_ids) def read_detail(self, object_list, bundle): - if 'schema' in bundle.request.path: + if "schema" in bundle.request.path: return True - return bundle.request.user.has_perm( - 'view_resourcebase', - bundle.obj.get_self_resource()) + return bundle.request.user.has_perm("view_resourcebase", bundle.obj.get_self_resource()) def create_list(self, object_list, bundle): # TODO implement if needed raise Unauthorized() def create_detail(self, object_list, bundle): - return bundle.request.user.has_perm( - 'add_resourcebase', - bundle.obj.get_self_resource()) + return bundle.request.user.has_perm("add_resourcebase", bundle.obj.get_self_resource()) def update_list(self, object_list, bundle): # TODO implement if needed raise Unauthorized() def update_detail(self, object_list, bundle): - return bundle.request.user.has_perm( - 'change_resourcebase', - bundle.obj.get_self_resource()) + return bundle.request.user.has_perm("change_resourcebase", bundle.obj.get_self_resource()) def delete_list(self, object_list, bundle): # TODO implement if needed raise Unauthorized() def delete_detail(self, object_list, bundle): - return bundle.request.user.has_perm( - 'delete_resourcebase', - bundle.obj.get_self_resource()) + return bundle.request.user.has_perm("delete_resourcebase", bundle.obj.get_self_resource()) class GeonodeApiKeyAuthentication(ApiKeyAuthentication): @@ -134,16 +124,12 @@ def filter_by_resource_ids(self, object_list, permitted_ids): return object_list.filter(dataset_styles__id__in=permitted_ids) def read_list(self, object_list, bundle): - permitted_ids = get_objects_for_user( - bundle.request.user, - 'base.view_resourcebase').values('id') + permitted_ids = get_objects_for_user(bundle.request.user, "base.view_resourcebase").values("id") return self.filter_by_resource_ids(object_list, permitted_ids) def delete_detail(self, object_list, bundle): - permitted_ids = get_objects_for_user( - bundle.request.user, - 'layer.change_dataset_style').values('id') + permitted_ids = get_objects_for_user(bundle.request.user, "layer.change_dataset_style").values("id") resource_obj = bundle.obj.get_self_resource() return resource_obj in permitted_ids @@ -165,8 +151,7 @@ def read_list(self, object_list, bundle): class GeoNodePeopleAuthorization(DjangoAuthorization): - """API authorization that allows only authenticated users to view list of users - """ + """API authorization that allows only authenticated users to view list of users""" def read_list(self, object_list, bundle): user = bundle.request.user @@ -177,26 +162,24 @@ def read_list(self, object_list, bundle): class GroupAuthorization(ApiLockdownAuthorization): - def read_list(self, object_list, bundle): groups = super().read_list(object_list, bundle) user = bundle.request.user if groups: if not user.is_authenticated or user.is_anonymous: - return groups.exclude(groupprofile__access='private') + return groups.exclude(groupprofile__access="private") elif not user.is_superuser: - return groups.filter(Q(groupprofile__in=user.group_list_all()) | ~Q(groupprofile__access='private')) + return groups.filter(Q(groupprofile__in=user.group_list_all()) | ~Q(groupprofile__access="private")) return groups class GroupProfileAuthorization(ApiLockdownAuthorization): - def read_list(self, object_list, bundle): groups = super().read_list(object_list, bundle) user = bundle.request.user if groups: if not user.is_authenticated or user.is_anonymous: - return groups.exclude(access='private') + return groups.exclude(access="private") elif not user.is_superuser: - return groups.filter(Q(pk__in=user.group_list_all()) | ~Q(access='private')) + return groups.filter(Q(pk__in=user.group_list_all()) | ~Q(access="private")) return groups diff --git a/geonode/api/paginator.py b/geonode/api/paginator.py index 403627d5644..dab5bf844b8 100644 --- a/geonode/api/paginator.py +++ b/geonode/api/paginator.py @@ -23,7 +23,6 @@ class CrossSiteXHRPaginator(Paginator): - def get_limit(self): """ Determines the proper maximum number of results to return. @@ -37,9 +36,9 @@ def get_limit(self): Default is 20 per page. """ - limit = self.request_data.get('limit', self.limit) + limit = self.request_data.get("limit", self.limit) if limit is None: - limit = getattr(settings, 'API_LIMIT_PER_PAGE', 20) + limit = getattr(settings, "API_LIMIT_PER_PAGE", 20) try: limit = int(limit) @@ -67,8 +66,8 @@ def get_offset(self): """ offset = self.offset - if 'offset' in self.request_data: - offset = self.request_data['offset'] + if "offset" in self.request_data: + offset = self.request_data["offset"] try: offset = int(offset) diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 10ea7a1416a..d4134258efe 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -62,7 +62,8 @@ ThesaurusKeywordResource, TopicCategoryResource, GroupResource, - FILTER_TYPES) + FILTER_TYPES, +) from .paginator import CrossSiteXHRPaginator from django.utils.translation import gettext as _ @@ -74,147 +75,133 @@ class CommonMetaApi: authorization = GeoNodeAuthorization() - allowed_methods = ['get'] + allowed_methods = ["get"] filtering = { - 'title': ALL, - 'keywords': ALL_WITH_RELATIONS, - 'tkeywords': ALL_WITH_RELATIONS, - 'regions': ALL_WITH_RELATIONS, - 'category': ALL_WITH_RELATIONS, - 'group': ALL_WITH_RELATIONS, - 'owner': ALL_WITH_RELATIONS, - 'date': ALL, - 'purpose': ALL, - 'uuid': ALL_WITH_RELATIONS, - 'abstract': ALL, - 'metadata': ALL_WITH_RELATIONS + "title": ALL, + "keywords": ALL_WITH_RELATIONS, + "tkeywords": ALL_WITH_RELATIONS, + "regions": ALL_WITH_RELATIONS, + "category": ALL_WITH_RELATIONS, + "group": ALL_WITH_RELATIONS, + "owner": ALL_WITH_RELATIONS, + "date": ALL, + "purpose": ALL, + "uuid": ALL_WITH_RELATIONS, + "abstract": ALL, + "metadata": ALL_WITH_RELATIONS, } - ordering = ['date', 'title', 'popular_count'] + ordering = ["date", "title", "popular_count"] max_limit = None class CommonModelApi(ModelResource): - keywords = fields.ToManyField(TagResource, 'keywords', null=True) - regions = fields.ToManyField(RegionResource, 'regions', null=True) - category = fields.ToOneField( - TopicCategoryResource, - 'category', - null=True, - full=True) - group = fields.ToOneField( - GroupResource, - 'group', - null=True, - full=True) - owner = fields.ToOneField(OwnersResource, 'owner', full=True) - tkeywords = fields.ToManyField( - ThesaurusKeywordResource, 'tkeywords', null=True) + keywords = fields.ToManyField(TagResource, "keywords", null=True) + regions = fields.ToManyField(RegionResource, "regions", null=True) + category = fields.ToOneField(TopicCategoryResource, "category", null=True, full=True) + group = fields.ToOneField(GroupResource, "group", null=True, full=True) + owner = fields.ToOneField(OwnersResource, "owner", full=True) + tkeywords = fields.ToManyField(ThesaurusKeywordResource, "tkeywords", null=True) VALUES = [ # fields in the db - 'id', - 'uuid', - 'name', - 'typename', - 'title', - 'date', - 'date_type', - 'edition', - 'purpose', - 'maintenance_frequency', - 'restriction_code_type', - 'constraints_other', - 'license', - 'language', - 'spatial_representation_type', - 'temporal_extent_start', - 'temporal_extent_end', - 'data_quality_statement', - 'abstract', - 'csw_wkt_geometry', - 'csw_type', - 'owner__username', - 'share_count', - 'popular_count', - 'srid', - 'bbox_polygon', - 'category__gn_description', - 'supplemental_information', - 'site_url', - 'thumbnail_url', - 'detail_url', - 'rating', - 'group__name', - 'has_time', - 'is_approved', - 'is_published', - 'dirty_state', - 'metadata_only' + "id", + "uuid", + "name", + "typename", + "title", + "date", + "date_type", + "edition", + "purpose", + "maintenance_frequency", + "restriction_code_type", + "constraints_other", + "license", + "language", + "spatial_representation_type", + "temporal_extent_start", + "temporal_extent_end", + "data_quality_statement", + "abstract", + "csw_wkt_geometry", + "csw_type", + "owner__username", + "share_count", + "popular_count", + "srid", + "bbox_polygon", + "category__gn_description", + "supplemental_information", + "site_url", + "thumbnail_url", + "detail_url", + "rating", + "group__name", + "has_time", + "is_approved", + "is_published", + "dirty_state", + "metadata_only", ] def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): if filters is None: filters = {} - orm_filters = super().build_filters( - filters=filters, ignore_bad_filters=ignore_bad_filters, **kwargs) - if 'type__in' in filters and (filters['type__in'] in FILTER_TYPES.keys() or filters['type__in'] in LAYER_TYPES): - orm_filters.update({'type': filters.getlist('type__in')}) - if 'app_type__in' in filters: - orm_filters.update({'resource_type': filters['app_type__in'].lower()}) - - _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata__')} + orm_filters = super().build_filters(filters=filters, ignore_bad_filters=ignore_bad_filters, **kwargs) + if "type__in" in filters and (filters["type__in"] in FILTER_TYPES.keys() or filters["type__in"] in LAYER_TYPES): + orm_filters.update({"type": filters.getlist("type__in")}) + if "app_type__in" in filters: + orm_filters.update({"resource_type": filters["app_type__in"].lower()}) + + _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith("metadata__")} if _metadata: orm_filters.update({"metadata_filters": _metadata}) - if 'extent' in filters: - orm_filters.update({'extent': filters['extent']}) - orm_filters['f_method'] = filters['f_method'] if 'f_method' in filters else 'and' + if "extent" in filters: + orm_filters.update({"extent": filters["extent"]}) + orm_filters["f_method"] = filters["f_method"] if "f_method" in filters else "and" if not settings.SEARCH_RESOURCES_EXTENDED: return self._remove_additional_filters(orm_filters) return orm_filters def _remove_additional_filters(self, orm_filters): - orm_filters.pop('abstract__icontains', None) - orm_filters.pop('purpose__icontains', None) - orm_filters.pop('f_method', None) + orm_filters.pop("abstract__icontains", None) + orm_filters.pop("purpose__icontains", None) + orm_filters.pop("f_method", None) return orm_filters def apply_filters(self, request, applicable_filters): - types = applicable_filters.pop('type', None) - extent = applicable_filters.pop('extent', None) - keywords = applicable_filters.pop('keywords__slug__in', None) - metadata_only = applicable_filters.pop('metadata_only', False) - filtering_method = applicable_filters.pop('f_method', 'and') - metadata_filters = applicable_filters.pop('metadata_filters', None) - if filtering_method == 'or': + types = applicable_filters.pop("type", None) + extent = applicable_filters.pop("extent", None) + keywords = applicable_filters.pop("keywords__slug__in", None) + metadata_only = applicable_filters.pop("metadata_only", False) + filtering_method = applicable_filters.pop("f_method", "and") + metadata_filters = applicable_filters.pop("metadata_filters", None) + if filtering_method == "or": filters = Q() for f in applicable_filters.items(): filters |= Q(f) semi_filtered = self.get_object_list(request).filter(filters) else: - semi_filtered = super().apply_filters( - request, - applicable_filters) + semi_filtered = super().apply_filters(request, applicable_filters) filtered = None if types: for the_type in types: if the_type in LAYER_TYPES: super_type = the_type - if 'vector_time' == the_type: - super_type = 'vector' + if "vector_time" == the_type: + super_type = "vector" if filtered: - if 'time' in the_type: - filtered = filtered | semi_filtered.filter( - Layer___subtype=super_type).exclude(Layer___has_time=False) + if "time" in the_type: + filtered = filtered | semi_filtered.filter(Layer___subtype=super_type).exclude( + Layer___has_time=False + ) else: - filtered = filtered | semi_filtered.filter( - Layer___subtype=super_type) + filtered = filtered | semi_filtered.filter(Layer___subtype=super_type) else: - if 'time' in the_type: - filtered = semi_filtered.filter( - Layer___subtype=super_type).exclude(Layer___has_time=False) + if "time" in the_type: + filtered = semi_filtered.filter(Layer___subtype=super_type).exclude(Layer___has_time=False) else: - filtered = semi_filtered.filter( - Layer___subtype=super_type) + filtered = semi_filtered.filter(Layer___subtype=super_type) else: _type_filter = FILTER_TYPES[the_type].__name__.lower() if filtered: @@ -240,15 +227,15 @@ def apply_filters(self, request, applicable_filters): metadata_only=metadata_only, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) def filter_h_keywords(self, queryset, keywords): treeqs = HierarchicalKeyword.objects.none() if keywords and len(keywords) > 0: for keyword in keywords: try: - kws = HierarchicalKeyword.objects.filter( - Q(name__iexact=keyword) | Q(slug__iexact=keyword)) + kws = HierarchicalKeyword.objects.filter(Q(name__iexact=keyword) | Q(slug__iexact=keyword)) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except ObjectDoesNotExist: @@ -268,7 +255,7 @@ def build_haystack_filters(self, parameters): # Retrieve Query Params # Text search - query = parameters.get('q', None) + query = parameters.get("q", None) # Types and subtypes to filter (map, layer, vector, etc) type_facets = parameters.getlist("type__in", []) @@ -312,62 +299,47 @@ def build_haystack_filters(self, parameters): elif type in LAYER_TYPES: subtypes.append(type) - if 'vector' in subtypes and 'vector_time' not in subtypes: - subtypes.append('vector_time') + if "vector" in subtypes and "vector_time" not in subtypes: + subtypes.append("vector_time") if len(subtypes) > 0: types.append("layer") sqs = SearchQuerySet().narrow(f"subtype:{','.join(map(str, subtypes))}") if len(types) > 0: - sqs = (SearchQuerySet() if sqs is None else sqs).narrow( - f"type:{','.join(map(str, types))}") + sqs = (SearchQuerySet() if sqs is None else sqs).narrow(f"type:{','.join(map(str, types))}") # Filter by Query Params # haystack bug? if boosted fields aren't included in the # query, then the score won't be affected by the boost if query: - if query.startswith('"') or query.startswith('\''): + if query.startswith('"') or query.startswith("'"): # Match exact phrase - phrase = query.replace('"', '') + phrase = query.replace('"', "") sqs = (SearchQuerySet() if sqs is None else sqs).filter( - SQ(title__exact=phrase) | - SQ(description__exact=phrase) | - SQ(content__exact=phrase) + SQ(title__exact=phrase) | SQ(description__exact=phrase) | SQ(content__exact=phrase) ) else: - words = [ - w for w in re.split( - r'\W', - query, - flags=re.UNICODE) if w] + words = [w for w in re.split(r"\W", query, flags=re.UNICODE) if w] for i, search_word in enumerate(words): if i == 0: - sqs = (SearchQuerySet() if sqs is None else sqs) \ - .filter( - SQ(title=Raw(search_word)) | - SQ(description=Raw(search_word)) | - SQ(content=Raw(search_word)) + sqs = (SearchQuerySet() if sqs is None else sqs).filter( + SQ(title=Raw(search_word)) | SQ(description=Raw(search_word)) | SQ(content=Raw(search_word)) ) elif search_word in {"AND", "OR"}: pass elif words[i - 1] == "OR": # previous word OR this word sqs = sqs.filter_or( - SQ(title=Raw(search_word)) | - SQ(description=Raw(search_word)) | - SQ(content=Raw(search_word)) + SQ(title=Raw(search_word)) | SQ(description=Raw(search_word)) | SQ(content=Raw(search_word)) ) else: # previous word AND this word sqs = sqs.filter( - SQ(title=Raw(search_word)) | - SQ(description=Raw(search_word)) | - SQ(content=Raw(search_word)) + SQ(title=Raw(search_word)) | SQ(description=Raw(search_word)) | SQ(content=Raw(search_word)) ) # filter by category if category: - sqs = (SearchQuerySet() if sqs is None else sqs).narrow( - f"category:{','.join(map(str, category))}") + sqs = (SearchQuerySet() if sqs is None else sqs).narrow(f"category:{','.join(map(str, category))}") # filter by keyword: use filter_or with keywords_exact # not using exact leads to fuzzy matching and too many results @@ -375,9 +347,7 @@ def build_haystack_filters(self, parameters): # selected if keywords: for keyword in keywords: - sqs = ( - SearchQuerySet() if sqs is None else sqs).filter_or( - keywords_exact=keyword) + sqs = (SearchQuerySet() if sqs is None else sqs).filter_or(keywords_exact=keyword) # filter by regions: use filter_or with regions_exact # not using exact leads to fuzzy matching and too many results @@ -385,62 +355,47 @@ def build_haystack_filters(self, parameters): # selected if regions: for region in regions: - sqs = ( - SearchQuerySet() if sqs is None else sqs).filter_or( - regions_exact__exact=region) + sqs = (SearchQuerySet() if sqs is None else sqs).filter_or(regions_exact__exact=region) # filter by owner if owner: - sqs = ( - SearchQuerySet() if sqs is None else sqs).narrow( - f"owner__username:{','.join(map(str, owner))}") + sqs = (SearchQuerySet() if sqs is None else sqs).narrow(f"owner__username:{','.join(map(str, owner))}") # filter by date if date_start: - sqs = (SearchQuerySet() if sqs is None else sqs).filter( - SQ(date__gte=date_start) - ) + sqs = (SearchQuerySet() if sqs is None else sqs).filter(SQ(date__gte=date_start)) if date_end: - sqs = (SearchQuerySet() if sqs is None else sqs).filter( - SQ(date__lte=date_end) - ) + sqs = (SearchQuerySet() if sqs is None else sqs).filter(SQ(date__lte=date_end)) # Filter by geographic bounding box if bbox: - left, bottom, right, top = bbox.split(',') - sqs = ( - SearchQuerySet() if sqs is None else sqs).exclude( - SQ( - bbox_top__lte=bottom) | SQ( - bbox_bottom__gte=top) | SQ( - bbox_left__gte=right) | SQ( - bbox_right__lte=left)) + left, bottom, right, top = bbox.split(",") + sqs = (SearchQuerySet() if sqs is None else sqs).exclude( + SQ(bbox_top__lte=bottom) + | SQ(bbox_bottom__gte=top) + | SQ(bbox_left__gte=right) + | SQ(bbox_right__lte=left) + ) # Apply sort if sort.lower() == "-date": - sqs = ( - SearchQuerySet() if sqs is None else sqs).order_by("-date") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("-date") elif sort.lower() == "date": - sqs = ( - SearchQuerySet() if sqs is None else sqs).order_by("date") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("date") elif sort.lower() == "title": - sqs = (SearchQuerySet() if sqs is None else sqs).order_by( - "title_sortable") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("title_sortable") elif sort.lower() == "-title": - sqs = (SearchQuerySet() if sqs is None else sqs).order_by( - "-title_sortable") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("-title_sortable") elif sort.lower() == "-popular_count": - sqs = (SearchQuerySet() if sqs is None else sqs).order_by( - "-popular_count") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("-popular_count") else: - sqs = ( - SearchQuerySet() if sqs is None else sqs).order_by("-date") + sqs = (SearchQuerySet() if sqs is None else sqs).order_by("-date") return sqs def get_search(self, request, **kwargs): - self.method_check(request, allowed=['get']) + self.method_check(request, allowed=["get"]) self.is_authenticated(request) self.throttle_check(request) @@ -449,43 +404,47 @@ def get_search(self, request, **kwargs): if not settings.SKIP_PERMS_FILTER: - filter_set = get_objects_for_user( - request.user, 'base.view_resourcebase') + filter_set = get_objects_for_user(request.user, "base.view_resourcebase") filter_set = get_visible_resources( filter_set, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) - filter_set_ids = filter_set.values_list('id') + filter_set_ids = filter_set.values_list("id") # Do the query using the filterset and the query term. Facet the # results if len(filter_set) > 0: - sqs = sqs.filter(id__in=filter_set_ids).facet('type').facet('subtype').facet( - 'owner') .facet('keywords').facet('regions').facet('category') + sqs = ( + sqs.filter(id__in=filter_set_ids) + .facet("type") + .facet("subtype") + .facet("owner") + .facet("keywords") + .facet("regions") + .facet("category") + ) else: sqs = None else: - sqs = sqs.facet('type').facet('subtype').facet( - 'owner').facet('keywords').facet('regions').facet('category') + sqs = sqs.facet("type").facet("subtype").facet("owner").facet("keywords").facet("regions").facet("category") if sqs: # Build the Facet dict facets = {} - for facet in sqs.facet_counts()['fields']: + for facet in sqs.facet_counts()["fields"]: facets[facet] = {} - for item in sqs.facet_counts()['fields'][facet]: + for item in sqs.facet_counts()["fields"][facet]: facets[facet][item[0]] = item[1] # Paginate the results - paginator = Paginator(sqs, request.GET.get('limit')) + paginator = Paginator(sqs, request.GET.get("limit")) try: - page = paginator.page( - int(request.GET.get('offset') or 0) / - int(request.GET.get('limit') or 0 + 1)) + page = paginator.page(int(request.GET.get("offset") or 0) / int(request.GET.get("limit") or 0 + 1)) except InvalidPage: raise Http404("Sorry, no results on that page.") @@ -510,7 +469,7 @@ def get_search(self, request, **kwargs): "meta": { "limit": settings.CLIENT_RESULTS_LIMIT, "next": next_page, - "offset": int(getattr(request.GET, 'offset', 0)), + "offset": int(getattr(request.GET, "offset", 0)), "previous": previous_page, "total_count": total_count, "facets": facets, @@ -522,8 +481,7 @@ def get_search(self, request, **kwargs): return self.create_response(request, object_list) def get_haystack_api_fields(self, haystack_object): - return {k: v for k, v in haystack_object.get_stored_fields().items() - if not re.search('_exact$|_sortable$', k)} + return {k: v for k, v in haystack_object.get_stored_fields().items() if not re.search("_exact$|_sortable$", k)} def get_list(self, request, **kwargs): """ @@ -537,9 +495,7 @@ def get_list(self, request, **kwargs): # TODO: Uncached for now. Invalidation that works for everyone may be # impossible. base_bundle = self.build_bundle(request=request) - objects = self.obj_get_list( - bundle=base_bundle, - **self.remove_api_resource_names(kwargs)) + objects = self.obj_get_list(bundle=base_bundle, **self.remove_api_resource_names(kwargs)) sorted_objects = self.apply_sorting(objects, options=request.GET) paginator = self._meta.paginator_class( @@ -548,21 +504,19 @@ def get_list(self, request, **kwargs): resource_uri=self.get_resource_uri(), limit=self._meta.limit, max_limit=self._meta.max_limit, - collection_name=self._meta.collection_name) + collection_name=self._meta.collection_name, + ) to_be_serialized = paginator.page() - to_be_serialized = self.alter_list_data_to_serialize( - request, - to_be_serialized) + to_be_serialized = self.alter_list_data_to_serialize(request, to_be_serialized) - return self.create_response( - request, to_be_serialized, response_objects=objects) + return self.create_response(request, to_be_serialized, response_objects=objects) def format_objects(self, objects): """ Format the objects for output in a response. """ - for key in ('site_url', 'has_time'): + for key in ("site_url", "has_time"): if key in self.VALUES: idx = self.VALUES.index(key) del self.VALUES[idx] @@ -571,26 +525,20 @@ def format_objects(self, objects): formatted_objects = [] for obj in objects: formatted_obj = model_to_dict(obj, fields=self.VALUES) - if 'site_url' not in formatted_obj or len(formatted_obj['site_url']) == 0: - formatted_obj['site_url'] = settings.SITEURL + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL - formatted_obj['owner__username'] = obj.owner.username - formatted_obj['owner_name'] = obj.owner.get_full_name() or obj.owner.username + formatted_obj["owner__username"] = obj.owner.username + formatted_obj["owner_name"] = obj.owner.get_full_name() or obj.owner.username - if formatted_obj.get('metadata', None): - formatted_obj['metadata'] = [model_to_dict(_m) for _m in formatted_obj['metadata']] + if formatted_obj.get("metadata", None): + formatted_obj["metadata"] = [model_to_dict(_m) for _m in formatted_obj["metadata"]] formatted_objects.append(formatted_obj) return formatted_objects - def create_response( - self, - request, - data, - response_class=HttpResponse, - response_objects=None, - **response_kwargs): + def create_response(self, request, data, response_class=HttpResponse, response_objects=None, **response_kwargs): """ Extracts the common "which-format/serialize/return-response" cycle. @@ -601,44 +549,39 @@ def create_response( # to see the resource at all. filtered_objects_ids = None try: - if data['objects']: + if data["objects"]: filtered_objects_ids = [ - item.id for item in data['objects'] if request.user.has_perm( - 'view_resourcebase', item.get_self_resource())] + item.id + for item in data["objects"] + if request.user.has_perm("view_resourcebase", item.get_self_resource()) + ] except Exception: pass - if isinstance( - data, - dict) and 'objects' in data and not isinstance( - data['objects'], - list): + if isinstance(data, dict) and "objects" in data and not isinstance(data["objects"], list): if filtered_objects_ids: - data['objects'] = [ - x for x in list( - self.format_objects( - data['objects'])) if x['id'] in filtered_objects_ids] + data["objects"] = [ + x for x in list(self.format_objects(data["objects"])) if x["id"] in filtered_objects_ids + ] else: - data['objects'] = list(self.format_objects(data['objects'])) + data["objects"] = list(self.format_objects(data["objects"])) # give geonode version - data['geonode_version'] = get_version() + data["geonode_version"] = get_version() desired_format = self.determine_format(request) serialized = self.serialize(request, data, desired_format) - return response_class( - content=serialized, - content_type=build_content_type(desired_format), - **response_kwargs) + return response_class(content=serialized, content_type=build_content_type(desired_format), **response_kwargs) def prepend_urls(self): if settings.HAYSTACK_SEARCH: return [ - url(r"^(?P{})/search{}$".format( - self._meta.resource_name, trailing_slash() + url( + r"^(?P{})/search{}$".format(self._meta.resource_name, trailing_slash()), + self.wrap_view("get_search"), + name="api_get_search", ), - self.wrap_view('get_search'), name="api_get_search"), ] else: return [] @@ -656,13 +599,12 @@ class ResourceBaseResource(CommonModelApi): class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator - queryset = ResourceBase.objects.polymorphic_queryset() \ - .distinct().order_by('-date') - resource_name = 'base' - excludes = ['csw_anytext', 'metadata_xml'] - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + queryset = ResourceBase.objects.polymorphic_queryset().distinct().order_by("-date") + resource_name = "base" + excludes = ["csw_anytext", "metadata_xml"] + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) class FeaturedResourceBaseResource(CommonModelApi): @@ -671,37 +613,27 @@ class FeaturedResourceBaseResource(CommonModelApi): class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator - queryset = ResourceBase.objects.filter(featured=True).order_by('-date') - resource_name = 'featured' - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + queryset = ResourceBase.objects.filter(featured=True).order_by("-date") + resource_name = "featured" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) class LayerResource(CommonModelApi): """Dataset API""" - links = fields.ListField( - attribute='links', - null=True, - use_in='all', - default=[]) + + links = fields.ListField(attribute="links", null=True, use_in="all", default=[]) if check_ogc_backend(geoserver.BACKEND_PACKAGE): - default_style = fields.ForeignKey( - 'geonode.api.api.StyleResource', - attribute='default_style', - null=True) - styles = fields.ManyToManyField( - 'geonode.api.api.StyleResource', - attribute='styles', - null=True, - use_in='detail') + default_style = fields.ForeignKey("geonode.api.api.StyleResource", attribute="default_style", null=True) + styles = fields.ManyToManyField("geonode.api.api.StyleResource", attribute="styles", null=True, use_in="detail") def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() - metadata_only = _filters.pop('metadata_only', False) + metadata_only = _filters.pop("metadata_only", False) orm_filters = super().build_filters(_filters) - orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] return orm_filters def format_objects(self, objects): @@ -712,54 +644,50 @@ def format_objects(self, objects): for obj in objects: # convert the object to a dict using the standard values. # includes other values - values = self.VALUES + [ - 'alternate', - 'name' - ] + values = self.VALUES + ["alternate", "name"] formatted_obj = model_to_dict(obj, fields=values) username = obj.owner.get_username() - full_name = (obj.owner.get_full_name() or username) - formatted_obj['owner__username'] = username - formatted_obj['owner_name'] = full_name + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name if obj.category: - formatted_obj['category__gn_description'] = _(obj.category.gn_description) + formatted_obj["category__gn_description"] = _(obj.category.gn_description) if obj.group: - formatted_obj['group'] = obj.group + formatted_obj["group"] = obj.group try: - formatted_obj['group_name'] = GroupProfile.objects.get(slug=obj.group.name) + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) except GroupProfile.DoesNotExist: - formatted_obj['group_name'] = obj.group + formatted_obj["group_name"] = obj.group - formatted_obj['keywords'] = [k.name for k in obj.keywords.all()] if obj.keywords else [] - formatted_obj['regions'] = [r.name for r in obj.regions.all()] if obj.regions else [] + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] # provide style information bundle = self.build_bundle(obj=obj) - formatted_obj['default_style'] = self.default_style.dehydrate( - bundle, for_list=True) + formatted_obj["default_style"] = self.default_style.dehydrate(bundle, for_list=True) # Add resource uri - formatted_obj['resource_uri'] = self.get_resource_uri(bundle) + formatted_obj["resource_uri"] = self.get_resource_uri(bundle) - formatted_obj['links'] = self.dehydrate_ogc_links(bundle) + formatted_obj["links"] = self.dehydrate_ogc_links(bundle) - if 'site_url' not in formatted_obj or len(formatted_obj['site_url']) == 0: - formatted_obj['site_url'] = settings.SITEURL + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL # Probe Remote Services - formatted_obj['store_type'] = 'dataset' - formatted_obj['online'] = True - if hasattr(obj, 'subtype'): - formatted_obj['store_type'] = obj.subtype - if obj.subtype in ['tileStore', 'remote'] and hasattr(obj, 'remote_service'): + formatted_obj["store_type"] = "dataset" + formatted_obj["online"] = True + if hasattr(obj, "subtype"): + formatted_obj["store_type"] = obj.subtype + if obj.subtype in ["tileStore", "remote"] and hasattr(obj, "remote_service"): if obj.remote_service: - formatted_obj['online'] = (obj.remote_service.probe == 200) + formatted_obj["online"] = obj.remote_service.probe == 200 else: - formatted_obj['online'] = False + formatted_obj["online"] = False - formatted_obj['gtype'] = self.dehydrate_gtype(bundle) + formatted_obj["gtype"] = self.dehydrate_gtype(bundle) - formatted_obj['processed'] = obj.instance_is_processed + formatted_obj["processed"] = obj.instance_is_processed # put the object on the response stack formatted_objects.append(formatted_obj) return formatted_objects @@ -769,13 +697,7 @@ def _dehydrate_links(self, bundle, link_types=None): dehydrated = [] obj = bundle.obj - link_fields = [ - 'extension', - 'link_type', - 'name', - 'mime', - 'url' - ] + link_fields = ["extension", "link_type", "name", "mime", "url"] links = obj.link_set.all() if link_types: @@ -790,13 +712,12 @@ def dehydrate_links(self, bundle): return self._dehydrate_links(bundle) def dehydrate_ogc_links(self, bundle): - return self._dehydrate_links(bundle, ['OGC:WMS', 'OGC:WFS', 'OGC:WCS']) + return self._dehydrate_links(bundle, ["OGC:WMS", "OGC:WFS", "OGC:WCS"]) def dehydrate_gtype(self, bundle): return bundle.obj.gtype - def build_bundle( - self, obj=None, data=None, request=None, **kwargs): + def build_bundle(self, obj=None, data=None, request=None, **kwargs): """Override build_bundle method to add additional info.""" if obj is None and self._meta.object_class: @@ -804,10 +725,7 @@ def build_bundle( elif obj: obj = self.populate_object(obj) - return Bundle( - obj=obj, - data=data, - request=request, **kwargs) + return Bundle(obj=obj, data=data, request=request, **kwargs) def populate_object(self, obj): """Populate results with necessary fields @@ -820,27 +738,22 @@ def populate_object(self, obj): # copy parent attribute before modifying VALUES = CommonModelApi.VALUES[:] - VALUES.append('typename') + VALUES.append("typename") class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator - queryset = Dataset.objects.distinct().order_by('-date') - resource_name = 'datasets' - detail_uri_name = 'id' + queryset = Dataset.objects.distinct().order_by("-date") + resource_name = "datasets" + detail_uri_name = "id" include_resource_uri = True - allowed_methods = ['get', 'patch'] - excludes = ['csw_anytext', 'metadata_xml'] - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + allowed_methods = ["get", "patch"] + excludes = ["csw_anytext", "metadata_xml"] + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) filtering = CommonMetaApi.filtering # Allow filtering using ID - filtering.update({ - 'id': ALL, - 'name': ALL, - 'alternate': ALL, - 'metadata_only': ALL - }) + filtering.update({"id": ALL, "name": ALL, "alternate": ALL, "metadata_only": ALL}) class MapResource(CommonModelApi): @@ -849,9 +762,9 @@ class MapResource(CommonModelApi): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() - metadata_only = _filters.pop('metadata_only', False) + metadata_only = _filters.pop("metadata_only", False) orm_filters = super().build_filters(_filters) - orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] return orm_filters def format_objects(self, objects): @@ -866,53 +779,47 @@ def format_objects(self, objects): # convert the object to a dict using the standard values. formatted_obj = model_to_dict(obj, fields=self.VALUES) username = obj.owner.get_username() - full_name = (obj.owner.get_full_name() or username) - formatted_obj['owner__username'] = username - formatted_obj['owner_name'] = full_name + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name if obj.category: - formatted_obj['category__gn_description'] = _(obj.category.gn_description) + formatted_obj["category__gn_description"] = _(obj.category.gn_description) if obj.group: - formatted_obj['group'] = obj.group + formatted_obj["group"] = obj.group try: - formatted_obj['group_name'] = GroupProfile.objects.get(slug=obj.group.name) + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) except GroupProfile.DoesNotExist: - formatted_obj['group_name'] = obj.group + formatted_obj["group_name"] = obj.group - formatted_obj['keywords'] = [k.name for k in obj.keywords.all()] if obj.keywords else [] - formatted_obj['regions'] = [r.name for r in obj.regions.all()] if obj.regions else [] + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] - if 'site_url' not in formatted_obj or len(formatted_obj['site_url']) == 0: - formatted_obj['site_url'] = settings.SITEURL + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL # Probe Remote Services - formatted_obj['store_type'] = 'map' - formatted_obj['online'] = True + formatted_obj["store_type"] = "map" + formatted_obj["online"] = True # get map layers map_datasets = obj.maplayers formatted_datasets = [] - map_dataset_fields = [ - 'id', - 'name', - 'ows_url', - 'local' - ] + map_dataset_fields = ["id", "name", "ows_url", "local"] for layer in map_datasets.iterator(): - formatted_map_dataset = model_to_dict( - layer, fields=map_dataset_fields) + formatted_map_dataset = model_to_dict(layer, fields=map_dataset_fields) formatted_datasets.append(formatted_map_dataset) - formatted_obj['layers'] = formatted_datasets + formatted_obj["layers"] = formatted_datasets formatted_objects.append(formatted_obj) return formatted_objects class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator - queryset = Map.objects.distinct().order_by('-date') - resource_name = 'maps' - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + queryset = Map.objects.distinct().order_by("-date") + resource_name = "maps" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) class GeoAppResource(CommonModelApi): @@ -931,27 +838,27 @@ def format_objects(self, objects): # convert the object to a dict using the standard values. formatted_obj = model_to_dict(obj, fields=self.VALUES) username = obj.owner.get_username() - full_name = (obj.owner.get_full_name() or username) - formatted_obj['owner__username'] = username - formatted_obj['owner_name'] = full_name + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name if obj.category: - formatted_obj['category__gn_description'] = obj.category.gn_description + formatted_obj["category__gn_description"] = obj.category.gn_description if obj.group: - formatted_obj['group'] = obj.group + formatted_obj["group"] = obj.group try: - formatted_obj['group_name'] = GroupProfile.objects.get(slug=obj.group.name) + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) except GroupProfile.DoesNotExist: - formatted_obj['group_name'] = obj.group + formatted_obj["group_name"] = obj.group - formatted_obj['keywords'] = [k.name for k in obj.keywords.all()] if obj.keywords else [] - formatted_obj['regions'] = [r.name for r in obj.regions.all()] if obj.regions else [] + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] - if 'site_url' not in formatted_obj or len(formatted_obj['site_url']) == 0: - formatted_obj['site_url'] = settings.SITEURL + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL # Probe Remote Services - formatted_obj['store_type'] = 'geoapp' - formatted_obj['online'] = True + formatted_obj["store_type"] = "geoapp" + formatted_obj["online"] = True formatted_objects.append(formatted_obj) return formatted_objects @@ -959,12 +866,12 @@ def format_objects(self, objects): class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator filtering = CommonMetaApi.filtering - filtering.update({'app_type': ALL}) - queryset = GeoApp.objects.distinct().order_by('-date') - resource_name = 'geoapps' - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + filtering.update({"app_type": ALL}) + queryset = GeoApp.objects.distinct().order_by("-date") + resource_name = "geoapps" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) class DocumentResource(CommonModelApi): @@ -973,9 +880,9 @@ class DocumentResource(CommonModelApi): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() - metadata_only = _filters.pop('metadata_only', False) + metadata_only = _filters.pop("metadata_only", False) orm_filters = super().build_filters(_filters) - orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] return orm_filters def format_objects(self, objects): @@ -990,27 +897,27 @@ def format_objects(self, objects): # convert the object to a dict using the standard values. formatted_obj = model_to_dict(obj, fields=self.VALUES) username = obj.owner.get_username() - full_name = (obj.owner.get_full_name() or username) - formatted_obj['owner__username'] = username - formatted_obj['owner_name'] = full_name + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name if obj.category: - formatted_obj['category__gn_description'] = _(obj.category.gn_description) + formatted_obj["category__gn_description"] = _(obj.category.gn_description) if obj.group: - formatted_obj['group'] = obj.group + formatted_obj["group"] = obj.group try: - formatted_obj['group_name'] = GroupProfile.objects.get(slug=obj.group.name) + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) except GroupProfile.DoesNotExist: - formatted_obj['group_name'] = obj.group + formatted_obj["group_name"] = obj.group - formatted_obj['keywords'] = [k.name for k in obj.keywords.all()] if obj.keywords else [] - formatted_obj['regions'] = [r.name for r in obj.regions.all()] if obj.regions else [] + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] - if 'site_url' not in formatted_obj or len(formatted_obj['site_url']) == 0: - formatted_obj['site_url'] = settings.SITEURL + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL # Probe Remote Services - formatted_obj['store_type'] = 'dataset' - formatted_obj['online'] = True + formatted_obj["store_type"] = "dataset" + formatted_obj["online"] = True formatted_objects.append(formatted_obj) return formatted_objects @@ -1018,9 +925,9 @@ def format_objects(self, objects): class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator filtering = CommonMetaApi.filtering - filtering.update({'subtype': ALL}) - queryset = Document.objects.distinct().order_by('-date') - resource_name = 'documents' - authentication = MultiAuthentication(SessionAuthentication(), - OAuthAuthentication(), - GeonodeApiKeyAuthentication()) + filtering.update({"subtype": ALL}) + queryset = Document.objects.distinct().order_by("-date") + resource_name = "documents" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 56ebc396da8..919c3d1cc8c 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -39,14 +39,10 @@ from geonode.groups.models import GroupProfile from geonode.base.auth import get_or_create_token from geonode.tests.base import GeoNodeBaseTestSupport -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models class PermissionsApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - @classmethod def setUpClass(cls): super().setUpClass() @@ -60,8 +56,8 @@ def tearDownClass(cls): def setUp(self): super().setUp() - self.user = 'admin' - self.passwd = 'admin' + self.user = "admin" + self.passwd = "admin" self.perm_spec = {"users": {}, "groups": {}} def test_dataset_get_list_unauth_all_public(self): @@ -69,43 +65,31 @@ def test_dataset_get_list_unauth_all_public(self): Test that the correct number of layers are returned when the client is not logged in and all are public """ - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 8) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) def test_datasets_get_list_unauth_some_public(self): """ Test that if a layer is not public then not all are returned when the client is not logged in """ - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) layer = Dataset.objects.first() layer.set_permissions(self.perm_spec) resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) def test_datasets_get_list_auth_some_public(self): """ Test that if a layer is not public then all are returned if the client is not logged in """ - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) self.api_client.client.login(username=self.user, password=self.passwd) layer = Dataset.objects.first() @@ -113,32 +97,28 @@ def test_datasets_get_list_auth_some_public(self): resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 8) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) def test_dataset_get_list_dataset_private_to_one_user(self): """ Test that if a layer is only visible by admin, then does not appear in the unauthenticated list nor in the list when logged is as bobby """ - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) - perm_spec = {"users": {"admin": ['view_resourcebase']}, "groups": {}} + perm_spec = {"users": {"admin": ["view_resourcebase"]}, "groups": {}} layer = Dataset.objects.first() layer.set_permissions(perm_spec) resp = self.api_client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) self.api_client.client.login(username=self.user, password=self.passwd) resp = self.api_client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 8) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) layer.is_published = False layer.save() @@ -146,31 +126,26 @@ def test_dataset_get_list_dataset_private_to_one_user(self): # with resource publishing with self.settings(RESOURCE_PUBLISHING=True): resp = self.api_client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) self.api_client.client.login(username=self.user, password=self.passwd) resp = self.api_client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) def test_dataset_get_detail_unauth_dataset_not_public(self): """ Test that layer detail gives 404 when not public and not logged in """ - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) layer = Dataset.objects.first() layer.set_permissions(self.perm_spec) layer.clear_dirty_state() - self.assertHttpNotFound(self.api_client.get( - f"{list_url + str(layer.id)}/")) + self.assertHttpNotFound(self.api_client.get(f"{list_url + str(layer.id)}/")) self.api_client.client.login(username=self.user, password=self.passwd) resp = self.api_client.get(f"{list_url + str(layer.id)}/") @@ -180,53 +155,56 @@ def test_dataset_get_detail_unauth_dataset_not_public(self): with self.settings(DELAYED_SECURITY_SIGNALS=True): if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.security import sync_geofence_with_guardian + sync_geofence_with_guardian(layer, self.perm_spec) self.assertTrue(layer.dirty_state) self.client.login(username=self.user, password=self.passwd) resp = self.client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) self.client.logout() resp = self.client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) from django.contrib.auth import get_user_model + get_user_model().objects.create( - username='imnew', - password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ - 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') - self.client.login(username='imnew', password='thepwd') + username="imnew", + password="pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ + 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=", + ) + self.client.login(username="imnew", password="thepwd") resp = self.client.get(list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) def test_new_user_has_access_to_old_datasets(self): """Test that a new user can access the public available layers""" from django.contrib.auth import get_user_model + get_user_model().objects.create( - username='imnew', - password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ - 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') + username="imnew", + password="pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ + 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=", + ) - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) - self.api_client.client.login(username='imnew', password='thepwd') + self.api_client.client.login(username="imnew", password="thepwd") resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) # with delayed security if check_ogc_backend(geoserver.BACKEND_PACKAGE): _ogc_geofence_enabled = settings.OGC_SERVER try: - _ogc_geofence_enabled['default']['GEOFENCE_SECURITY_ENABLED'] = True - with self.settings(DELAYED_SECURITY_SIGNALS=True, - OGC_SERVER=_ogc_geofence_enabled, - DEFAULT_ANONYMOUS_VIEW_PERMISSION=True): + _ogc_geofence_enabled["default"]["GEOFENCE_SECURITY_ENABLED"] = True + with self.settings( + DELAYED_SECURITY_SIGNALS=True, + OGC_SERVER=_ogc_geofence_enabled, + DEFAULT_ANONYMOUS_VIEW_PERMISSION=True, + ): layer = Dataset.objects.first() layer.set_default_permissions() layer.refresh_from_db() @@ -234,55 +212,47 @@ def test_new_user_has_access_to_old_datasets(self): self.client.login(username=self.user, password=self.passwd) resp = self.client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) self.client.logout() resp = self.client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) - self.client.login(username='imnew', password='thepwd') + self.client.login(username="imnew", password="thepwd") resp = self.client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) finally: - _ogc_geofence_enabled['default']['GEOFENCE_SECURITY_ENABLED'] = False + _ogc_geofence_enabled["default"]["GEOFENCE_SECURITY_ENABLED"] = False @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_outh_token(self): - user = 'admin' + user = "admin" _user = get_user_model().objects.get(username=user) token = get_or_create_token(_user) - auth_header = f'Bearer {token}' - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + auth_header = f"Bearer {token}" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) with self.settings(SESSION_EXPIRED_CONTROL_ENABLED=False, DELAYED_SECURITY_SIGNALS=False): # all public resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) - perm_spec = {"users": {"admin": ['view_resourcebase']}, "groups": {}} + perm_spec = {"users": {"admin": ["view_resourcebase"]}, "groups": {}} layer = Dataset.objects.first() layer.set_permissions(perm_spec) resp = self.api_client.get(list_url) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) resp = self.api_client.get(list_url, authentication=auth_header) - self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) layer.is_published = False layer.save() @override_settings(API_LOCKDOWN=True) def test_api_lockdown_false(self): - profiles_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'profiles'}) + profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) # test if results are returned for anonymous users if API_LOCKDOWN is set to False in settings filter_url = profiles_list_url @@ -291,126 +261,106 @@ def test_api_lockdown_false(self): # anonymous resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) # admin - self.api_client.client.login(username='admin', password='admin') + self.api_client.client.login(username="admin", password="admin") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 9) + self.assertEqual(len(self.deserialize(resp)["objects"]), 9) @override_settings(API_LOCKDOWN=True) def test_profiles_lockdown(self): - profiles_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'profiles'}) + profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) filter_url = profiles_list_url resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 6) + self.assertEqual(len(self.deserialize(resp)["objects"]), 6) # Returns limitted info about other users - bobby = get_user_model().objects.get(username='bobby') - profiles = self.deserialize(resp)['objects'] + bobby = get_user_model().objects.get(username="bobby") + profiles = self.deserialize(resp)["objects"] for profile in profiles: - if profile['username'] == 'bobby': - self.assertEquals(profile.get('email'), bobby.email) + if profile["username"] == "bobby": + self.assertEquals(profile.get("email"), bobby.email) else: - self.assertIsNone(profile.get('email')) + self.assertIsNone(profile.get("email")) @override_settings(API_LOCKDOWN=True) def test_owners_lockdown(self): - owners_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'owners'}) + owners_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "owners"}) filter_url = owners_list_url resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 9) + self.assertEqual(len(self.deserialize(resp)["objects"]), 9) # Returns limitted info about other users - bobby = get_user_model().objects.get(username='bobby') - owners = self.deserialize(resp)['objects'] + bobby = get_user_model().objects.get(username="bobby") + owners = self.deserialize(resp)["objects"] for owner in owners: - if owner['username'] == 'bobby': - self.assertEquals(owner.get('email'), bobby.email) + if owner["username"] == "bobby": + self.assertEquals(owner.get("email"), bobby.email) else: - self.assertIsNone(owner.get('email')) - self.assertIsNone(owner.get('first_name')) + self.assertIsNone(owner.get("email")) + self.assertIsNone(owner.get("first_name")) @override_settings(API_LOCKDOWN=True) def test_groups_lockdown(self): - groups_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'groups'}) + groups_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) filter_url = groups_list_url resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) @override_settings(API_LOCKDOWN=True) def test_regions_lockdown(self): - region_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'regions'}) + region_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "regions"}) filter_url = region_list_url resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertTrue(len(self.deserialize(resp)['objects']) >= 200) + self.assertTrue(len(self.deserialize(resp)["objects"]) >= 200) @override_settings(API_LOCKDOWN=True) def test_tags_lockdown(self): - tag_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'keywords'}) + tag_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "keywords"}) filter_url = tag_list_url resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) - self.api_client.client.login(username='bobby', password='bob') + self.api_client.client.login(username="bobby", password="bob") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 5) + self.assertEqual(len(self.deserialize(resp)["objects"]), 5) class SearchApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): @@ -418,12 +368,7 @@ class SearchApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): """Test the search""" # loading test thesausuri and initial data - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json', - "test_thesaurus.json" - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json", "test_thesaurus.json"] @classmethod def setUpClass(cls): @@ -440,21 +385,13 @@ def setUp(self): super().setUp() self.norman = get_user_model().objects.get(username="norman") - self.norman.groups.add(Group.objects.get(name='anonymous')) - self.test_user = get_user_model().objects.get(username='test_user') - self.test_user.groups.add(Group.objects.get(name='anonymous')) - self.bar = GroupProfile.objects.get(slug='bar') + self.norman.groups.add(Group.objects.get(name="anonymous")) + self.test_user = get_user_model().objects.get(username="test_user") + self.test_user.groups.add(Group.objects.get(name="anonymous")) + self.bar = GroupProfile.objects.get(slug="bar") self.anonymous_user = get_anonymous_user() - self.profiles_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'profiles'}) - self.groups_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'groups'}) + self.profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) + self.groups_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) def test_profiles_filters(self): """Test profiles filtering""" @@ -464,30 +401,30 @@ def test_profiles_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) filter_url = f"{self.profiles_list_url}?name__icontains=norm" # Anonymous resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) - self.api_client.client.login(username='admin', password='admin') + self.api_client.client.login(username="admin", password="admin") resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{self.profiles_list_url}?name__icontains=NoRmAN" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{self.profiles_list_url}?name__icontains=bar" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) def test_groups_filters(self): """Test groups filtering""" @@ -498,33 +435,29 @@ def test_groups_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{self.groups_list_url}?name__icontains=bar" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{self.groups_list_url}?name__icontains=BaR" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{self.groups_list_url}?name__icontains=foo" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) def test_category_filters(self): """Test category filtering""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) # check we get the correct layers number returnered filtering on one # and then two different categories @@ -532,13 +465,13 @@ def test_category_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 3) + self.assertEqual(len(self.deserialize(resp)["objects"]), 3) filter_url = f"{list_url}?category__identifier__in=location&category__identifier__in=biota" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 5) + self.assertEqual(len(self.deserialize(resp)["objects"]), 5) def test_metadata_filters(self): """Test category filtering""" @@ -551,15 +484,11 @@ def test_metadata_filters(self): "help_text": "this is the help text-updated", "field_type": "str-updated", "value": "my value-updated", - "category": "category" - } + "category": "category", + }, ) - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) _r.metadata.add(_m) # check we get the correct layers number returnered filtering on one # and then two different categories @@ -567,21 +496,17 @@ def test_metadata_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{list_url}?metadata__category=not-existing-category" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) def test_tag_filters(self): """Test keywords filtering""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) # check we get the correct layers number returnered filtering on one # and then two different keywords @@ -589,21 +514,17 @@ def test_tag_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{list_url}?keywords__slug__in=layertagunique&keywords__slug__in=populartag" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 8) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) def test_owner_filters(self): """Test owner filtering""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) # check we get the correct layers number returnered filtering on one # and then two different owners @@ -611,21 +532,17 @@ def test_owner_filters(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) filter_url = f"{list_url}?owner__username__in=user1&owner__username__in=foo" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 2) + self.assertEqual(len(self.deserialize(resp)["objects"]), 2) def test_title_filter(self): """Test title filtering""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) # check we get the correct layers number returnered filtering on the # title @@ -633,21 +550,17 @@ def test_title_filter(self): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) def test_date_filter(self): """Test date filtering""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) # check we get the correct layers number returnered filtering on the # dates step = timedelta(days=60) now = datetime.now() - fstring = '%Y-%m-%d' + fstring = "%Y-%m-%d" def to_date(val): return val.date().strftime(fstring) @@ -657,59 +570,49 @@ def to_date(val): resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) d3 = to_date(now - (3 * step)) filter_url = f"{list_url}?date__gte={d3}" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 3) + self.assertEqual(len(self.deserialize(resp)["objects"]), 3) d4 = to_date(now - (4 * step)) filter_url = f"{list_url}?date__range={d4},{to_date(now)}" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 4) + self.assertEqual(len(self.deserialize(resp)["objects"]), 4) def test_extended_text_filter(self): """Test that the extended text filter works as expected""" - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) - filter_url = f"{list_url}?title__icontains=layer2&abstract__icontains=layer2&purpose__icontains=layer2&f_method=or" + filter_url = ( + f"{list_url}?title__icontains=layer2&abstract__icontains=layer2&purpose__icontains=layer2&f_method=or" + ) resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 1) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) def test_the_api_should_return_all_datasets_with_metadata_false(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) user = get_user_model().objects.get(username="admin") token = get_or_create_token(user) - auth_header = f'Bearer {token}' + auth_header = f"Bearer {token}" resp = self.api_client.get(list_url, authentication=auth_header) self.assertValidJSONResponse(resp) self.assertEqual(Dataset.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) def test_the_api_should_return_all_datasets_with_metadata_true(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'datasets'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) user = get_user_model().objects.get(username="admin") token = get_or_create_token(user) - auth_header = f'Bearer {token}' + auth_header = f"Bearer {token}" url = f"{list_url}?metadata_only=True" resp = self.api_client.get(url, authentication=auth_header) @@ -717,22 +620,14 @@ def test_the_api_should_return_all_datasets_with_metadata_true(self): self.assertEqual(Dataset.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) def test_the_api_should_return_all_documents_with_metadata_false(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'documents'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "documents"}) resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertEqual(Document.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) def test_the_api_should_return_all_documents_with_metadata_true(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'documents'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "documents"}) url = f"{list_url}?metadata_only=True" resp = self.api_client.get(url) @@ -740,22 +635,14 @@ def test_the_api_should_return_all_documents_with_metadata_true(self): self.assertEqual(Document.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) def test_the_api_should_return_all_maps_with_metadata_false(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'maps'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "maps"}) resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertEqual(Map.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) def test_the_api_should_return_all_maps_with_metadata_true(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'maps'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "maps"}) url = f"{list_url}?metadata_only=True" resp = self.api_client.get(url) @@ -763,21 +650,13 @@ def test_the_api_should_return_all_maps_with_metadata_true(self): self.assertEqual(Map.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) def test_api_will_return_a_valid_json_response(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) def test_will_return_empty_if_the_thesaurus_does_not_exists(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) url = f"{list_url}?thesaurus=invalid-identifier" resp = self.api_client.get(url) @@ -785,11 +664,7 @@ def test_will_return_empty_if_the_thesaurus_does_not_exists(self): self.assertEqual(resp.json()["meta"]["total_count"], 0) def test_will_return_keywords_for_the_selected_thesaurus_if_exists(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) @@ -797,11 +672,7 @@ def test_will_return_keywords_for_the_selected_thesaurus_if_exists(self): self.assertEqual(resp.json()["meta"]["total_count"], 36) def test_will_return_empty_if_the_alt_label_does_not_exists(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) url = f"{list_url}?alt_label=invalid-alt_label" resp = self.api_client.get(url) @@ -809,11 +680,7 @@ def test_will_return_empty_if_the_alt_label_does_not_exists(self): self.assertEqual(resp.json()["meta"]["total_count"], 0) def test_will_return_keywords_for_the_selected_alt_label_if_exists(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) url = f"{list_url}?alt_label=ac" resp = self.api_client.get(url) @@ -821,11 +688,7 @@ def test_will_return_keywords_for_the_selected_alt_label_if_exists(self): self.assertEqual(resp.json()["meta"]["total_count"], 1) def test_will_return_empty_if_the_kaywordId_does_not_exists(self): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) url = f"{list_url}?id=12365478954862" resp = self.api_client.get(url) @@ -835,21 +698,49 @@ def test_will_return_empty_if_the_kaywordId_does_not_exists(self): @patch("geonode.api.api.get_language") def test_will_return_expected_keyword_label_for_existing_lang(self, lang): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) lang.return_value = "de" url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) # the german translations exists, for the other labels, the alt_label will be used expected_labels = [ - "", "ac", "Adressen", "af", "am", "au", "br", "bu", - "cp", "ef", "el", "er", "foo_keyword", "ge", "gg", "gn", "hb", "hh", - "hy", "lc", "lu", "mf", "mr", "nz", "of", "oi", "pd", - "pf", "ps", "rs", "sd", "so", "sr", "su", "tn", "us" + "", + "ac", + "Adressen", + "af", + "am", + "au", + "br", + "bu", + "cp", + "ef", + "el", + "er", + "foo_keyword", + "ge", + "gg", + "gn", + "hb", + "hh", + "hy", + "lc", + "lu", + "mf", + "mr", + "nz", + "of", + "oi", + "pd", + "pf", + "ps", + "rs", + "sd", + "so", + "sr", + "su", + "tn", + "us", ] actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]] self.assertValidJSONResponse(resp) @@ -857,82 +748,94 @@ def test_will_return_expected_keyword_label_for_existing_lang(self, lang): @patch("geonode.api.api.get_language") def test_will_return_default_keyword_label_for_not_existing_lang(self, lang): - list_url = reverse( - "api_dispatch_list", - kwargs={ - "api_name": "api", - "resource_name": "thesaurus/keywords"}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) lang.return_value = "ke" url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) # no translations exists, the alt_label will be used for all keywords expected_labels = [ - "", "ac", "ad", "af", "am", "au", "br", "bu", - "cp", "ef", "el", "er", "foo_keyword", "ge", "gg", "gn", "hb", "hh", - "hy", "lc", "lu", "mf", "mr", "nz", "of", "oi", "pd", - "pf", "ps", "rs", "sd", "so", "sr", "su", "tn", "us", + "", + "ac", + "ad", + "af", + "am", + "au", + "br", + "bu", + "cp", + "ef", + "el", + "er", + "foo_keyword", + "ge", + "gg", + "gn", + "hb", + "hh", + "hy", + "lc", + "lu", + "mf", + "mr", + "nz", + "of", + "oi", + "pd", + "pf", + "ps", + "rs", + "sd", + "so", + "sr", + "su", + "tn", + "us", ] actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]] self.assertValidJSONResponse(resp) self.assertListEqual(expected_labels, actual_labels) def test_the_api_should_return_all_map_categories_with_metadata_false(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'categories'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) url = f"{list_url}?type=map" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) - actual = sum([x['count'] for x in resp.json()['objects']]) + actual = sum([x["count"] for x in resp.json()["objects"]]) self.assertEqual(9, actual) def test_the_api_should_return_all_map_categories_with_metadata_true(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'categories'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) - x = Map.objects.get(title='map metadata true') + x = Map.objects.get(title="map metadata true") x.metadata_only = False x.save() url = f"{list_url}?type=map" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) # by adding a new layer, the total should increase - actual = sum([x['count'] for x in resp.json()['objects']]) + actual = sum([x["count"] for x in resp.json()["objects"]]) self.assertEqual(10, actual) def test_the_api_should_return_all_document_categories_with_metadata_false(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'categories'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) url = f"{list_url}?type=document" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) - actual = sum([x['count'] for x in resp.json()['objects']]) + actual = sum([x["count"] for x in resp.json()["objects"]]) self.assertEqual(0, actual) def test_the_api_should_return_all_document_categories_with_metadata_true(self): - list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'categories'}) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) - x = Document.objects.get(title='doc metadata true') + x = Document.objects.get(title="doc metadata true") x.metadata_only = False x.save() url = f"{list_url}?type=document" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) # by adding a new layer, the total should increase - actual = sum([x['count'] for x in resp.json()['objects']]) + actual = sum([x["count"] for x in resp.json()["objects"]]) self.assertEqual(0, actual) diff --git a/geonode/api/urls.py b/geonode/api/urls.py index c955b00c492..26303ab433e 100644 --- a/geonode/api/urls.py +++ b/geonode/api/urls.py @@ -18,18 +18,14 @@ ######################################################################### from tastypie.api import Api from dynamic_rest import routers -from drf_spectacular.views import ( - SpectacularAPIView, - SpectacularRedocView, - SpectacularSwaggerView -) +from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView from django.urls import path from . import api as resources from . import resourcebase_api as resourcebase_resources -api = Api(api_name='api') +api = Api(api_name="api") api.register(resources.GroupCategoryResource()) api.register(resources.GroupResource()) @@ -51,7 +47,7 @@ router = routers.DynamicRouter() urlpatterns = [ - path('schema/', SpectacularAPIView.as_view(), name='schema'), - path('schema/swagger-ui/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), - path('schema/redoc/', SpectacularRedocView.as_view(url_name='schema'), name='redoc'), + path("schema/", SpectacularAPIView.as_view(), name="schema"), + path("schema/swagger-ui/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("schema/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"), ] diff --git a/geonode/api/views.py b/geonode/api/views.py index bd340597e3d..69b73cb15ae 100644 --- a/geonode/api/views.py +++ b/geonode/api/views.py @@ -32,10 +32,7 @@ from ..utils import json_response from ..decorators import superuser_or_apiauth -from ..base.auth import ( - get_token_object_from_session, - extract_headers, - get_auth_token) +from ..base.auth import get_token_object_from_session, extract_headers, get_auth_token def verify_access_token(request, key): @@ -46,9 +43,9 @@ def verify_access_token(request, key): if not token or token.key != key: token = AccessToken.objects.get(token=key) if not token.is_valid(): - raise OAuthToolkitError('AccessToken is not valid.') + raise OAuthToolkitError("AccessToken is not valid.") if token.is_expired(): - raise OAuthToolkitError('AccessToken has expired.') + raise OAuthToolkitError("AccessToken has expired.") except AccessToken.DoesNotExist: raise FatalClientError("AccessToken not found at all.") except Exception: @@ -62,46 +59,38 @@ def user_info(request): user = request.user if not user: - out = {'success': False, - 'status': 'error', - 'errors': {'user': ['User is not authenticated']} - } + out = {"success": False, "status": "error", "errors": {"user": ["User is not authenticated"]}} return json_response(out, status=401) access_token = None - if 'Authorization' not in headers or 'Bearer' not in headers["Authorization"]: + if "Authorization" not in headers or "Bearer" not in headers["Authorization"]: access_token = get_auth_token(user) if not access_token: - out = { - 'success': False, - 'status': 'error', - 'errors': {'auth': ['No token provided.']} - } + out = {"success": False, "status": "error", "errors": {"auth": ["No token provided."]}} return json_response(out, status=403) else: - access_token = headers["Authorization"].replace('Bearer ', '') + access_token = headers["Authorization"].replace("Bearer ", "") groups = [group.name for group in user.groups.all()] if user.is_superuser: groups.append("admin") - user_info = json.dumps({ - "sub": str(user.id), - "name": " ".join([user_field(user, 'first_name'), user_field(user, 'last_name')]), - "given_name": user_field(user, 'first_name'), - "family_name": user_field(user, 'last_name'), - "email": user_email(user), - "preferred_username": user_username(user), - "groups": groups, - "access_token": str(access_token) - }) - - response = HttpResponse( - user_info, - content_type="application/json" + user_info = json.dumps( + { + "sub": str(user.id), + "name": " ".join([user_field(user, "first_name"), user_field(user, "last_name")]), + "given_name": user_field(user, "first_name"), + "family_name": user_field(user, "last_name"), + "email": user_email(user), + "preferred_username": user_username(user), + "groups": groups, + "access_token": str(access_token), + } ) - response['Cache-Control'] = 'no-store' - response['Pragma'] = 'no-cache' + + response = HttpResponse(user_info, content_type="application/json") + response["Cache-Control"] = "no-store" + response["Pragma"] = "no-cache" return response @@ -109,55 +98,38 @@ def user_info(request): @csrf_exempt def verify_token(request): - if (request.POST and 'token' in request.POST): + if request.POST and "token" in request.POST: token = None try: - access_token = request.POST.get('token') + access_token = request.POST.get("token") token = verify_access_token(request, access_token) except Exception as e: - return HttpResponse( - json.dumps({ - 'error': str(e) - }), - status=403, - content_type="application/json" - ) + return HttpResponse(json.dumps({"error": str(e)}), status=403, content_type="application/json") if token: - token_info = json.dumps({ - 'client_id': token.application.client_id, - 'user_id': token.user.id, - 'username': token.user.username, - 'issued_to': token.user.username, - 'access_token': access_token, - 'email': token.user.email, - 'verified_email': 'true', - 'access_type': 'online', - 'expires_in': (token.expires - timezone.now()).total_seconds() * 1000 - }) - - response = HttpResponse( - token_info, - content_type="application/json" + token_info = json.dumps( + { + "client_id": token.application.client_id, + "user_id": token.user.id, + "username": token.user.username, + "issued_to": token.user.username, + "access_token": access_token, + "email": token.user.email, + "verified_email": "true", + "access_type": "online", + "expires_in": (token.expires - timezone.now()).total_seconds() * 1000, + } ) + + response = HttpResponse(token_info, content_type="application/json") response["Authorization"] = f"Bearer {access_token}" return response else: return HttpResponse( - json.dumps({ - 'error': 'No access_token from server.' - }), - status=403, - content_type="application/json" + json.dumps({"error": "No access_token from server."}), status=403, content_type="application/json" ) - return HttpResponse( - json.dumps({ - 'error': 'invalid_request' - }), - status=403, - content_type="application/json" - ) + return HttpResponse(json.dumps({"error": "invalid_request"}), status=403, content_type="application/json") @csrf_exempt @@ -166,18 +138,13 @@ def roles(request): groups = [group.name for group in Group.objects.all()] groups.append("admin") - return HttpResponse( - json.dumps({ - 'groups': groups - }), - content_type="application/json" - ) + return HttpResponse(json.dumps({"groups": groups}), content_type="application/json") @csrf_exempt @superuser_or_apiauth() def users(request): - user_name = request.path_info.rsplit('/', 1)[-1] + user_name = request.path_info.rsplit("/", 1)[-1] User = get_user_model() if user_name is None or not user_name or user_name == "users": @@ -195,25 +162,12 @@ def users(request): if user.is_superuser: groups.append("admin") - json_object.append({ - 'username': user.username, - 'groups': groups - }) + json_object.append({"username": user.username, "groups": groups}) - return HttpResponse( - json.dumps({ - 'users': json_object - }), - content_type="application/json" - ) + return HttpResponse(json.dumps({"users": json_object}), content_type="application/json") @csrf_exempt @superuser_or_apiauth() def admin_role(request): - return HttpResponse( - json.dumps({ - 'adminRole': 'admin' - }), - content_type="application/json" - ) + return HttpResponse(json.dumps({"adminRole": "admin"}), content_type="application/json") diff --git a/geonode/apps.py b/geonode/apps.py index efa25ca5b53..7bbf85cfbec 100644 --- a/geonode/apps.py +++ b/geonode/apps.py @@ -22,8 +22,9 @@ def run_setup_hooks(*args, **kwargs): from django.conf import settings from .celery_app import app as celery_app + if celery_app not in settings.INSTALLED_APPS: - settings.INSTALLED_APPS += (celery_app, ) + settings.INSTALLED_APPS += (celery_app,) class AppConfig(BaseAppConfig): diff --git a/geonode/base/__init__.py b/geonode/base/__init__.py index a57e8a3f4a7..48ab2e6d723 100644 --- a/geonode/base/__init__.py +++ b/geonode/base/__init__.py @@ -25,12 +25,19 @@ class BaseAppConfig(NotificationsAppConfigBase): - name = 'geonode.base' - NOTIFICATIONS = (("request_download_resourcebase", _("Request to download a resource"), - _("A request for downloading a resource was sent")), - ("request_resource_edit", _("Request resource change"), - _("Owner has requested permissions to modify a resource")), - ) + name = "geonode.base" + NOTIFICATIONS = ( + ( + "request_download_resourcebase", + _("Request to download a resource"), + _("A request for downloading a resource was sent"), + ), + ( + "request_resource_edit", + _("Request resource change"), + _("Owner has requested permissions to modify a resource"), + ), + ) def register_url_event(event_type=None): @@ -41,14 +48,17 @@ def register_url_event(event_type=None): >> register_url_event()(TemplateView.view_as_view()) """ + def _register_url_event(view): @wraps(view) def inner(*args, **kwargs): if settings.MONITORING_ENABLED: request = args[0] - register_event(request, event_type or 'view', request.path) + register_event(request, event_type or "view", request.path) return view(*args, **kwargs) + return inner + return _register_url_event @@ -68,17 +78,18 @@ def register_event(request, event_type, resource): return from geonode.base.models import ResourceBase + if isinstance(resource, str): - resource_type = 'url' + resource_type = "url" resource_name = request.path resource_id = None elif isinstance(resource, ResourceBase): resource_type = resource.__class__._meta.verbose_name_raw - resource_name = getattr(resource, 'alternate', None) or resource.title + resource_name = getattr(resource, "alternate", None) or resource.title resource_id = resource.id else: raise ValueError(f"Invalid resource: {resource}") - if request and hasattr(request, 'register_event'): + if request and hasattr(request, "register_event"): request.register_event(event_type, resource_type, resource_name, resource_id) @@ -88,4 +99,4 @@ def register_proxy_event(request): """ -default_app_config = 'geonode.base.BaseAppConfig' +default_app_config = "geonode.base.BaseAppConfig" diff --git a/geonode/base/admin.py b/geonode/base/admin.py index 7a5fc719869..ccf7712392f 100755 --- a/geonode/base/admin.py +++ b/geonode/base/admin.py @@ -46,27 +46,24 @@ Menu, MenuItem, Configuration, - Thesaurus, ThesaurusLabel, ThesaurusKeyword, ThesaurusKeywordLabel, + Thesaurus, + ThesaurusLabel, + ThesaurusKeyword, + ThesaurusKeywordLabel, ) -from geonode.base.forms import ( - BatchEditForm, - ThesaurusImportForm, - UserAndGroupPermissionsForm -) +from geonode.base.forms import BatchEditForm, ThesaurusImportForm, UserAndGroupPermissionsForm from geonode.base.widgets import TaggitSelect2Custom def metadata_batch_edit(modeladmin, request, queryset): - ids = ','.join(str(element.pk) for element in queryset) + ids = ",".join(str(element.pk) for element in queryset) resource = queryset[0].class_name.lower() - form = BatchEditForm({ - 'ids': ids - }) + form = BatchEditForm({"ids": ids}) name_space_mapper = { - 'dataset': 'dataset_batch_metadata', - 'map': 'map_batch_metadata', - 'document': 'document_batch_metadata' + "dataset": "dataset_batch_metadata", + "map": "map_batch_metadata", + "document": "document_batch_metadata", } try: @@ -75,65 +72,50 @@ def metadata_batch_edit(modeladmin, request, queryset): name_space = None return render( - request, - "base/batch_edit.html", - context={ - 'form': form, - 'ids': ids, - 'model': resource, - 'name_space': name_space - } + request, "base/batch_edit.html", context={"form": form, "ids": ids, "model": resource, "name_space": name_space} ) -metadata_batch_edit.short_description = 'Metadata batch edit' +metadata_batch_edit.short_description = "Metadata batch edit" def set_user_and_group_dataset_permission(modeladmin, request, queryset): - ids = ','.join(str(element.pk) for element in queryset) + ids = ",".join(str(element.pk) for element in queryset) resource = queryset[0].__class__.__name__.lower() - model_mapper = { - "profile": "people", - "groupprofile": "groups" - } + model_mapper = {"profile": "people", "groupprofile": "groups"} - form = UserAndGroupPermissionsForm({ - 'permission_type': 'view', - 'mode': 'set', - 'ids': ids, - }) + form = UserAndGroupPermissionsForm( + { + "permission_type": "view", + "mode": "set", + "ids": ids, + } + ) return render( - request, - "base/user_and_group_permissions.html", - context={ - "form": form, - "model": model_mapper[resource] - } + request, "base/user_and_group_permissions.html", context={"form": form, "model": model_mapper[resource]} ) -set_user_and_group_dataset_permission.short_description = 'Set layer permissions' +set_user_and_group_dataset_permission.short_description = "Set layer permissions" class LicenseAdmin(TabbedTranslationAdmin): model = License - list_display = ('id', 'name') - list_display_links = ('name',) + list_display = ("id", "name") + list_display_links = ("name",) class TopicCategoryAdmin(TabbedTranslationAdmin): model = TopicCategory - list_display_links = ('identifier',) - list_display = ( - 'identifier', - 'description', - 'gn_description', - 'fa_class', - 'is_choice') + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "fa_class", "is_choice") if settings.MODIFY_TOPICCATEGORY is False: - exclude = ('identifier', 'description',) + exclude = ( + "identifier", + "description", + ) def has_add_permission(self, request): # the records are from the standard TC 211 list, so no way to add @@ -152,16 +134,19 @@ def has_delete_permission(self, request, obj=None): class RegionAdmin(TabbedTranslationAdmin): model = Region - list_display_links = ('name',) - list_display = ('code', 'name', 'parent') - search_fields = ('code', 'name',) + list_display_links = ("name",) + list_display = ("code", "name", "parent") + search_fields = ( + "code", + "name", + ) group_fieldsets = True class SpatialRepresentationTypeAdmin(TabbedTranslationAdmin): model = SpatialRepresentationType - list_display_links = ('identifier',) - list_display = ('identifier', 'description', 'gn_description', 'is_choice') + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "is_choice") def has_add_permission(self, request): # the records are from the standard TC 211 list, so no way to add @@ -174,8 +159,8 @@ def has_delete_permission(self, request, obj=None): class RestrictionCodeTypeAdmin(TabbedTranslationAdmin): model = RestrictionCodeType - list_display_links = ('identifier',) - list_display = ('identifier', 'description', 'gn_description', 'is_choice') + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "is_choice") def has_add_permission(self, request): # the records are from the standard TC 211 list, so no way to add @@ -188,39 +173,42 @@ def has_delete_permission(self, request, obj=None): class ContactRoleAdmin(admin.ModelAdmin): model = ContactRole - list_display_links = ('id',) - list_display = ('id', 'contact', 'resource', 'role') - list_editable = ('contact', 'resource', 'role') - form = forms.modelform_factory(ContactRole, fields='__all__') + list_display_links = ("id",) + list_display = ("id", "contact", "resource", "role") + list_editable = ("contact", "resource", "role") + form = forms.modelform_factory(ContactRole, fields="__all__") class LinkAdmin(admin.ModelAdmin): model = Link - list_display_links = ('id',) - list_display = ('id', 'resource', 'extension', 'link_type', 'name', 'mime') - list_filter = ('resource', 'extension', 'link_type', 'mime') - search_fields = ('name', 'resource__title',) - form = forms.modelform_factory(Link, fields='__all__') + list_display_links = ("id",) + list_display = ("id", "resource", "extension", "link_type", "name", "mime") + list_filter = ("resource", "extension", "link_type", "mime") + search_fields = ( + "name", + "resource__title", + ) + form = forms.modelform_factory(Link, fields="__all__") class HierarchicalKeywordAdmin(TreeAdmin): - search_fields = ('name', ) + search_fields = ("name",) form = movenodeform_factory(HierarchicalKeyword) class MenuPlaceholderAdmin(admin.ModelAdmin): model = MenuPlaceholder - list_display = ('name', ) + list_display = ("name",) class MenuAdmin(admin.ModelAdmin): model = Menu - list_display = ('title', 'placeholder', 'order') + list_display = ("title", "placeholder", "order") class MenuItemAdmin(admin.ModelAdmin): model = MenuItem - list_display = ('title', 'menu', 'order', 'blank_target', 'url') + list_display = ("title", "menu", "order", "blank_target", "url") class ConfigurationAdmin(admin.ModelAdmin): @@ -246,15 +234,13 @@ class ThesaurusAdmin(admin.ModelAdmin): change_list_template = "admin/thesauri/change_list.html" model = Thesaurus - list_display = ('id', 'identifier') - list_display_links = ('id', 'identifier') - ordering = ('identifier',) + list_display = ("id", "identifier") + list_display_links = ("id", "identifier") + ordering = ("identifier",) def get_urls(self): urls = super().get_urls() - my_urls = [ - path('importrdf/', self.import_rdf, name="base_thesaurus_importrdf") - ] + my_urls = [path("importrdf/", self.import_rdf, name="base_thesaurus_importrdf")] return my_urls + urls def import_rdf(self, request): @@ -262,7 +248,7 @@ def import_rdf(self, request): try: rdf_file = request.FILES["rdf_file"] name = slugify(rdf_file.name) - call_command('load_thesaurus', file=rdf_file, name=name) + call_command("load_thesaurus", file=rdf_file, name=name) self.message_user(request, "Your RDF file has been imported", messages.SUCCESS) return redirect("..") except Exception as e: @@ -271,58 +257,66 @@ def import_rdf(self, request): form = ThesaurusImportForm() payload = {"form": form} - return render( - request, "admin/thesauri/upload_form.html", payload - ) + return render(request, "admin/thesauri/upload_form.html", payload) class ThesaurusLabelAdmin(admin.ModelAdmin): model = ThesaurusLabel - list_display = ('thesaurus_id', 'lang', 'label') - list_display_links = ('label',) - ordering = ('thesaurus__identifier', 'lang') + list_display = ("thesaurus_id", "lang", "label") + list_display_links = ("label",) + ordering = ("thesaurus__identifier", "lang") def thesaurus_id(self, obj): return obj.thesaurus.identifier - thesaurus_id.short_description = 'Thesaurus' - thesaurus_id.admin_order_field = 'thesaurus__identifier' + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "thesaurus__identifier" class ThesaurusKeywordAdmin(admin.ModelAdmin): model = ThesaurusKeyword - list_display = ('thesaurus_id', 'about', 'alt_label',) - list_display_links = ('about', 'alt_label',) - ordering = ('thesaurus__identifier', 'alt_label',) - list_filter = ('thesaurus_id',) + list_display = ( + "thesaurus_id", + "about", + "alt_label", + ) + list_display_links = ( + "about", + "alt_label", + ) + ordering = ( + "thesaurus__identifier", + "alt_label", + ) + list_filter = ("thesaurus_id",) def thesaurus_id(self, obj): return obj.thesaurus.identifier - thesaurus_id.short_description = 'Thesaurus' - thesaurus_id.admin_order_field = 'thesaurus__identifier' + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "thesaurus__identifier" class ThesaurusKeywordLabelAdmin(admin.ModelAdmin): model = ThesaurusKeywordLabel - list_display = ('thesaurus_id', 'keyword_id', 'lang', 'label') - list_display_links = ('lang', 'label') - ordering = ('keyword__thesaurus__identifier', 'keyword__alt_label', 'lang') - list_filter = ('keyword__thesaurus__identifier', 'keyword_id', 'lang') + list_display = ("thesaurus_id", "keyword_id", "lang", "label") + list_display_links = ("lang", "label") + ordering = ("keyword__thesaurus__identifier", "keyword__alt_label", "lang") + list_filter = ("keyword__thesaurus__identifier", "keyword_id", "lang") def thesaurus_id(self, obj): return obj.keyword.thesaurus.identifier - thesaurus_id.short_description = 'Thesaurus' - thesaurus_id.admin_order_field = 'keyword__thesaurus__identifier' + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "keyword__thesaurus__identifier" def keyword_id(self, obj): return obj.keyword.alt_label - keyword_id.short_description = 'Keyword' - keyword_id.admin_order_field = 'keyword__alt_label' + keyword_id.short_description = "Keyword" + keyword_id.admin_order_field = "keyword__alt_label" admin.site.register(TopicCategory, TopicCategoryAdmin) @@ -345,7 +339,7 @@ def keyword_id(self, obj): class ResourceBaseAdminForm(autocomplete.FutureModelForm): - keywords = TagField(widget=TaggitSelect2Custom('autocomplete_hierachical_keyword')) + keywords = TagField(widget=TaggitSelect2Custom("autocomplete_hierachical_keyword")) def delete_queryset(self, request, queryset): """ @@ -354,6 +348,7 @@ def delete_queryset(self, request, queryset): """ for obj in queryset: from geonode.resource.manager import resource_manager + resource_manager.delete(obj.uuid, instance=obj) class Meta: diff --git a/geonode/base/api/exceptions.py b/geonode/base/api/exceptions.py index 0d0fe21ed8f..a1377af067c 100644 --- a/geonode/base/api/exceptions.py +++ b/geonode/base/api/exceptions.py @@ -31,7 +31,7 @@ def geonode_exception_handler(exc, context): response.data = { "success": False, "errors": [str(detail)], - "code": exc.code if hasattr(exc, "code") else exc.default_code + "code": exc.code if hasattr(exc, "code") else exc.default_code, } return response diff --git a/geonode/base/api/filters.py b/geonode/base/api/filters.py index 6e5830441a1..8d75d0e1a53 100644 --- a/geonode/base/api/filters.py +++ b/geonode/base/api/filters.py @@ -29,9 +29,8 @@ class DynamicSearchFilter(SearchFilter): - def get_search_fields(self, view, request): - return request.GET.getlist('search_fields', []) + return request.GET.getlist("search_fields", []) class ExtentFilter(BaseFilterBackend): @@ -40,8 +39,8 @@ class ExtentFilter(BaseFilterBackend): """ def filter_queryset(self, request, queryset, view): - if request.query_params.get('extent'): - return filter_bbox(queryset, request.query_params.get('extent')) + if request.query_params.get("extent"): + return filter_bbox(queryset, request.query_params.get("extent")) return queryset @@ -51,7 +50,7 @@ class FavoriteFilter(BaseFilterBackend): """ def filter_queryset(self, request, queryset, _): - if strtobool(request.query_params.get("favorite", 'False')): + if strtobool(request.query_params.get("favorite", "False")): c_types = list({r.polymorphic_ctype.model for r in queryset}) return queryset.filter( pk__in=Subquery( @@ -71,20 +70,12 @@ class FacetVisibleResourceFilter(BaseFilterBackend): def filter_queryset(self, request, queryset, _): _filter = {} - _with_resources = ast.literal_eval(request.GET.get("with_resources", 'False')) + _with_resources = ast.literal_eval(request.GET.get("with_resources", "False")) if _with_resources: - _filter["id__in"] = [ - _facet.id - for _facet in queryset - if _facet.resourcebase_set.exists() - ] - elif 'with_resources' in request.GET and not _with_resources: + _filter["id__in"] = [_facet.id for _facet in queryset if _facet.resourcebase_set.exists()] + elif "with_resources" in request.GET and not _with_resources: # check that the facet has been passed and is false - _filter["id__in"] = [ - _facet.id - for _facet in queryset - if not _facet.resourcebase_set.exists() - ] + _filter["id__in"] = [_facet.id for _facet in queryset if not _facet.resourcebase_set.exists()] return queryset.filter(**_filter) diff --git a/geonode/base/api/pagination.py b/geonode/base/api/pagination.py index 0b5675e5c83..7fc34cdaea9 100644 --- a/geonode/base/api/pagination.py +++ b/geonode/base/api/pagination.py @@ -20,9 +20,9 @@ from rest_framework.response import Response from rest_framework.pagination import PageNumberPagination -DEFAULT_PAGE = getattr(settings, 'REST_API_DEFAULT_PAGE', 1) -DEFAULT_PAGE_SIZE = getattr(settings, 'REST_API_DEFAULT_PAGE_SIZE', 10) -DEFAULT_PAGE_QUERY_PARAM = getattr(settings, 'REST_API_DEFAULT_PAGE_QUERY_PARAM', 'page_size') +DEFAULT_PAGE = getattr(settings, "REST_API_DEFAULT_PAGE", 1) +DEFAULT_PAGE_SIZE = getattr(settings, "REST_API_DEFAULT_PAGE_SIZE", 10) +DEFAULT_PAGE_QUERY_PARAM = getattr(settings, "REST_API_DEFAULT_PAGE_QUERY_PARAM", "page_size") class GeoNodeApiPagination(PageNumberPagination): @@ -33,13 +33,10 @@ class GeoNodeApiPagination(PageNumberPagination): def get_paginated_response(self, data): _paginated_response = { - 'links': { - 'next': self.get_next_link(), - 'previous': self.get_previous_link() - }, - 'total': self.page.paginator.count, - 'page': int(self.request.GET.get('page', DEFAULT_PAGE)), # can not set default = self.page - DEFAULT_PAGE_QUERY_PARAM: int(self.request.GET.get(DEFAULT_PAGE_QUERY_PARAM, self.page_size)) + "links": {"next": self.get_next_link(), "previous": self.get_previous_link()}, + "total": self.page.paginator.count, + "page": int(self.request.GET.get("page", DEFAULT_PAGE)), # can not set default = self.page + DEFAULT_PAGE_QUERY_PARAM: int(self.request.GET.get(DEFAULT_PAGE_QUERY_PARAM, self.page_size)), } _paginated_response.update(data) return Response(_paginated_response) diff --git a/geonode/base/api/permissions.py b/geonode/base/api/permissions.py index 24764d91ae9..c645c204116 100644 --- a/geonode/base/api/permissions.py +++ b/geonode/base/api/permissions.py @@ -23,11 +23,14 @@ from rest_framework import permissions from rest_framework.filters import BaseFilterBackend -from geonode.security.permissions import BASIC_MANAGE_PERMISSIONS, DOWNLOAD_PERMISSIONS, EDIT_PERMISSIONS, VIEW_PERMISSIONS +from geonode.security.permissions import ( + BASIC_MANAGE_PERMISSIONS, + DOWNLOAD_PERMISSIONS, + EDIT_PERMISSIONS, + VIEW_PERMISSIONS, +) from distutils.util import strtobool -from geonode.security.utils import ( - get_users_with_perms, - get_visible_resources) +from geonode.security.utils import get_users_with_perms, get_visible_resources from geonode.groups.models import GroupProfile from rest_framework.permissions import DjangoModelPermissions from guardian.shortcuts import get_objects_for_user @@ -39,13 +42,13 @@ class IsSelf(permissions.BasePermission): - """ Grant permission only if the current instance is the request user. + """Grant permission only if the current instance is the request user. Used to allow users to edit their own account, nothing to others (even superusers). """ def has_permission(self, request, view): - """ Always return False here. + """Always return False here. The fine-grained permissions are handled in has_object_permission(). """ return False @@ -60,7 +63,7 @@ def has_object_permission(self, request, view, obj): class IsSelfOrReadOnly(IsSelf): - """ Grant permissions if instance *IS* the request user, or read-only. + """Grant permissions if instance *IS* the request user, or read-only. Used to allow users to edit their own account, and others to read. """ @@ -73,7 +76,7 @@ def has_object_permission(self, request, view, obj): class IsSelfOrAdmin(IsSelf): - """ Grant R/W to self and superusers/staff members. Deny others. """ + """Grant R/W to self and superusers/staff members. Deny others.""" def has_permission(self, request, view): user = request.user @@ -92,7 +95,7 @@ def has_object_permission(self, request, view, obj): class IsSelfOrAdminOrReadOnly(IsSelfOrAdmin): - """ Grant R/W to self and superusers/staff members, R/O to others. """ + """Grant R/W to self and superusers/staff members, R/O to others.""" def has_permission(self, request, view): if request.method in permissions.SAFE_METHODS: @@ -109,7 +112,7 @@ def has_object_permission(self, request, view, obj): class IsSelfOrAdminOrAuthenticatedReadOnly(IsSelfOrAdmin): - """ Grant R/W to self and superusers/staff members, R/O to auth. """ + """Grant R/W to self and superusers/staff members, R/O to auth.""" def has_object_permission(self, request, view, obj): user = request.user @@ -136,9 +139,9 @@ def has_object_permission(self, request, view, obj): _request_matches = False if isinstance(obj, get_user_model()) and obj == request.user: _request_matches = True - elif hasattr(obj, 'owner'): + elif hasattr(obj, "owner"): _request_matches = obj.owner == request.user - elif hasattr(obj, 'user'): + elif hasattr(obj, "user"): _request_matches = obj.user == request.user if not _request_matches: @@ -167,7 +170,7 @@ class IsManagerEditOrAdmin(permissions.BasePermission): """ def has_permission(self, request, view): - if request.method in ['POST', 'DELETE']: + if request.method in ["POST", "DELETE"]: user = request.user return user and (user.is_superuser or user.is_staff) @@ -184,7 +187,7 @@ def has_object_permission(self, request, view, obj): return True is_group_manager = user and isinstance(obj, GroupProfile) and obj.user_is_role(user, "manager") - if is_group_manager and request.method == 'PATCH': + if is_group_manager and request.method == "PATCH": return True return False @@ -209,17 +212,17 @@ def filter_queryset(self, request, queryset, view): metadata_only=metadata_only, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, ) class UserHasPerms(DjangoModelPermissions): perms_map = { - 'GET': [f'base.{x}' for x in VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS], - 'POST': ['base.add_resourcebase'] + [f'base.{x}' for x in EDIT_PERMISSIONS], - 'PUT': [f'base.{x}' for x in EDIT_PERMISSIONS], - 'PATCH': [f'base.{x}' for x in EDIT_PERMISSIONS], - 'DELETE': [f'base.{x}' for x in BASIC_MANAGE_PERMISSIONS], + "GET": [f"base.{x}" for x in VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS], + "POST": ["base.add_resourcebase"] + [f"base.{x}" for x in EDIT_PERMISSIONS], + "PUT": [f"base.{x}" for x in EDIT_PERMISSIONS], + "PATCH": [f"base.{x}" for x in EDIT_PERMISSIONS], + "DELETE": [f"base.{x}" for x in BASIC_MANAGE_PERMISSIONS], } def __init__(self, perms_dict={}): @@ -230,21 +233,26 @@ def __call__(self): def has_permission(self, request, view): from geonode.base.models import ResourceBase + queryset = self._queryset(view) if request.user.is_superuser: return True - if view.kwargs.get('pk'): + if view.kwargs.get("pk"): # if a single resource is called, we check the perms for that resource - res = get_object_or_404(ResourceBase, pk=view.kwargs.get('pk')) + res = get_object_or_404(ResourceBase, pk=view.kwargs.get("pk")) # if the request is for a single resource, we take the specific or the default. If none is defined we keep the original one defined above - resource_type_specific_perms = self.perms_dict.get(res.get_real_instance().resource_type, self.perms_dict.get('default', {})) - perms = resource_type_specific_perms.get(request.method, []) or self.get_required_permissions(request.method, queryset.model) + resource_type_specific_perms = self.perms_dict.get( + res.get_real_instance().resource_type, self.perms_dict.get("default", {}) + ) + perms = resource_type_specific_perms.get(request.method, []) or self.get_required_permissions( + request.method, queryset.model + ) # getting the user permission for that resource resource_perms = list(res.get_user_perms(request.user)) - if getattr(res, 'get_real_instance', None): + if getattr(res, "get_real_instance", None): resource_perms.extend(list(res.get_real_instance().get_user_perms(request.user))) groups = get_groups_with_perms(res, attach_perms=True) @@ -254,12 +262,12 @@ def has_permission(self, request, view): if group.user_set.filter(username=request.user).exists(): resource_perms = list(chain(resource_perms, perm)) - if request.user.has_perm('base.add_resourcebase'): - resource_perms.append('add_resourcebase') + if request.user.has_perm("base.add_resourcebase"): + resource_perms.append("add_resourcebase") # merging all available permissions into a single list available_perms = list(set(resource_perms)) # fixup the permissions name - perms_without_base = [x.replace('base.', '') for x in perms] + perms_without_base = [x.replace("base.", "") for x in perms] # if at least one of the permissions is available the request is True rule = resource_type_specific_perms.get("rule", any) return rule([_perm in available_perms for _perm in perms_without_base]) @@ -273,7 +281,9 @@ def has_permission(self, request, view): rule = _default_defined_perms.get("rule", any) return rule([request.user.has_perm(_perm) for _perm in _defined_perms]) - perms = self.perms_dict.get(request.method, None) or self.get_required_permissions(request.method, queryset.model) + perms = self.perms_dict.get(request.method, None) or self.get_required_permissions( + request.method, queryset.model + ) # check if the user have one of the perms in all the resource available return get_objects_for_user(request.user, perms).exists() diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 0d45469f844..03b09c871d4 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -48,11 +48,9 @@ SpatialRepresentationType, ThesaurusKeyword, ThesaurusKeywordLabel, - ExtraMetadata + ExtraMetadata, ) -from geonode.groups.models import ( - GroupCategory, - GroupProfile) +from geonode.groups.models import GroupCategory, GroupProfile from geonode.utils import build_absolute_uri from geonode.security.utils import get_resources_with_perms @@ -64,56 +62,47 @@ class BaseDynamicModelSerializer(DynamicModelSerializer): - def to_representation(self, instance): data = super().to_representation(instance) if not isinstance(data, int): try: path = reverse(self.Meta.view_name) - if not path.endswith('/'): + if not path.endswith("/"): path = f"{path}/" url = urljoin(path, str(instance.pk)) - data['link'] = build_absolute_uri(url) + data["link"] = build_absolute_uri(url) except (TypeError, NoReverseMatch) as e: logger.exception(e) return data class ResourceBaseToRepresentationSerializerMixin(DynamicModelSerializer): - def to_representation(self, instance): - request = self.context.get('request') + request = self.context.get("request") data = super(ResourceBaseToRepresentationSerializerMixin, self).to_representation(instance) if request: - data['perms'] = instance.get_user_perms(request.user).union( - instance.get_self_resource().get_user_perms(request.user) - ).union( - instance.get_real_instance().get_user_perms(request.user) + data["perms"] = ( + instance.get_user_perms(request.user) + .union(instance.get_self_resource().get_user_perms(request.user)) + .union(instance.get_real_instance().get_user_perms(request.user)) ) if not request.user.is_anonymous and getattr(settings, "FAVORITE_ENABLED", False): favorite = Favorite.objects.filter(user=request.user, object_id=instance.pk).count() - data['favorite'] = favorite > 0 + data["favorite"] = favorite > 0 # Adding links to resource_base api - obj_id = data.get('pk', None) + obj_id = data.get("pk", None) if obj_id: dehydrated = [] - link_fields = [ - 'extension', - 'link_type', - 'name', - 'mime', - 'url' - ] + link_fields = ["extension", "link_type", "name", "mime", "url"] links = Link.objects.filter( - resource_id=int(obj_id), - link_type__in=['OGC:WMS', 'OGC:WFS', 'OGC:WCS', 'image', 'metadata'] + resource_id=int(obj_id), link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS", "image", "metadata"] ) for lnk in links: formatted_link = model_to_dict(lnk, fields=link_fields) dehydrated.append(formatted_link) if len(dehydrated) > 0: - data['links'] = dehydrated + data["links"] = dehydrated return data @@ -122,13 +111,12 @@ class ResourceBaseTypesSerializer(DynamicEphemeralSerializer): count = serializers.IntegerField() class Meta: - name = 'resource-types' + name = "resource-types" class PermSpecSerialiazer(DynamicEphemeralSerializer): - class Meta: - name = 'perm-spec' + name = "perm-spec" class PermSpecFieldSerialiazer(DynamicEphemeralSerializer): perm_spec = serializers.ListField() @@ -138,108 +126,95 @@ class PermSpecFieldSerialiazer(DynamicEphemeralSerializer): class GroupSerializer(DynamicModelSerializer): - class Meta: model = Group - name = 'group' - fields = ('pk', 'name') + name = "group" + fields = ("pk", "name") class GroupProfileSerializer(BaseDynamicModelSerializer): - class Meta: model = GroupProfile - name = 'group_profile' - view_name = 'group-profiles-list' - fields = ('pk', 'title', 'group', 'slug', 'logo', 'description', - 'email', 'keywords', 'access', 'categories') + name = "group_profile" + view_name = "group-profiles-list" + fields = ("pk", "title", "group", "slug", "logo", "description", "email", "keywords", "access", "categories") group = DynamicRelationField(GroupSerializer, embed=True, many=False) - keywords = serializers.SlugRelatedField(many=True, slug_field='slug', read_only=True) - categories = serializers.SlugRelatedField( - many=True, slug_field='slug', queryset=GroupCategory.objects.all()) + keywords = serializers.SlugRelatedField(many=True, slug_field="slug", read_only=True) + categories = serializers.SlugRelatedField(many=True, slug_field="slug", queryset=GroupCategory.objects.all()) class SimpleHierarchicalKeywordSerializer(DynamicModelSerializer): - class Meta: model = HierarchicalKeyword - name = 'HierarchicalKeyword' - fields = ('name', 'slug') + name = "HierarchicalKeyword" + fields = ("name", "slug") def to_representation(self, value): - return {'name': value.name, 'slug': value.slug} + return {"name": value.name, "slug": value.slug} class _ThesaurusKeywordSerializerMixIn: - def to_representation(self, value): _i18n = {} for _i18n_label in ThesaurusKeywordLabel.objects.filter(keyword__id=value.id).iterator(): _i18n[_i18n_label.lang] = _i18n_label.label return { - 'name': value.alt_label, - 'slug': slugify(value.about), - 'uri': value.about, - 'thesaurus': { - 'name': value.thesaurus.title, - 'slug': value.thesaurus.identifier, - 'uri': value.thesaurus.about + "name": value.alt_label, + "slug": slugify(value.about), + "uri": value.about, + "thesaurus": { + "name": value.thesaurus.title, + "slug": value.thesaurus.identifier, + "uri": value.thesaurus.about, }, - 'i18n': _i18n + "i18n": _i18n, } class SimpleThesaurusKeywordSerializer(_ThesaurusKeywordSerializerMixIn, DynamicModelSerializer): - class Meta: model = ThesaurusKeyword - name = 'ThesaurusKeyword' - fields = ('alt_label', ) + name = "ThesaurusKeyword" + fields = ("alt_label",) class SimpleRegionSerializer(DynamicModelSerializer): - class Meta: model = Region - name = 'Region' - fields = ('code', 'name') + name = "Region" + fields = ("code", "name") class SimpleTopicCategorySerializer(DynamicModelSerializer): - class Meta: model = TopicCategory - name = 'TopicCategory' - fields = ('identifier',) + name = "TopicCategory" + fields = ("identifier",) class RestrictionCodeTypeSerializer(DynamicModelSerializer): - class Meta: model = RestrictionCodeType - name = 'RestrictionCodeType' - fields = ('identifier',) + name = "RestrictionCodeType" + fields = ("identifier",) class LicenseSerializer(DynamicModelSerializer): - class Meta: model = License - name = 'License' - fields = ('identifier',) + name = "License" + fields = ("identifier",) class SpatialRepresentationTypeSerializer(DynamicModelSerializer): - class Meta: model = SpatialRepresentationType - name = 'SpatialRepresentationType' - fields = ('identifier',) + name = "SpatialRepresentationType" + fields = ("identifier",) class AvatarUrlField(DynamicComputedField): - def __init__(self, avatar_size, **kwargs): self.avatar_size = avatar_size super().__init__(**kwargs) @@ -249,7 +224,6 @@ def get_attribute(self, instance): class EmbedUrlField(DynamicComputedField): - def __init__(self, **kwargs): super().__init__(**kwargs) @@ -259,14 +233,13 @@ def get_attribute(self, instance): except Exception as e: logger.exception(e) _instance = None - if _instance and hasattr(_instance, 'embed_url') and _instance.embed_url != NotImplemented: + if _instance and hasattr(_instance, "embed_url") and _instance.embed_url != NotImplemented: return build_absolute_uri(_instance.embed_url) else: return "" class DetailUrlField(DynamicComputedField): - def __init__(self, **kwargs): super().__init__(**kwargs) @@ -277,8 +250,8 @@ def get_attribute(self, instance): class ExtraMetadataSerializer(DynamicModelSerializer): class Meta: model = ExtraMetadata - name = 'ExtraMetadata' - fields = ('pk', 'metadata') + name = "ExtraMetadata" + fields = ("pk", "metadata") def to_representation(self, obj): @@ -293,7 +266,6 @@ def to_representation(self, obj): class ThumbnailUrlField(DynamicComputedField): - def __init__(self, **kwargs): super().__init__(**kwargs) @@ -321,45 +293,43 @@ def __init__(self, **kwargs): super().__init__(**kwargs) def get_attribute(self, instance): - _user = self.context.get('request') + _user = self.context.get("request") if _user and not _user.user.is_anonymous: return Favorite.objects.filter(object_id=instance.pk, user=_user.user).exists() return False class UserSerializer(BaseDynamicModelSerializer): - class Meta: - ref_name = 'UserProfile' + ref_name = "UserProfile" model = get_user_model() - name = 'user' - view_name = 'users-list' - fields = ('pk', 'username', 'first_name', 'last_name', 'avatar', 'perms', 'is_superuser', 'is_staff') + name = "user" + view_name = "users-list" + fields = ("pk", "username", "first_name", "last_name", "avatar", "perms", "is_superuser", "is_staff") @classmethod def setup_eager_loading(cls, queryset): - """ Perform necessary eager loading of data. """ + """Perform necessary eager loading of data.""" queryset = queryset.prefetch_related() return queryset def to_representation(self, instance): # Dehydrate users private fields - request = self.context.get('request') + request = self.context.get("request") data = super().to_representation(instance) if not request or not request.user or not request.user.is_authenticated: - if 'perms' in data: - del data['perms'] + if "perms" in data: + del data["perms"] elif not request.user.is_superuser and not request.user.is_staff: - if data['username'] != request.user.username: - if 'perms' in data: - del data['perms'] + if data["username"] != request.user.username: + if "perms" in data: + del data["perms"] return data avatar = AvatarUrlField(240, read_only=True) class ContactRoleField(DynamicComputedField): - def __init__(self, contat_type, **kwargs): self.contat_type = contat_type super().__init__(**kwargs) @@ -372,17 +342,15 @@ def to_representation(self, value): class DataBlobField(DynamicRelationField): - def value_to_string(self, obj): value = self.value_from_object(obj) return self.get_prep_value(value) class DataBlobSerializer(DynamicModelSerializer): - class Meta: model = ResourceBase - fields = ('pk', 'blob') + fields = ("pk", "blob") def to_internal_value(self, data): return data @@ -395,41 +363,48 @@ def to_representation(self, value): class ResourceExecutionRequestSerializer(DynamicModelSerializer): - class Meta: model = ResourceBase - fields = ('pk',) + fields = ("pk",) def to_representation(self, instance): data = [] - request = self.context.get('request', None) - if request and request.user and not request.user.is_anonymous and ResourceBase.objects.filter(pk=instance).count() == 1: + request = self.context.get("request", None) + if ( + request + and request.user + and not request.user.is_anonymous + and ResourceBase.objects.filter(pk=instance).count() == 1 + ): _resource = ResourceBase.objects.get(pk=instance) executions = ExecutionRequest.objects.filter( - Q(user=request.user) & - ~Q(status=ExecutionRequest.STATUS_FINISHED) & ( - (Q(input_params__uuid=_resource.uuid) | - Q(output_params__output__uuid=_resource.uuid) | - Q(geonode_resource=_resource)) + Q(user=request.user) + & ~Q(status=ExecutionRequest.STATUS_FINISHED) + & ( + ( + Q(input_params__uuid=_resource.uuid) + | Q(output_params__output__uuid=_resource.uuid) + | Q(geonode_resource=_resource) + ) ) - ).order_by('-created') + ).order_by("-created") for execution in executions: - data.append({ - 'exec_id': execution.exec_id, - 'user': execution.user.username, - 'status': execution.status, - 'func_name': execution.func_name, - 'created': execution.created, - 'finished': execution.finished, - 'last_updated': execution.last_updated, - 'input_params': execution.input_params, - 'output_params': execution.output_params, - 'status_url': urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': execution.exec_id}) - ) - }, + data.append( + { + "exec_id": execution.exec_id, + "user": execution.user.username, + "status": execution.status, + "func_name": execution.func_name, + "created": execution.created, + "finished": execution.finished, + "last_updated": execution.last_updated, + "input_params": execution.input_params, + "output_params": execution.output_params, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": execution.exec_id}) + ), + }, ) return data @@ -438,101 +413,145 @@ class ResourceBaseSerializer( ResourceBaseToRepresentationSerializerMixin, BaseDynamicModelSerializer, ): - def __init__(self, *args, **kwargs): # Instantiate the superclass normally super().__init__(*args, **kwargs) - self.fields['pk'] = serializers.CharField(read_only=True) - self.fields['uuid'] = serializers.CharField(read_only=True) - self.fields['resource_type'] = serializers.CharField(required=False) - self.fields['polymorphic_ctype_id'] = serializers.CharField(read_only=True) - self.fields['owner'] = DynamicRelationField(UserSerializer, embed=True, many=False, read_only=True, required=False) - self.fields['poc'] = ContactRoleField('poc', read_only=True) - self.fields['metadata_author'] = ContactRoleField('metadata_author', read_only=True) - self.fields['title'] = serializers.CharField() - self.fields['abstract'] = serializers.CharField(required=False) - self.fields['attribution'] = serializers.CharField(required=False) - self.fields['doi'] = serializers.CharField(required=False) - self.fields['alternate'] = serializers.CharField(read_only=True) - self.fields['date'] = serializers.DateTimeField(required=False) - self.fields['date_type'] = serializers.CharField(required=False) - self.fields['temporal_extent_start'] = serializers.DateTimeField(required=False) - self.fields['temporal_extent_end'] = serializers.DateTimeField(required=False) - self.fields['edition'] = serializers.CharField(required=False) - self.fields['purpose'] = serializers.CharField(required=False) - self.fields['maintenance_frequency'] = serializers.CharField(required=False) - self.fields['constraints_other'] = serializers.CharField(required=False) - self.fields['language'] = serializers.CharField(required=False) - self.fields['supplemental_information'] = serializers.CharField(required=False) - self.fields['data_quality_statement'] = serializers.CharField(required=False) - self.fields['bbox_polygon'] = fields.GeometryField(read_only=True, required=False) - self.fields['ll_bbox_polygon'] = fields.GeometryField(read_only=True, required=False) - self.fields['srid'] = serializers.CharField(required=False) - self.fields['group'] = DynamicRelationField(GroupSerializer, embed=True, many=False) - self.fields['popular_count'] = serializers.CharField(required=False) - self.fields['share_count'] = serializers.CharField(required=False) - self.fields['rating'] = serializers.CharField(required=False) - self.fields['featured'] = serializers.BooleanField(required=False) - self.fields['is_published'] = serializers.BooleanField(required=False, read_only=True) - self.fields['is_approved'] = serializers.BooleanField(required=False, read_only=True) - self.fields['detail_url'] = DetailUrlField(read_only=True) - self.fields['created'] = serializers.DateTimeField(read_only=True) - self.fields['last_updated'] = serializers.DateTimeField(read_only=True) - self.fields['raw_abstract'] = serializers.CharField(read_only=True) - self.fields['raw_purpose'] = serializers.CharField(read_only=True) - self.fields['raw_constraints_other'] = serializers.CharField(read_only=True) - self.fields['raw_supplemental_information'] = serializers.CharField(read_only=True) - self.fields['raw_data_quality_statement'] = serializers.CharField(read_only=True) - self.fields['metadata_only'] = serializers.BooleanField(required=False) - self.fields['processed'] = serializers.BooleanField(read_only=True) - self.fields['state'] = serializers.CharField(read_only=True) - self.fields['sourcetype'] = serializers.CharField(read_only=True) - - self.fields['embed_url'] = EmbedUrlField(required=False) - self.fields['thumbnail_url'] = ThumbnailUrlField(read_only=True) - self.fields['keywords'] = DynamicRelationField( - SimpleHierarchicalKeywordSerializer, embed=False, many=True) - self.fields['tkeywords'] = DynamicRelationField( - SimpleThesaurusKeywordSerializer, embed=False, many=True) - self.fields['regions'] = DynamicRelationField( - SimpleRegionSerializer, embed=True, many=True, read_only=True) - self.fields['category'] = DynamicRelationField( - SimpleTopicCategorySerializer, embed=True, many=False) - self.fields['restriction_code_type'] = DynamicRelationField( - RestrictionCodeTypeSerializer, embed=True, many=False) - self.fields['license'] = DynamicRelationField( - LicenseSerializer, embed=True, many=False) - self.fields['spatial_representation_type'] = DynamicRelationField( - SpatialRepresentationTypeSerializer, embed=True, many=False) - self.fields['blob'] = serializers.JSONField(required=False, write_only=True) - self.fields['is_copyable'] = serializers.BooleanField(read_only=True) - - self.fields['download_url'] = DownloadLinkField(read_only=True) - - self.fields['favorite'] = FavoriteField(read_only=True) + self.fields["pk"] = serializers.CharField(read_only=True) + self.fields["uuid"] = serializers.CharField(read_only=True) + self.fields["resource_type"] = serializers.CharField(required=False) + self.fields["polymorphic_ctype_id"] = serializers.CharField(read_only=True) + self.fields["owner"] = DynamicRelationField( + UserSerializer, embed=True, many=False, read_only=True, required=False + ) + self.fields["poc"] = ContactRoleField("poc", read_only=True) + self.fields["metadata_author"] = ContactRoleField("metadata_author", read_only=True) + self.fields["title"] = serializers.CharField() + self.fields["abstract"] = serializers.CharField(required=False) + self.fields["attribution"] = serializers.CharField(required=False) + self.fields["doi"] = serializers.CharField(required=False) + self.fields["alternate"] = serializers.CharField(read_only=True) + self.fields["date"] = serializers.DateTimeField(required=False) + self.fields["date_type"] = serializers.CharField(required=False) + self.fields["temporal_extent_start"] = serializers.DateTimeField(required=False) + self.fields["temporal_extent_end"] = serializers.DateTimeField(required=False) + self.fields["edition"] = serializers.CharField(required=False) + self.fields["purpose"] = serializers.CharField(required=False) + self.fields["maintenance_frequency"] = serializers.CharField(required=False) + self.fields["constraints_other"] = serializers.CharField(required=False) + self.fields["language"] = serializers.CharField(required=False) + self.fields["supplemental_information"] = serializers.CharField(required=False) + self.fields["data_quality_statement"] = serializers.CharField(required=False) + self.fields["bbox_polygon"] = fields.GeometryField(read_only=True, required=False) + self.fields["ll_bbox_polygon"] = fields.GeometryField(read_only=True, required=False) + self.fields["srid"] = serializers.CharField(required=False) + self.fields["group"] = DynamicRelationField(GroupSerializer, embed=True, many=False) + self.fields["popular_count"] = serializers.CharField(required=False) + self.fields["share_count"] = serializers.CharField(required=False) + self.fields["rating"] = serializers.CharField(required=False) + self.fields["featured"] = serializers.BooleanField(required=False) + self.fields["is_published"] = serializers.BooleanField(required=False, read_only=True) + self.fields["is_approved"] = serializers.BooleanField(required=False, read_only=True) + self.fields["detail_url"] = DetailUrlField(read_only=True) + self.fields["created"] = serializers.DateTimeField(read_only=True) + self.fields["last_updated"] = serializers.DateTimeField(read_only=True) + self.fields["raw_abstract"] = serializers.CharField(read_only=True) + self.fields["raw_purpose"] = serializers.CharField(read_only=True) + self.fields["raw_constraints_other"] = serializers.CharField(read_only=True) + self.fields["raw_supplemental_information"] = serializers.CharField(read_only=True) + self.fields["raw_data_quality_statement"] = serializers.CharField(read_only=True) + self.fields["metadata_only"] = serializers.BooleanField(required=False) + self.fields["processed"] = serializers.BooleanField(read_only=True) + self.fields["state"] = serializers.CharField(read_only=True) + self.fields["sourcetype"] = serializers.CharField(read_only=True) + + self.fields["embed_url"] = EmbedUrlField(required=False) + self.fields["thumbnail_url"] = ThumbnailUrlField(read_only=True) + self.fields["keywords"] = DynamicRelationField(SimpleHierarchicalKeywordSerializer, embed=False, many=True) + self.fields["tkeywords"] = DynamicRelationField(SimpleThesaurusKeywordSerializer, embed=False, many=True) + self.fields["regions"] = DynamicRelationField(SimpleRegionSerializer, embed=True, many=True, read_only=True) + self.fields["category"] = DynamicRelationField(SimpleTopicCategorySerializer, embed=True, many=False) + self.fields["restriction_code_type"] = DynamicRelationField( + RestrictionCodeTypeSerializer, embed=True, many=False + ) + self.fields["license"] = DynamicRelationField(LicenseSerializer, embed=True, many=False) + self.fields["spatial_representation_type"] = DynamicRelationField( + SpatialRepresentationTypeSerializer, embed=True, many=False + ) + self.fields["blob"] = serializers.JSONField(required=False, write_only=True) + self.fields["is_copyable"] = serializers.BooleanField(read_only=True) + + self.fields["download_url"] = DownloadLinkField(read_only=True) + + self.fields["favorite"] = FavoriteField(read_only=True) metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True) class Meta: model = ResourceBase - name = 'resource' - view_name = 'base-resources-list' + name = "resource" + view_name = "base-resources-list" fields = ( - 'pk', 'uuid', 'resource_type', 'polymorphic_ctype_id', 'perms', - 'owner', 'poc', 'metadata_author', - 'keywords', 'tkeywords', 'regions', 'category', - 'title', 'abstract', 'attribution', 'alternate', 'doi', 'bbox_polygon', 'll_bbox_polygon', 'srid', - 'date', 'date_type', 'edition', 'purpose', 'maintenance_frequency', - 'restriction_code_type', 'constraints_other', 'license', 'language', - 'spatial_representation_type', 'temporal_extent_start', 'temporal_extent_end', - 'supplemental_information', 'data_quality_statement', 'group', - 'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved', - 'detail_url', 'embed_url', 'created', 'last_updated', - 'raw_abstract', 'raw_purpose', 'raw_constraints_other', - 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed', 'state', - 'data', 'subtype', 'sourcetype', 'is_copyable', - 'blob', "metadata", 'executions' + "pk", + "uuid", + "resource_type", + "polymorphic_ctype_id", + "perms", + "owner", + "poc", + "metadata_author", + "keywords", + "tkeywords", + "regions", + "category", + "title", + "abstract", + "attribution", + "alternate", + "doi", + "bbox_polygon", + "ll_bbox_polygon", + "srid", + "date", + "date_type", + "edition", + "purpose", + "maintenance_frequency", + "restriction_code_type", + "constraints_other", + "license", + "language", + "spatial_representation_type", + "temporal_extent_start", + "temporal_extent_end", + "supplemental_information", + "data_quality_statement", + "group", + "popular_count", + "share_count", + "rating", + "featured", + "is_published", + "is_approved", + "detail_url", + "embed_url", + "created", + "last_updated", + "raw_abstract", + "raw_purpose", + "raw_constraints_other", + "raw_supplemental_information", + "raw_data_quality_statement", + "metadata_only", + "processed", + "state", + "data", + "subtype", + "sourcetype", + "is_copyable", + "blob", + "metadata", + "executions" # TODO # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry, # metadata_uploaded, metadata_uploaded_preserve, metadata_xml, @@ -576,8 +595,8 @@ class Meta: def to_internal_value(self, data): if isinstance(data, str): data = json.loads(data) - if 'data' in data: - data['blob'] = data.pop('data') + if "data" in data: + data["blob"] = data.pop("data") data = super(ResourceBaseSerializer, self).to_internal_value(data) return data @@ -586,7 +605,7 @@ def to_internal_value(self, data): """ data = DataBlobField( DataBlobSerializer, - source='id', + source="id", many=False, embed=False, deferred=True, @@ -598,7 +617,7 @@ def to_internal_value(self, data): """ executions = DynamicRelationField( ResourceExecutionRequestSerializer, - source='id', + source="id", embed=False, deferred=True, required=False, @@ -611,12 +630,12 @@ class FavoriteSerializer(DynamicModelSerializer): class Meta: model = Favorite - name = 'favorites' - fields = 'resource', + name = "favorites" + fields = ("resource",) def to_representation(self, value): data = super().to_representation(value) - return data['resource'] + return data["resource"] def get_resource(self, instance): resource = ResourceBase.objects.get(pk=instance.object_id) @@ -624,73 +643,66 @@ def get_resource(self, instance): class BaseResourceCountSerializer(BaseDynamicModelSerializer): - def to_representation(self, instance): - request = self.context.get('request') + request = self.context.get("request") filter_options = {} if request.query_params: filter_options = { - 'type_filter': request.query_params.get('type'), - 'title_filter': request.query_params.get('title__icontains') + "type_filter": request.query_params.get("type"), + "title_filter": request.query_params.get("title__icontains"), } data = super().to_representation(instance) if not isinstance(data, int): try: count_filter = {self.Meta.count_type: instance} - data['count'] = get_resources_with_perms( - request.user, filter_options).filter(**count_filter).count() + data["count"] = get_resources_with_perms(request.user, filter_options).filter(**count_filter).count() except (TypeError, NoReverseMatch) as e: logger.exception(e) return data class HierarchicalKeywordSerializer(BaseResourceCountSerializer): - class Meta: - name = 'keywords' + name = "keywords" model = HierarchicalKeyword - count_type = 'keywords' - view_name = 'keywords-list' - fields = '__all__' + count_type = "keywords" + view_name = "keywords-list" + fields = "__all__" class ThesaurusKeywordSerializer(_ThesaurusKeywordSerializerMixIn, BaseResourceCountSerializer): - class Meta: model = ThesaurusKeyword - name = 'tkeywords' - view_name = 'tkeywords-list' - count_type = 'tkeywords' - fields = '__all__' + name = "tkeywords" + view_name = "tkeywords-list" + count_type = "tkeywords" + fields = "__all__" class RegionSerializer(BaseResourceCountSerializer): - class Meta: - name = 'regions' + name = "regions" model = Region - count_type = 'regions' - view_name = 'regions-list' - fields = '__all__' + count_type = "regions" + view_name = "regions-list" + fields = "__all__" class TopicCategorySerializer(BaseResourceCountSerializer): - class Meta: - name = 'categories' + name = "categories" model = TopicCategory - count_type = 'category' - view_name = 'categories-list' - fields = '__all__' + count_type = "category" + view_name = "categories-list" + fields = "__all__" class OwnerSerializer(BaseResourceCountSerializer): - class Meta: - name = 'owners' - count_type = 'owner' - view_name = 'owners-list' + name = "owners" + count_type = "owner" + view_name = "owners-list" model = get_user_model() - fields = ('pk', 'username', 'first_name', 'last_name', 'avatar', 'perms') + fields = ("pk", "username", "first_name", "last_name", "avatar", "perms") avatar = AvatarUrlField(240, read_only=True) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 19eb884a0ca..c07fecae534 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -54,7 +54,7 @@ ResourceBase, TopicCategory, ThesaurusKeyword, - ExtraMetadata + ExtraMetadata, ) from geonode.layers.models import Dataset @@ -68,23 +68,18 @@ logger = logging.getLogger(__name__) -test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) +test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) class BaseApiTests(APITestCase): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json', - "test_thesaurus.json" - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json", "test_thesaurus.json"] def setUp(self): self.maxDiff = None - create_models(b'document') - create_models(b'map') - create_models(b'dataset') + create_models(b"document") + create_models(b"map") + create_models(b"dataset") def test_groups_list(self): """ @@ -97,39 +92,39 @@ def test_groups_list(self): pub_invite_2 = GroupProfile.objects.create(slug="pub_invite_2", title="pub_invite_2", access="public-invite") try: # Anonymous can access only public groups - url = reverse('group-profiles-list') - response = self.client.get(url, format='json') + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) logger.debug(response.data) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 4) - self.assertEqual(len(response.data['group_profiles']), 4) - self.assertTrue(all([_g['access'] != 'private' for _g in response.data['group_profiles']])) + self.assertEqual(response.data["total"], 4) + self.assertEqual(len(response.data["group_profiles"]), 4) + self.assertTrue(all([_g["access"] != "private" for _g in response.data["group_profiles"]])) # Admin can access all groups - self.assertTrue(self.client.login(username='admin', password='admin')) - url = reverse('group-profiles-list') - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) logger.debug(response.data) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 6) - self.assertEqual(len(response.data['group_profiles']), 6) + self.assertEqual(response.data["total"], 6) + self.assertEqual(len(response.data["group_profiles"]), 6) # Bobby can access public groups and the ones he is member of - self.assertTrue(self.client.login(username='bobby', password='bob')) - priv_1.join(get_user_model().objects.get(username='bobby')) - url = reverse('group-profiles-list') - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + priv_1.join(get_user_model().objects.get(username="bobby")) + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) logger.debug(response.data) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 5) - self.assertEqual(len(response.data['group_profiles']), 5) - self.assertTrue(any([_g['slug'] == 'priv_1' for _g in response.data['group_profiles']])) + self.assertEqual(response.data["total"], 5) + self.assertEqual(len(response.data["group_profiles"]), 5) + self.assertTrue(any([_g["slug"] == "priv_1" for _g in response.data["group_profiles"]])) - url = reverse('group-profiles-detail', kwargs={'pk': priv_1.pk}) - response = self.client.get(url, format='json') + url = reverse("group-profiles-detail", kwargs={"pk": priv_1.pk}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) logger.debug(response.data) finally: @@ -149,33 +144,33 @@ def test_create_group(self): "slug": "group_title", "description": "test", "access": "private", - "categories": [] + "categories": [], } try: # Anonymous - url = reverse('group-profiles-list') - response = self.client.post(url, data=data, format='json') + url = reverse("group-profiles-list") + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Registered member - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.post(url, data=data, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Group manager group = GroupProfile.objects.create(slug="test_group_manager", title="test_group_manager") - group.join(get_user_model().objects.get(username='norman'), role='manager') - self.assertTrue(self.client.login(username='norman', password='norman')) - response = self.client.post(url, data=data, format='json') + group.join(get_user_model().objects.get(username="norman"), role="manager") + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.post(url, data=data, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 201) - self.assertEqual(response.json()['group_profile']['title'], 'group title') + self.assertEqual(response.json()["group_profile"]["title"], "group title") finally: - GroupProfile.objects.get(slug='group_title').delete() + GroupProfile.objects.get(slug="group_title").delete() group.delete() def test_edit_group(self): @@ -183,29 +178,29 @@ def test_edit_group(self): Ensure only admins and group managers can edit a group. """ group = GroupProfile.objects.create(slug="pub_1", title="pub_1", access="public") - data = {'title': 'new_title'} + data = {"title": "new_title"} try: # Anonymous url = f"{reverse('group-profiles-list')}/{group.id}/" - response = self.client.patch(url, data=data, format='json') + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Registered member - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.patch(url, data=data, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Group manager - group.join(get_user_model().objects.get(username='bobby'), role='manager') - response = self.client.patch(url, data=data, format='json') + group.join(get_user_model().objects.get(username="bobby"), role="manager") + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(GroupProfile.objects.get(id=group.id).title, data['title']) + self.assertEqual(GroupProfile.objects.get(id=group.id).title, data["title"]) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.patch(url, data={'title': 'admin_title'}, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.patch(url, data={"title": "admin_title"}, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(GroupProfile.objects.get(id=group.id).title, 'admin_title') + self.assertEqual(GroupProfile.objects.get(id=group.id).title, "admin_title") finally: group.delete() @@ -217,48 +212,43 @@ def test_delete_group(self): try: # Anonymous url = f"{reverse('group-profiles-list')}/{group.id}/" - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 403) # Registered member - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.delete(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 403) # Group manager - group.join(get_user_model().objects.get(username='bobby'), role='manager') - response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format='json') + group.join(get_user_model().objects.get(username="bobby"), role="manager") + response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format="json") self.assertEqual(response.status_code, 403) # Admin can delete a group - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format="json") self.assertEqual(response.status_code, 204) finally: group.delete() def test_group_resources_shows_related_permissions(self): - ''' + """ Calling the resources endpoint of the groups should return also the group permission on that specific resource - ''' + """ group = GroupProfile.objects.create(slug="group1", title="group1", access="public") - bobby = get_user_model().objects.filter(username='bobby').get() + bobby = get_user_model().objects.filter(username="bobby").get() GroupMember.objects.get_or_create(group=group, user=bobby, role="member") dataset = Dataset.objects.first() - dataset.set_permissions( - {'groups': {group: ['base.view_resourcebase']}} - ) + dataset.set_permissions({"groups": {group: ["base.view_resourcebase"]}}) try: - self.assertTrue(self.client.login(username='bobby', password='bob')) + self.assertTrue(self.client.login(username="bobby", password="bob")) url = f"{reverse('group-profiles-list')}/{group.id}/resources" - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) perms = response.json().get("resources", [])[0].get("perms") - self.assertListEqual( - ["view_resourcebase"], - perms - ) + self.assertListEqual(["view_resourcebase"], perms) finally: group.delete() @@ -266,105 +256,99 @@ def test_users_list(self): """ Ensure we can access the users list. """ - url = reverse('users-list') + url = reverse("users-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 403) # Authorized - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) logger.debug(response.data) - self.assertEqual(response.data['total'], 9) - self.assertEqual(len(response.data['users']), 9) + self.assertEqual(response.data["total"], 9) + self.assertEqual(len(response.data["users"]), 9) # response has link to the response - self.assertTrue('link' in response.data['users'][0].keys()) + self.assertTrue("link" in response.data["users"][0].keys()) - url = reverse('users-detail', kwargs={'pk': 1}) - response = self.client.get(url, format='json') + url = reverse("users-detail", kwargs={"pk": 1}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) logger.debug(response.data) - self.assertEqual(response.data['user']['username'], 'admin') - self.assertIsNotNone(response.data['user']['avatar']) + self.assertEqual(response.data["user"]["username"], "admin") + self.assertIsNotNone(response.data["user"]["avatar"]) # anonymous users are not in contributors group - self.assertNotIn('add_resource', get_user_model().objects.get(id=-1).perms) + self.assertNotIn("add_resource", get_user_model().objects.get(id=-1).perms) try: # Bobby - group_user = get_user_model().objects.create(username='group_user') - bobby = get_user_model().objects.filter(username='bobby').get() + group_user = get_user_model().objects.create(username="group_user") + bobby = get_user_model().objects.filter(username="bobby").get() groupx = GroupProfile.objects.create(slug="groupx", title="groupx", access="private") groupx.join(bobby) groupx.join(group_user) - self.assertTrue(self.client.login(username='bobby', password='bob')) - url = reverse('users-detail', kwargs={'pk': group_user.id}) + self.assertTrue(self.client.login(username="bobby", password="bob")) + url = reverse("users-detail", kwargs={"pk": group_user.id}) # Bobby can access other users details from same group - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) # Bobby can see himself in the list - url = reverse('users-list') - response = self.client.get(url, format='json') + url = reverse("users-list") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertIn('"username": "bobby"', json.dumps(response.data['users'])) + self.assertIn('"username": "bobby"', json.dumps(response.data["users"])) # Bobby can access its own details - url = reverse('users-detail', kwargs={'pk': bobby.id}) - response = self.client.get(url, format='json') + url = reverse("users-detail", kwargs={"pk": bobby.id}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['user']['username'], 'bobby') - self.assertIsNotNone(response.data['user']['avatar']) + self.assertEqual(response.data["user"]["username"], "bobby") + self.assertIsNotNone(response.data["user"]["avatar"]) # default contributor group_perm is returned in perms - self.assertIn('add_resource', response.data['user']['perms']) + self.assertIn("add_resource", response.data["user"]["perms"]) # Bobby can't access other users perms list - url = reverse('users-detail', kwargs={'pk': group_user.id}) - response = self.client.get(url, format='json') + url = reverse("users-detail", kwargs={"pk": group_user.id}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['user']['username'], 'group_user') - self.assertIsNotNone(response.data['user']['avatar']) + self.assertEqual(response.data["user"]["username"], "group_user") + self.assertIsNotNone(response.data["user"]["avatar"]) # default contributor group_perm is returned in perms - self.assertNotIn('perms', response.data['user']) + self.assertNotIn("perms", response.data["user"]) finally: group_user.delete() groupx.delete() def test_user_resources_shows_related_permissions(self): - ''' + """ Calling the resources endpoint of the user should return also the user permission on that specific resource - ''' - bobby = get_user_model().objects.filter(username='bobby').get() + """ + bobby = get_user_model().objects.filter(username="bobby").get() dataset = Dataset.objects.first() - dataset.set_permissions( - {'users': {bobby: ['base.view_resourcebase', 'base.change_resourcebase']}} - ) - self.assertTrue(self.client.login(username='bobby', password='bob')) + dataset.set_permissions({"users": {bobby: ["base.view_resourcebase", "base.change_resourcebase"]}}) + self.assertTrue(self.client.login(username="bobby", password="bob")) url = f"{reverse('users-list')}/{bobby.id}/resources" - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) perms = response.json().get("resources", [])[0].get("perms") - self.assertSetEqual( - {"view_resourcebase", "change_resourcebase"}, - set(perms) - ) + self.assertSetEqual({"view_resourcebase", "change_resourcebase"}, set(perms)) def test_get_self_user_details_outside_registered_member(self): try: user = get_user_model().objects.create_user( - username='non_registered_member', - email="non_registered_member@geonode.org", - password='password') + username="non_registered_member", email="non_registered_member@geonode.org", password="password" + ) # remove user from registered members group - reg_mem_group = Group.objects.get(name='registered-members') + reg_mem_group = Group.objects.get(name="registered-members") reg_mem_group.user_set.remove(user) - url = reverse('users-detail', kwargs={'pk': user.pk}) + url = reverse("users-detail", kwargs={"pk": user.pk}) self.assertTrue(self.client.login(username="non_registered_member", password="password")) - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) finally: user.delete() @@ -372,16 +356,15 @@ def test_get_self_user_details_outside_registered_member(self): def test_get_self_user_details_with_no_group(self): try: user = get_user_model().objects.create_user( - username='no_group_member', - email="no_group_member@geonode.org", - password='password') + username="no_group_member", email="no_group_member@geonode.org", password="password" + ) # remove user from all groups user.groups.clear() - url = reverse('users-detail', kwargs={'pk': user.pk}) + url = reverse("users-detail", kwargs={"pk": user.pk}) self.assertTrue(self.client.login(username="no_group_member", password="password")) - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) finally: user.delete() @@ -390,18 +373,18 @@ def test_register_users(self): """ Ensure users are created with default groups. """ - url = reverse('users-list') + url = reverse("users-list") user_data = { - 'username': 'new_user', + "username": "new_user", } self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.post(url, data=user_data, format='json') + response = self.client.post(url, data=user_data, format="json") self.assertEqual(response.status_code, 201) # default contributor group_perm is returned in perms - self.assertIn('add_resource', response.data['user']['perms']) + self.assertIn("add_resource", response.data["user"]["perms"]) # Anonymous self.assertIsNone(self.client.logout()) - response = self.client.post(url, data={'username': 'new_user_1'}, format='json') + response = self.client.post(url, data={"username": "new_user_1"}, format="json") self.assertEqual(response.status_code, 403) def test_update_user_profile(self): @@ -410,34 +393,33 @@ def test_update_user_profile(self): """ try: user = get_user_model().objects.create_user( - username='user_test_delete', - email="user_test_delete@geonode.org", - password='user') - url = reverse('users-detail', kwargs={'pk': user.pk}) - data = {'first_name': 'user'} + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) + data = {"first_name": "user"} # Anonymous - response = self.client.patch(url, data=data, format='json') + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Another registered user self.assertTrue(self.client.login(username="bobby", password="bob")) - response = self.client.patch(url, data=data, format='json') + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 403) # User self profile self.assertTrue(self.client.login(username="user_test_delete", password="user")) - response = self.client.patch(url, data=data, format='json') + response = self.client.patch(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Group manager group = GroupProfile.objects.create(slug="test_group_manager", title="test_group_manager") group.join(user) - group.join(get_user_model().objects.get(username='norman'), role='manager') - self.assertTrue(self.client.login(username='norman', password='norman')) - response = self.client.post(url, data=data, format='json') + group.join(get_user_model().objects.get(username="norman"), role="manager") + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 403) # Admin can edit user self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.patch(url, data={'first_name': 'user_admin'}, format='json') + response = self.client.patch(url, data={"first_name": "user_admin"}, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(get_user_model().objects.get(username='user_test_delete').first_name, 'user_admin') + self.assertEqual(get_user_model().objects.get(username="user_test_delete").first_name, "user_admin") finally: user.delete() group.delete() @@ -448,27 +430,26 @@ def test_delete_user_profile(self): """ try: user = get_user_model().objects.create_user( - username='user_test_delete', - email="user_test_delete@geonode.org", - password='user') - url = reverse('users-detail', kwargs={'pk': user.pk}) + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) # Anonymous can't read - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 403) # Anonymous can't delete user - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 403) # Bob can't delete user self.assertTrue(self.client.login(username="bobby", password="bob")) - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 403) # User can not delete self profile self.assertTrue(self.client.login(username="user_test_delete", password="user")) - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 403) # Admin can delete user self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 204) finally: user.delete() @@ -477,143 +458,144 @@ def test_base_resources(self): """ Ensure we can access the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 28) + self.assertEqual(response.data["total"], 28) url = f"{reverse('base-resources-list')}?filter{{metadata_only}}=false" # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) - response.data['resources'][0].get('executions') + self.assertEqual(response.data["total"], 26) + response.data["resources"][0].get("executions") # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) logger.debug(response.data) # Remove public permissions to Layers from geonode.layers.utils import set_datasets_permissions + set_datasets_permissions( "read", # permissions_name None, # resources_names == None (all layers) [get_anonymous_user()], # users_usernames None, # groups_names - True, # delete_flag + True, # delete_flag ) - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # response has link to the response - self.assertTrue('link' in response.data['resources'][0].keys()) + self.assertTrue("link" in response.data["resources"][0].keys()) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) logger.debug(response.data) # Bobby - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) logger.debug(response.data) # Norman - self.assertTrue(self.client.login(username='norman', password='norman')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) logger.debug(response.data) # Pagination # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get(f"{url}&page_size=17", format='json') + response = self.client.get(f"{url}&page_size=17", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 17) + self.assertEqual(len(response.data["resources"]), 17) # Check user permissions - resource = ResourceBase.objects.filter(owner__username='bobby').first() - self.assertEqual(resource.owner.username, 'bobby') + resource = ResourceBase.objects.filter(owner__username="bobby").first() + self.assertEqual(resource.owner.username, "bobby") # Admin url_with_id = f"{reverse('base-resources-list')}/{resource.id}?filter{{metadata_only}}=false" - response = self.client.get(f"{url_with_id}", format='json') - self.assertEqual(response.data['resource']['state'], enumerations.STATE_PROCESSED) - self.assertEqual(response.data['resource']['sourcetype'], enumerations.SOURCE_TYPE_LOCAL) - self.assertTrue('change_resourcebase' in list(response.data['resource']['perms'])) + response = self.client.get(f"{url_with_id}", format="json") + self.assertEqual(response.data["resource"]["state"], enumerations.STATE_PROCESSED) + self.assertEqual(response.data["resource"]["sourcetype"], enumerations.SOURCE_TYPE_LOCAL) + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) # Annonymous self.assertIsNone(self.client.logout()) - response = self.client.get(f"{url_with_id}", format='json') - self.assertFalse('change_resourcebase' in list(response.data['resource']['perms'])) + response = self.client.get(f"{url_with_id}", format="json") + self.assertFalse("change_resourcebase" in list(response.data["resource"]["perms"])) # user owner - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(f"{url_with_id}", format='json') - self.assertTrue('change_resourcebase' in list(response.data['resource']['perms'])) + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) # user not owner and not assigned - self.assertTrue(self.client.login(username='norman', password='norman')) - response = self.client.get(f"{url_with_id}", format='json') - self.assertFalse('change_resourcebase' in list(response.data['resource']['perms'])) + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertFalse("change_resourcebase" in list(response.data["resource"]["perms"])) # Check executions are returned when deffered # all resources - response = self.client.get(f'{url}&include[]=executions', format='json') + response = self.client.get(f"{url}&include[]=executions", format="json") self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data['resources'][0].get('executions')) + self.assertIsNotNone(response.data["resources"][0].get("executions")) # specific resource exec_req = ExecutionRequest.objects.create( user=resource.owner, - func_name='test', + func_name="test", geonode_resource=resource, input_params={ "uuid": resource.uuid, "owner": resource.owner.username, "resource_type": resource.resource_type, - "defaults": f"{{\"owner\":\"{resource.owner.username}\"}}" - } + "defaults": f'{{"owner":"{resource.owner.username}"}}', + }, ) - expected_executions_results = [{ - 'exec_id': exec_req.exec_id, - 'user': exec_req.user.username, - 'status': exec_req.status, - 'func_name': exec_req.func_name, - 'created': exec_req.created, - 'finished': exec_req.finished, - 'last_updated': exec_req.last_updated, - 'input_params': exec_req.input_params, - 'output_params': exec_req.output_params, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': exec_req.exec_id}) - ) - }] - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(f'{url_with_id}&include[]=executions', format='json') + expected_executions_results = [ + { + "exec_id": exec_req.exec_id, + "user": exec_req.user.username, + "status": exec_req.status, + "func_name": exec_req.func_name, + "created": exec_req.created, + "finished": exec_req.finished, + "last_updated": exec_req.last_updated, + "input_params": exec_req.input_params, + "output_params": exec_req.output_params, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": exec_req.exec_id}) + ), + } + ] + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url_with_id}&include[]=executions", format="json") self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data['resource'].get('executions')) - self.assertEqual(response.data['resource'].get('executions'), expected_executions_results) + self.assertIsNotNone(response.data["resource"].get("executions")) + self.assertEqual(response.data["resource"].get("executions"), expected_executions_results) # test 'tkeywords' try: @@ -621,83 +603,72 @@ def test_base_resources(self): resource.tkeywords.add(_tkw) self.assertEqual(6, resource.tkeywords.count()) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(f"{url_with_id}", format='json') - self.assertIsNotNone(response.data['resource']['tkeywords']) - self.assertEqual(6, len(response.data['resource']['tkeywords'])) + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertIsNotNone(response.data["resource"]["tkeywords"]) + self.assertEqual(6, len(response.data["resource"]["tkeywords"])) self.assertListEqual( [ { - 'name': '', - 'slug': 'http-inspire-ec-europa-eu-theme-37', - 'uri': 'http://inspire.ec.europa.eu/theme#37', - 'thesaurus': { - 'name': 'GEMET - INSPIRE themes, version 1.0', - 'slug': 'inspire-theme', - 'uri': 'http://inspire.ec.europa.eu/theme' + "name": "", + "slug": "http-inspire-ec-europa-eu-theme-37", + "uri": "http://inspire.ec.europa.eu/theme#37", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", }, - 'i18n': {} + "i18n": {}, }, { - 'name': '', - 'slug': 'http-localhost-8001-thesaurus-no-about-thesauro-38', - 'uri': 'http://localhost:8001//thesaurus/no-about-thesauro#38', - 'thesaurus': { - 'name': 'Thesauro without the about', - 'slug': 'no-about-thesauro', - 'uri': '' - }, - 'i18n': {} + "name": "", + "slug": "http-localhost-8001-thesaurus-no-about-thesauro-38", + "uri": "http://localhost:8001//thesaurus/no-about-thesauro#38", + "thesaurus": {"name": "Thesauro without the about", "slug": "no-about-thesauro", "uri": ""}, + "i18n": {}, }, { - 'name': 'bar_keyword', - 'slug': 'http-localhost-8001-thesaurus-no-about-thesauro-bar-keyword', - 'uri': 'http://localhost:8001//thesaurus/no-about-thesauro#bar_keyword', - 'thesaurus': { - 'name': 'Thesauro without the about', - 'slug': 'no-about-thesauro', 'uri': '' - }, - 'i18n': {} + "name": "bar_keyword", + "slug": "http-localhost-8001-thesaurus-no-about-thesauro-bar-keyword", + "uri": "http://localhost:8001//thesaurus/no-about-thesauro#bar_keyword", + "thesaurus": {"name": "Thesauro without the about", "slug": "no-about-thesauro", "uri": ""}, + "i18n": {}, }, { - 'name': 'foo_keyword', - 'slug': 'http-inspire-ec-europa-eu-theme-foo-keyword', - 'uri': 'http://inspire.ec.europa.eu/theme#foo_keyword', - 'thesaurus': { - 'name': 'GEMET - INSPIRE themes, version 1.0', - 'slug': 'inspire-theme', - 'uri': 'http://inspire.ec.europa.eu/theme' + "name": "foo_keyword", + "slug": "http-inspire-ec-europa-eu-theme-foo-keyword", + "uri": "http://inspire.ec.europa.eu/theme#foo_keyword", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", }, - 'i18n': {} + "i18n": {}, }, { - 'name': 'mf', - 'slug': 'http-inspire-ec-europa-eu-theme-mf', - 'uri': 'http://inspire.ec.europa.eu/theme/mf', - 'thesaurus': { - 'name': 'GEMET - INSPIRE themes, version 1.0', - 'slug': 'inspire-theme', - 'uri': 'http://inspire.ec.europa.eu/theme' + "name": "mf", + "slug": "http-inspire-ec-europa-eu-theme-mf", + "uri": "http://inspire.ec.europa.eu/theme/mf", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", }, - 'i18n': { - 'en': 'Meteorological geographical features' - } + "i18n": {"en": "Meteorological geographical features"}, }, { - 'name': 'us', - 'slug': 'http-inspire-ec-europa-eu-theme-us', - 'uri': 'http://inspire.ec.europa.eu/theme/us', - 'thesaurus': { - 'name': 'GEMET - INSPIRE themes, version 1.0', - 'slug': 'inspire-theme', - 'uri': 'http://inspire.ec.europa.eu/theme' + "name": "us", + "slug": "http-inspire-ec-europa-eu-theme-us", + "uri": "http://inspire.ec.europa.eu/theme/us", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", }, - 'i18n': { - 'en': 'Utility and governmental services' - } - } + "i18n": {"en": "Utility and governmental services"}, + }, ], - response.data['resource']['tkeywords'] + response.data["resource"]["tkeywords"], ) finally: resource.tkeywords.set(ThesaurusKeyword.objects.none()) @@ -707,191 +678,201 @@ def test_write_resources(self): """ Ensure we can perform write oprtation afainst the Resource Bases. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Check user permissions - for resource_type in ['dataset', 'document', 'map']: - resource = ResourceBase.objects.filter(owner__username='bobby', resource_type=resource_type).first() - self.assertEqual(resource.owner.username, 'bobby') - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(f"{url}/{resource.id}/", format='json') - self.assertTrue('change_resourcebase' in list(response.data['resource']['perms'])) - self.assertEqual(True, response.data['resource']['is_published'], response.data['resource']['is_published']) + for resource_type in ["dataset", "document", "map"]: + resource = ResourceBase.objects.filter(owner__username="bobby", resource_type=resource_type).first() + self.assertEqual(resource.owner.username, "bobby") + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url}/{resource.id}/", format="json") + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) + self.assertEqual(True, response.data["resource"]["is_published"], response.data["resource"]["is_published"]) data = { "title": "Foo Title", "abstract": "Foo Abstract", "attribution": "Foo Attribution", "doi": "321-12345-987654321", - "is_published": False # this is a read-only field so should not updated + "is_published": False, # this is a read-only field so should not updated } - response = self.client.patch(f"{url}/{resource.id}/", data=data, format='json') + response = self.client.patch(f"{url}/{resource.id}/", data=data, format="json") self.assertEqual(response.status_code, 200, response.status_code) - response = self.client.get(f"{url}/{resource.id}/", format='json') - self.assertEqual('Foo Title', response.data['resource']['title'], response.data['resource']['title']) - self.assertEqual('Foo Abstract', response.data['resource']['abstract'], response.data['resource']['abstract']) - self.assertEqual('Foo Attribution', response.data['resource']['attribution'], response.data['resource']['attribution']) - self.assertEqual('321-12345-987654321', response.data['resource']['doi'], response.data['resource']['doi']) - self.assertEqual(True, response.data['resource']['is_published'], response.data['resource']['is_published']) + response = self.client.get(f"{url}/{resource.id}/", format="json") + self.assertEqual("Foo Title", response.data["resource"]["title"], response.data["resource"]["title"]) + self.assertEqual( + "Foo Abstract", response.data["resource"]["abstract"], response.data["resource"]["abstract"] + ) + self.assertEqual( + "Foo Attribution", response.data["resource"]["attribution"], response.data["resource"]["attribution"] + ) + self.assertEqual("321-12345-987654321", response.data["resource"]["doi"], response.data["resource"]["doi"]) + self.assertEqual(True, response.data["resource"]["is_published"], response.data["resource"]["is_published"]) def test_delete_user_with_resource(self): - owner, created = get_user_model().objects.get_or_create(username='delet-owner') + owner, created = get_user_model().objects.get_or_create(username="delet-owner") Dataset( - title='Test Remove User', - abstract='abstract', - name='Test Remove User', - alternate='Test Remove User', + title="Test Remove User", + abstract="abstract", + name="Test Remove User", + alternate="Test Remove User", uuid=str(uuid4()), owner=owner, - subtype='raster', - category=TopicCategory.objects.get(identifier='elevation') + subtype="raster", + category=TopicCategory.objects.get(identifier="elevation"), ).save() # Delete user and check if default user is updated owner.delete() self.assertEqual( - ResourceBase.objects.get(title='Test Remove User').owner, - get_user_model().objects.get(username='admin') + ResourceBase.objects.get(title="Test Remove User").owner, get_user_model().objects.get(username="admin") ) def test_search_resources(self): """ Ensure we can search across the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get( - f"{url}?search=ca&search_fields=title&search_fields=abstract", format='json') + response = self.client.get(f"{url}?search=ca&search_fields=title&search_fields=abstract", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # Pagination - self.assertEqual(len(response.data['resources']), 1) + self.assertEqual(len(response.data["resources"]), 1) def test_filter_resources(self): """ Ensure we can filter across the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) # Filter by owner == bobby - response = self.client.get(f"{url}?filter{{owner.username}}=bobby&filter{{metadata_only}}=false", format='json') + response = self.client.get(f"{url}?filter{{owner.username}}=bobby&filter{{metadata_only}}=false", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 3) + self.assertEqual(response.data["total"], 3) # Pagination - self.assertEqual(len(response.data['resources']), 3) + self.assertEqual(len(response.data["resources"]), 3) # Filter by resource_type == document - response = self.client.get(f"{url}?filter{{resource_type}}=document&filter{{metadata_only}}=false", format='json') + response = self.client.get( + f"{url}?filter{{resource_type}}=document&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 9) + self.assertEqual(response.data["total"], 9) # Pagination - self.assertEqual(len(response.data['resources']), 9) + self.assertEqual(len(response.data["resources"]), 9) # Filter by resource_type == layer and title like 'common morx' response = self.client.get( - f"{url}?filter{{resource_type}}=dataset&filter{{title.icontains}}=common morx&filter{{metadata_only}}=false", format='json') + f"{url}?filter{{resource_type}}=dataset&filter{{title.icontains}}=common morx&filter{{metadata_only}}=false", + format="json", + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # Pagination - self.assertEqual(len(response.data['resources']), 1) + self.assertEqual(len(response.data["resources"]), 1) # Filter by Keywords - response = self.client.get( - f"{url}?filter{{keywords.name}}=here&filter{{metadata_only}}=false", format='json') + response = self.client.get(f"{url}?filter{{keywords.name}}=here&filter{{metadata_only}}=false", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # Pagination - self.assertEqual(len(response.data['resources']), 1) + self.assertEqual(len(response.data["resources"]), 1) # Filter by Metadata Regions response = self.client.get( - f"{url}?filter{{regions.name.icontains}}=Italy&filter{{metadata_only}}=false", format='json') + f"{url}?filter{{regions.name.icontains}}=Italy&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 0) + self.assertEqual(response.data["total"], 0) # Pagination - self.assertEqual(len(response.data['resources']), 0) + self.assertEqual(len(response.data["resources"]), 0) # Filter by Metadata Categories response = self.client.get( - f"{url}?filter{{category.identifier}}=elevation&filter{{metadata_only}}=false", format='json') + f"{url}?filter{{category.identifier}}=elevation&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 6) + self.assertEqual(response.data["total"], 6) # Pagination - self.assertEqual(len(response.data['resources']), 6) + self.assertEqual(len(response.data["resources"]), 6) # Extent Filter - response = self.client.get(f"{url}?page_size=26&extent=-180,-90,180,90&filter{{metadata_only}}=false", format='json') + response = self.client.get( + f"{url}?page_size=26&extent=-180,-90,180,90&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 26) + self.assertEqual(len(response.data["resources"]), 26) - response = self.client.get(f"{url}?page_size=26&extent=0,0,100,100&filter{{metadata_only}}=false", format='json') + response = self.client.get( + f"{url}?page_size=26&extent=0,0,100,100&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 26) + self.assertEqual(response.data["total"], 26) # Pagination - self.assertEqual(len(response.data['resources']), 26) + self.assertEqual(len(response.data["resources"]), 26) - response = self.client.get(f"{url}?page_size=26&extent=-10,-10,-1,-1&filter{{metadata_only}}=false", format='json') + response = self.client.get( + f"{url}?page_size=26&extent=-10,-10,-1,-1&filter{{metadata_only}}=false", format="json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 12) + self.assertEqual(response.data["total"], 12) # Pagination - self.assertEqual(len(response.data['resources']), 12) + self.assertEqual(len(response.data["resources"]), 12) # Extent Filter: Crossing Dateline extent = "-180.0000,56.9689,-162.5977,70.7435,155.9180,56.9689,180.0000,70.7435" - response = self.client.get( - f"{url}?page_size=26&extent={extent}&filter{{metadata_only}}=false", format='json') + response = self.client.get(f"{url}?page_size=26&extent={extent}&filter{{metadata_only}}=false", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 12) + self.assertEqual(response.data["total"], 12) # Pagination - self.assertEqual(len(response.data['resources']), 12) + self.assertEqual(len(response.data["resources"]), 12) def test_sort_resources(self): """ Ensure we can sort the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get( - f"{url}?sort[]=title", format='json') + response = self.client.get(f"{url}?sort[]=title", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 28) + self.assertEqual(response.data["total"], 28) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) resource_titles = [] - for _r in response.data['resources']: - resource_titles.append(_r['title']) + for _r in response.data["resources"]: + resource_titles.append(_r["title"]) sorted_resource_titles = sorted(resource_titles.copy()) self.assertEqual(resource_titles, sorted_resource_titles) - response = self.client.get( - f"{url}?sort[]=-title", format='json') + response = self.client.get(f"{url}?sort[]=-title", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 28) + self.assertEqual(response.data["total"], 28) # Pagination - self.assertEqual(len(response.data['resources']), 10) + self.assertEqual(len(response.data["resources"]), 10) resource_titles = [] - for _r in response.data['resources']: - resource_titles.append(_r['title']) + for _r in response.data["resources"]: + resource_titles.append(_r["title"]) reversed_resource_titles = sorted(resource_titles.copy()) self.assertNotEqual(resource_titles, reversed_resource_titles) @@ -900,93 +881,88 @@ def test_perms_resources(self): """ Ensure we can Get & Set Permissions across the Resource Base list. """ - url = reverse('base-resources-list') - bobby = get_user_model().objects.get(username='bobby') - norman = get_user_model().objects.get(username='norman') - anonymous_group = Group.objects.get(name='anonymous') - contributors_group = Group.objects.get(name='registered-members') + url = reverse("base-resources-list") + bobby = get_user_model().objects.get(username="bobby") + norman = get_user_model().objects.get(username="norman") + anonymous_group = Group.objects.get(name="anonymous") + contributors_group = Group.objects.get(name="registered-members") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - resource = ResourceBase.objects.filter(owner__username='bobby').first() - set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", 'permissions') - get_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", 'permissions') + resource = ResourceBase.objects.filter(owner__username="bobby").first() + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + get_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") - url = reverse('base-resources-detail', kwargs={'pk': resource.pk}) - response = self.client.get(url, format='json') + url = reverse("base-resources-detail", kwargs={"pk": resource.pk}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(int(response.data['resource']['pk']), int(resource.pk)) + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) - response = self.client.get(get_perms_url, format='json') + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertDictEqual( resource_perm_spec, { - 'users': [ - { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'owner', - 'is_staff': False, - 'is_superuser': False, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "owner", + "is_staff": False, + "is_superuser": False, }, { - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'first_name': 'admin', - 'id': 1, - 'last_name': '', - 'permissions': 'manage', - 'username': 'admin', - 'is_staff': True, - 'is_superuser': True, - } + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, ], - 'organizations': [], - 'groups': [ - { - 'id': anonymous_group.id, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'download' + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "download"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", }, - { - 'id': contributors_group.id, - 'title': 'Registered Members', - 'name': contributors_group.name, - 'permissions': 'none' - } - ] + ], }, - resource_perm_spec + resource_perm_spec, ) # Add perms to Norman resource_perm_spec_patch = { "uuid": resource.uuid, - 'users': [ + "users": [ { - 'id': norman.id, - 'username': norman.username, - 'first_name': norman.first_name, - 'last_name': norman.last_name, - 'avatar': '', - 'permissions': 'edit', - 'is_staff': False, - 'is_superuser': False, + "id": norman.id, + "username": norman.username, + "first_name": norman.first_name, + "last_name": norman.last_name, + "avatar": "", + "permissions": "edit", + "is_staff": False, + "is_superuser": False, } - ] + ], } response = self.client.patch(set_perms_url, data=resource_perm_spec_patch, format="json") self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('status')) - self.assertIsNotNone(response.data.get('status_url')) - status = response.data.get('status') - resp_js = json.loads(response.content.decode('utf-8')) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) status_url = resp_js.get("status_url", None) execution_id = resp_js.get("execution_id", "") self.assertIsNotNone(status_url) @@ -994,7 +970,7 @@ def test_perms_resources(self): for _cnt in range(0, 10): response = self.client.get(f"{status_url}") self.assertEqual(response.status_code, 200) - resp_js = json.loads(response.content.decode('utf-8')) + resp_js = json.loads(response.content.decode("utf-8")) logger.error(f"[{_cnt + 1}] ... {resp_js}") if resp_js.get("status", "") == "finished": status = resp_js.get("status", "") @@ -1004,111 +980,101 @@ def test_perms_resources(self): sleep(3.0) self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) - response = self.client.get(get_perms_url, format='json') + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertDictEqual( resource_perm_spec, { - 'users': [ - { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'owner', - 'is_staff': False, - 'is_superuser': False, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "owner", + "is_staff": False, + "is_superuser": False, }, { - 'id': norman.id, - 'username': norman.username, - 'first_name': norman.first_name, - 'last_name': norman.last_name, - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'edit', - 'is_staff': False, - 'is_superuser': False, + "id": norman.id, + "username": norman.username, + "first_name": norman.first_name, + "last_name": norman.last_name, + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "edit", + "is_staff": False, + "is_superuser": False, }, { - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'first_name': 'admin', - 'id': 1, - 'last_name': '', - 'permissions': 'manage', - 'username': 'admin', - 'is_staff': True, - 'is_superuser': True, - } + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, ], - 'organizations': [], - 'groups': [ - { - 'id': anonymous_group.id, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'download' + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "download"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", }, - { - 'id': contributors_group.id, - 'title': 'Registered Members', - 'name': contributors_group.name, - 'permissions': 'none' - } - ] + ], }, - resource_perm_spec + resource_perm_spec, ) # Remove perms to Norman resource_perm_spec = { "uuid": resource.uuid, - 'users': [ + "users": [ { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'owner', - 'is_staff': False, - 'is_superuser': False, + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "owner", + "is_staff": False, + "is_superuser": False, }, { - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'first_name': 'admin', - 'id': 1, - 'last_name': '', - 'permissions': 'manage', - 'username': 'admin', - 'is_staff': True, - 'is_superuser': True, - } + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, ], - 'organizations': [], - 'groups': [ + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "view"}, { - 'id': anonymous_group.id, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'view' + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", }, - { - 'id': contributors_group.id, - 'title': 'Registered Members', - 'name': contributors_group.name, - 'permissions': 'none' - } - ] + ], } response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('status')) - self.assertIsNotNone(response.data.get('status_url')) - status = response.data.get('status') - resp_js = json.loads(response.content.decode('utf-8')) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) status_url = resp_js.get("status_url", None) execution_id = resp_js.get("execution_id", "") self.assertIsNotNone(status_url) @@ -1116,7 +1082,7 @@ def test_perms_resources(self): for _cnt in range(0, 10): response = self.client.get(f"{status_url}") self.assertEqual(response.status_code, 200) - resp_js = json.loads(response.content.decode('utf-8')) + resp_js = json.loads(response.content.decode("utf-8")) logger.error(f"[{_cnt + 1}] ... {resp_js}") if resp_js.get("status", "") == "finished": status = resp_js.get("status", "") @@ -1126,67 +1092,62 @@ def test_perms_resources(self): sleep(3.0) self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) - response = self.client.get(get_perms_url, format='json') + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertDictEqual( resource_perm_spec, { - 'users': [ - { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'owner', - 'is_staff': False, - 'is_superuser': False, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "owner", + "is_staff": False, + "is_superuser": False, }, { - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'first_name': 'admin', - 'id': 1, - 'last_name': '', - 'permissions': 'manage', - 'username': 'admin', - 'is_staff': True, - 'is_superuser': True, - } + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, ], - 'organizations': [], - 'groups': [ - { - 'id': anonymous_group.id, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'view' + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", }, - { - 'id': contributors_group.id, - 'title': 'Registered Members', - 'name': contributors_group.name, - 'permissions': 'none' - } - ] + ], }, - resource_perm_spec + resource_perm_spec, ) # Ensure get_perms and set_perms are done by users with correct permissions. # logout admin user self.assertIsNone(self.client.logout()) # get perms - response = self.client.get(get_perms_url, format='json') + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 403) # set perms - resource_perm_spec['uuid'] = resource.uuid + resource_perm_spec["uuid"] = resource.uuid response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") self.assertEqual(response.status_code, 403) # login resourse owner # get perms - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(get_perms_url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) # set perms response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") @@ -1196,51 +1157,48 @@ def test_featured_and_published_resources(self): """ Ensure we can Get & Set Permissions across the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - resources = ResourceBase.objects.filter(owner__username='bobby', metadata_only=False) + resources = ResourceBase.objects.filter(owner__username="bobby", metadata_only=False) - url = urljoin(f"{reverse('base-resources-list')}/", 'featured/') - response = self.client.get(url, format='json') + url = urljoin(f"{reverse('base-resources-list')}/", "featured/") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 0) + self.assertEqual(response.data["total"], 0) # Pagination - self.assertEqual(len(response.data['resources']), 0) + self.assertEqual(len(response.data["resources"]), 0) - resources.filter(resource_type='map').update(featured=True) - url = urljoin(f"{reverse('base-resources-list')}/", 'featured/') - response = self.client.get(url, format='json') + resources.filter(resource_type="map").update(featured=True) + url = urljoin(f"{reverse('base-resources-list')}/", "featured/") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], resources.filter(resource_type='map').count()) + self.assertEqual(response.data["total"], resources.filter(resource_type="map").count()) # Pagination - self.assertEqual(len(response.data['resources']), resources.filter(resource_type='map').count()) + self.assertEqual(len(response.data["resources"]), resources.filter(resource_type="map").count()) def test_resource_types(self): """ Ensure we can Get & Set Permissions across the Resource Base list. """ - url = urljoin(f"{reverse('base-resources-list')}/", 'resource_types/') - response = self.client.get(url, format='json') - r_types = [item for item in response.data['resource_types']] - r_type_names = [r_type['name'] for r_type in r_types] + url = urljoin(f"{reverse('base-resources-list')}/", "resource_types/") + response = self.client.get(url, format="json") + r_types = [item for item in response.data["resource_types"]] + r_type_names = [r_type["name"] for r_type in r_types] self.assertEqual(response.status_code, 200) - self.assertTrue('resource_types' in response.data) - self.assertTrue('dataset' in r_type_names) - self.assertTrue('map' in r_type_names) - self.assertTrue('document' in r_type_names) - self.assertFalse('service' in r_type_names) + self.assertTrue("resource_types" in response.data) + self.assertTrue("dataset" in r_type_names) + self.assertTrue("map" in r_type_names) + self.assertTrue("document" in r_type_names) + self.assertFalse("service" in r_type_names) - r_type_perms = {r_type['name']: r_type['allowed_perms'] for r_type in r_types} + r_type_perms = {r_type["name"]: r_type["allowed_perms"] for r_type in r_types} _pp_e = { "perms": { - "anonymous": [ - "view_resourcebase", - "download_resourcebase" - ], + "anonymous": ["view_resourcebase", "download_resourcebase"], "default": [ "delete_resourcebase", "view_resourcebase", @@ -1251,7 +1209,7 @@ def test_resource_types(self): "change_resourcebase_metadata", "download_resourcebase", "change_dataset_data", - "change_dataset_style" + "change_dataset_style", ], "registered-members": [ "delete_resourcebase", @@ -1262,95 +1220,53 @@ def test_resource_types(self): "change_resourcebase_metadata", "download_resourcebase", "change_dataset_data", - "change_dataset_style" - ] + "change_dataset_style", + ], }, "compact": { "anonymous": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View" - }, - { - "name": "download", - "label": "Download" - } + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, ], "default": [ - { - "name": "view", - "label": "View" - }, - { - "name": "download", - "label": "Download" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - }, - { - "name": "owner", - "label": "Owner" - }, - { - "name": "owner", - "label": "Owner" - } + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, ], "registered-members": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View" - }, - { - "name": "download", - "label": "Download" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - } - ] - } + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, } self.assertListEqual( - list(r_type_perms['dataset'].keys()), + list(r_type_perms["dataset"].keys()), list(_pp_e.keys()), - f"dataset : {list(r_type_perms['dataset'].keys())}") - for _key in r_type_perms['dataset']['perms'].keys(): + f"dataset : {list(r_type_perms['dataset'].keys())}", + ) + for _key in r_type_perms["dataset"]["perms"].keys(): self.assertListEqual( - sorted(list(set(r_type_perms['dataset']['perms'].get(_key)))), - sorted(list(set(_pp_e['perms'].get(_key)))), - f"{_key} : {list(set(r_type_perms['dataset']['perms'].get(_key)))}") - for _key in r_type_perms['dataset']['compact'].keys(): + sorted(list(set(r_type_perms["dataset"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['dataset']['perms'].get(_key)))}", + ) + for _key in r_type_perms["dataset"]["compact"].keys(): self.assertListEqual( - r_type_perms['dataset']['compact'].get(_key), - _pp_e['compact'].get(_key), - f"{_key} : {r_type_perms['dataset']['compact'].get(_key)}") + r_type_perms["dataset"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['dataset']['compact'].get(_key)}", + ) _pp_e = { "perms": { - "anonymous": [ - "view_resourcebase", - "download_resourcebase" - ], + "anonymous": ["view_resourcebase", "download_resourcebase"], "default": [ "change_resourcebase_metadata", "delete_resourcebase", @@ -1358,7 +1274,7 @@ def test_resource_types(self): "publish_resourcebase", "change_resourcebase", "view_resourcebase", - "download_resourcebase" + "download_resourcebase", ], "registered-members": [ "change_resourcebase_metadata", @@ -1367,88 +1283,49 @@ def test_resource_types(self): "publish_resourcebase", "change_resourcebase", "view_resourcebase", - "download_resourcebase" - ] + "download_resourcebase", + ], }, "compact": { "anonymous": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View Metadata" - }, - { - "name": "download", - "label": "View and Download" - } + {"name": "none", "label": "None"}, + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, ], "default": [ - { - "name": "view", - "label": "View Metadata" - }, - { - "name": "download", - "label": "View and Download" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - }, - { - "name": "owner", - "label": "Owner" - }, - { - "name": "owner", - "label": "Owner" - } + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, ], "registered-members": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View Metadata" - }, - { - "name": "download", - "label": "View and Download" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - } - ] - } + {"name": "none", "label": "None"}, + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, } self.assertListEqual( - list(r_type_perms['document'].keys()), + list(r_type_perms["document"].keys()), list(_pp_e.keys()), - f"document : {list(r_type_perms['document'].keys())}") - for _key in r_type_perms['document']['perms'].keys(): + f"document : {list(r_type_perms['document'].keys())}", + ) + for _key in r_type_perms["document"]["perms"].keys(): self.assertListEqual( - sorted(list(set(r_type_perms['document']['perms'].get(_key)))), - sorted(list(set(_pp_e['perms'].get(_key)))), - f"{_key} : {list(set(r_type_perms['document']['perms'].get(_key)))}") - for _key in r_type_perms['document']['compact'].keys(): + sorted(list(set(r_type_perms["document"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['document']['perms'].get(_key)))}", + ) + for _key in r_type_perms["document"]["compact"].keys(): self.assertListEqual( - r_type_perms['document']['compact'].get(_key), - _pp_e['compact'].get(_key), - f"{_key} : {r_type_perms['document']['compact'].get(_key)}") + r_type_perms["document"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['document']['compact'].get(_key)}", + ) _pp_e = { "perms": { @@ -1461,7 +1338,7 @@ def test_resource_types(self): "change_resourcebase_permissions", "publish_resourcebase", "change_resourcebase", - "view_resourcebase" + "view_resourcebase", ], "registered-members": [ "change_resourcebase_metadata", @@ -1469,92 +1346,57 @@ def test_resource_types(self): "change_resourcebase_permissions", "publish_resourcebase", "change_resourcebase", - "view_resourcebase" - ] + "view_resourcebase", + ], }, "compact": { - "anonymous": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View" - } - ], + "anonymous": [{"name": "none", "label": "None"}, {"name": "view", "label": "View"}], "default": [ - { - "name": "view", - "label": "View" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - }, - { - "name": "owner", - "label": "Owner" - }, - { - "name": "owner", - "label": "Owner" - } + {"name": "view", "label": "View"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, ], "registered-members": [ - { - "name": "none", - "label": "None" - }, - { - "name": "view", - "label": "View" - }, - { - "name": "edit", - "label": "Edit" - }, - { - "name": "manage", - "label": "Manage" - } - ] - } + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, } self.assertListEqual( - list(r_type_perms['map'].keys()), - list(_pp_e.keys()), - f"map : {list(r_type_perms['map'].keys())}") - for _key in r_type_perms['map']['perms'].keys(): + list(r_type_perms["map"].keys()), list(_pp_e.keys()), f"map : {list(r_type_perms['map'].keys())}" + ) + for _key in r_type_perms["map"]["perms"].keys(): self.assertListEqual( - sorted(list(set(r_type_perms['map']['perms'].get(_key)))), - sorted(list(set(_pp_e['perms'].get(_key)))), - f"{_key} : {list(set(r_type_perms['map']['perms'].get(_key)))}") - for _key in r_type_perms['map']['compact'].keys(): + sorted(list(set(r_type_perms["map"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['map']['perms'].get(_key)))}", + ) + for _key in r_type_perms["map"]["compact"].keys(): self.assertListEqual( - r_type_perms['map']['compact'].get(_key), - _pp_e['compact'].get(_key), - f"{_key} : {r_type_perms['map']['compact'].get(_key)}") + r_type_perms["map"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['map']['compact'].get(_key)}", + ) def test_get_favorites(self): """ Ensure we get user's favorite resources. """ dataset = Dataset.objects.first() - url = urljoin(f"{reverse('base-resources-list')}/", 'favorites/') + url = urljoin(f"{reverse('base-resources-list')}/", "favorites/") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 403) # Authenticated user - bobby = get_user_model().objects.get(username='bobby') - self.assertTrue(self.client.login(username='bobby', password='bob')) + bobby = get_user_model().objects.get(username="bobby") + self.assertTrue(self.client.login(username="bobby", password="bob")) favorite = Favorite.objects.create_favorite(dataset, bobby) - response = self.client.get(url, format='json') - self.assertEqual(response.data['total'], 1) + response = self.client.get(url, format="json") + self.assertEqual(response.data["total"], 1) self.assertEqual(response.status_code, 200) # clean up favorite.delete() @@ -1563,28 +1405,28 @@ def test_get_favorites_is_returned_in_the_base_endpoint_per_user(self): """ Ensure we get user's favorite resources. """ - dataset = Dataset.objects.order_by('-last_updated').first() - url = reverse('base-resources-list') - bobby = get_user_model().objects.get(username='bobby') + dataset = Dataset.objects.order_by("-last_updated").first() + url = reverse("base-resources-list") + bobby = get_user_model().objects.get(username="bobby") - self.client.login(username='bobby', password='bob') + self.client.login(username="bobby", password="bob") favorite = Favorite.objects.create_favorite(dataset, bobby) - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - resource_have_tag = [r.get('favorite', False) for r in response.json().get("resources", {})] + resource_have_tag = [r.get("favorite", False) for r in response.json().get("resources", {})] # check that there is at last 1 favorite for the user self.assertTrue(any(resource_have_tag)) # clean up favorite.delete() - self.client.login(username='admin', password='admin') - response = self.client.get(url, format='json') + self.client.login(username="admin", password="admin") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - resource_have_tag = [r.get('favorite', False) for r in response.json().get("resources", {})] + resource_have_tag = [r.get("favorite", False) for r in response.json().get("resources", {})] # the admin should not have any favorite assigned to him self.assertFalse(all(resource_have_tag)) @@ -1592,33 +1434,28 @@ def test_get_favorites_is_returned_in_the_base_endpoint(self): """ Ensure we get user's favorite resources. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - resource_have_tag = ['favorite' in r.keys() for r in response.json().get("resources", {})] + resource_have_tag = ["favorite" in r.keys() for r in response.json().get("resources", {})] self.assertTrue(all(resource_have_tag)) def test_create_and_delete_favorites(self): """ Ensure we can add and remove resources to user's favorite. """ - bobby = get_user_model().objects.get(username='bobby') + bobby = get_user_model().objects.get(username="bobby") dataset = create_single_dataset(name="test_dataset_for_fav", owner=bobby) - dataset.set_permissions( - {'users': { - "bobby": ['base.add_resourcebase'] - } - } - ) + dataset.set_permissions({"users": {"bobby": ["base.add_resourcebase"]}}) url = urljoin(f"{reverse('base-resources-list')}/", f"{dataset.pk}/favorite/") # Anonymous - response = self.client.post(url, format='json') + response = self.client.post(url, format="json") self.assertEqual(response.status_code, 403) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) + self.assertTrue(self.client.login(username="bobby", password="bob")) response = self.client.post(url, format="json") self.assertEqual(response.data["message"], "Successfuly added resource to favorites") self.assertEqual(response.status_code, 201) @@ -1640,57 +1477,50 @@ def test_search_resources_with_favorite_true_and_no_favorite_should_return_0(sel """ Ensure we can search across the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get( - f"{url}?favorite=true", format='json') + response = self.client.get(f"{url}?favorite=true", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) # No favorite are saved, so the total should be 0 - self.assertEqual(response.data['total'], 0) - self.assertEqual(len(response.data['resources']), 0) + self.assertEqual(response.data["total"], 0) + self.assertEqual(len(response.data["resources"]), 0) def test_search_resources_with_favorite_true_and_favorite_should_return_1(self): """ Ensure we can search across the Resource Base list. """ - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - admin = get_user_model().objects.get(username='admin') + admin = get_user_model().objects.get(username="admin") dataset = Dataset.objects.first() Favorite.objects.create_favorite(dataset, admin) - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get( - f"{url}?favorite=true", format='json') + response = self.client.get(f"{url}?favorite=true", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) # 1 favorite is saved, so the total should be 1 - self.assertEqual(response.data['total'], 1) - self.assertEqual(len(response.data['resources']), 1) + self.assertEqual(response.data["total"], 1) + self.assertEqual(len(response.data["resources"]), 1) def test_search_resources_with_favorite_true_with_geoapps_icluded(self): - url = reverse('base-resources-list') + url = reverse("base-resources-list") # Admin - admin = get_user_model().objects.get(username='admin') + admin = get_user_model().objects.get(username="admin") try: - geo_app = GeoApp.objects.create( - title="Test GeoApp Favorite", - owner=admin, - resource_type="geostory" - ) + geo_app = GeoApp.objects.create(title="Test GeoApp Favorite", owner=admin, resource_type="geostory") Favorite.objects.create_favorite(geo_app, admin) - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - response = self.client.get( - f"{url}?favorite=true", format='json') + response = self.client.get(f"{url}?favorite=true", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 1) - self.assertEqual(len(response.data['resources']), 1) + self.assertEqual(response.data["total"], 1) + self.assertEqual(len(response.data["resources"]), 1) finally: geo_app.delete() @@ -1699,14 +1529,14 @@ def test_thumbnail_urls(self): Ensure the thumbnail url reflects the current active Thumb on the resource. """ # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) - resource = ResourceBase.objects.filter(owner__username='bobby').first() - url = reverse('base-resources-detail', kwargs={'pk': resource.pk}) - response = self.client.get(url, format='json') + resource = ResourceBase.objects.filter(owner__username="bobby").first() + url = reverse("base-resources-detail", kwargs={"pk": resource.pk}) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(int(response.data['resource']['pk']), int(resource.pk)) - thumbnail_url = response.data['resource']['thumbnail_url'] + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) + thumbnail_url = response.data["resource"]["thumbnail_url"] self.assertIsNone(thumbnail_url) def test_embed_urls(self): @@ -1714,22 +1544,22 @@ def test_embed_urls(self): Ensure the embed urls reflect the concrete instance ones. """ # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) resources = ResourceBase.objects.all() for resource in resources: url = f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}?filter{{metadata_only}}=false" - response = self.client.get(url, format='json') - if resource.title.endswith('metadata true'): + response = self.client.get(url, format="json") + if resource.title.endswith("metadata true"): self.assertEqual(response.status_code, 404) else: self.assertEqual(response.status_code, 200) - self.assertEqual(int(response.data['resource']['pk']), int(resource.pk)) - embed_url = response.data['resource']['embed_url'] + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) + embed_url = response.data["resource"]["embed_url"] self.assertIsNotNone(embed_url) instance = resource.get_real_instance() - if hasattr(instance, 'embed_url'): + if hasattr(instance, "embed_url"): if instance.embed_url != NotImplemented: self.assertEqual(build_absolute_uri(instance.embed_url), embed_url) else: @@ -1739,169 +1569,169 @@ def test_owners_list(self): """ Ensure we can access the list of owners. """ - url = reverse('owners-list') + url = reverse("owners-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 8) + self.assertEqual(response.data["total"], 8) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 8) - response = self.client.get(f"{url}?type=geoapp", format='json') + self.assertEqual(response.data["total"], 8) + response = self.client.get(f"{url}?type=geoapp", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 0) - response = self.client.get(f"{url}?type=dataset&title__icontains=CA", format='json') + self.assertEqual(response.data["total"], 0) + response = self.client.get(f"{url}?type=dataset&title__icontains=CA", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # response has link to the response - self.assertTrue('link' in response.data['owners'][0].keys()) + self.assertTrue("link" in response.data["owners"][0].keys()) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 8) + self.assertEqual(response.data["total"], 8) # Owners Filtering - response = self.client.get(f"{url}?filter{{username.icontains}}=bobby", format='json') + response = self.client.get(f"{url}?filter{{username.icontains}}=bobby", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) def test_categories_list(self): """ Ensure we can access the list of categories. """ - url = reverse('categories-list') + url = reverse("categories-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], TopicCategory.objects.count()) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], TopicCategory.objects.count()) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) # response has link to the response - self.assertTrue('link' in response.data['categories'][0].keys()) + self.assertTrue("link" in response.data["categories"][0].keys()) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], TopicCategory.objects.count()) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) # Categories Filtering - response = self.client.get(f"{url}?filter{{identifier.icontains}}=biota", format='json') + response = self.client.get(f"{url}?filter{{identifier.icontains}}=biota", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) def test_regions_list(self): """ Ensure we can access the list of regions. """ - url = reverse('regions-list') + url = reverse("regions-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], Region.objects.count()) + self.assertEqual(response.data["total"], Region.objects.count()) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], Region.objects.count()) + self.assertEqual(response.data["total"], Region.objects.count()) # response has link to the response - self.assertTrue('link' in response.data['regions'][0].keys()) + self.assertTrue("link" in response.data["regions"][0].keys()) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], Region.objects.count()) + self.assertEqual(response.data["total"], Region.objects.count()) # Regions Filtering - response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format='json') + response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 8) + self.assertEqual(response.data["total"], 8) def test_regions_with_resources(self): """ Ensure we can access the list of regions. """ - self.assertTrue(self.client.login(username='admin', password='admin')) - url = reverse('regions-list') - response = self.client.get(f"{url}?with_resources=True", format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}?with_resources=True", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 0) + self.assertEqual(response.data["total"], 0) - self.assertTrue(self.client.login(username='admin', password='admin')) - url = reverse('regions-list') - response = self.client.get(f"{url}?with_resources=False", format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}?with_resources=False", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], Region.objects.count()) + self.assertEqual(response.data["total"], Region.objects.count()) - self.assertTrue(self.client.login(username='admin', password='admin')) - url = reverse('regions-list') - response = self.client.get(f"{url}", format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], Region.objects.count()) + self.assertEqual(response.data["total"], Region.objects.count()) def test_keywords_list(self): """ Ensure we can access the list of keywords. """ - url = reverse('keywords-list') + url = reverse("keywords-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], len(HierarchicalKeyword.resource_keywords_tree(None))) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(None))) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - admin = get_user_model().objects.get(username='admin') - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + admin = get_user_model().objects.get(username="admin") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], len(HierarchicalKeyword.resource_keywords_tree(admin))) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(admin))) # response has link to the response - if response.data['total'] > 0: - self.assertTrue('link' in response.data['keywords'][0].keys()) + if response.data["total"] > 0: + self.assertTrue("link" in response.data["keywords"][0].keys()) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - bobby = get_user_model().objects.get(username='bobby') - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + bobby = get_user_model().objects.get(username="bobby") + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], len(HierarchicalKeyword.resource_keywords_tree(bobby))) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(bobby))) # Keywords Filtering - response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format='json') + response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], 0) + self.assertEqual(response.data["total"], 0) # Testing Hierarchical Keywords tree try: - HierarchicalKeyword.objects.filter(slug__in=['a', 'a1', 'a2', 'b', 'b1']).delete() - HierarchicalKeyword.add_root(name='a') - HierarchicalKeyword.add_root(name='b') - a = HierarchicalKeyword.objects.get(slug='a') - b = HierarchicalKeyword.objects.get(slug='b') - a.add_child(name='a1') - a.add_child(name='a2') - b.add_child(name='b1') - resources = ResourceBase.objects.filter(owner__username='bobby') + HierarchicalKeyword.objects.filter(slug__in=["a", "a1", "a2", "b", "b1"]).delete() + HierarchicalKeyword.add_root(name="a") + HierarchicalKeyword.add_root(name="b") + a = HierarchicalKeyword.objects.get(slug="a") + b = HierarchicalKeyword.objects.get(slug="b") + a.add_child(name="a1") + a.add_child(name="a2") + b.add_child(name="b1") + resources = ResourceBase.objects.filter(owner__username="bobby") res1 = resources.first() res2 = resources.last() self.assertNotEqual(res1, res2) - res1.keywords.add(HierarchicalKeyword.objects.get(slug='a1')) - res1.keywords.add(HierarchicalKeyword.objects.get(slug='a2')) - res2.keywords.add(HierarchicalKeyword.objects.get(slug='b1')) + res1.keywords.add(HierarchicalKeyword.objects.get(slug="a1")) + res1.keywords.add(HierarchicalKeyword.objects.get(slug="a2")) + res2.keywords.add(HierarchicalKeyword.objects.get(slug="b1")) resource_keywords = HierarchicalKeyword.resource_keywords_tree(bobby) logger.error(resource_keywords) @@ -1945,95 +1775,89 @@ def test_keywords_list(self): ... ] """ - url = reverse('keywords-list') + url = reverse("keywords-list") # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) for _kw in response.data["keywords"]: - if _kw.get('href') in ['a', 'b']: - self.assertListEqual(_kw.get('tags'), [], _kw.get('tags')) - self.assertEqual(len(_kw.get('nodes')), 2) - for _kw_child in _kw.get('nodes'): - self.assertEqual(_kw_child.get('tags')[0], 1) + if _kw.get("href") in ["a", "b"]: + self.assertListEqual(_kw.get("tags"), [], _kw.get("tags")) + self.assertEqual(len(_kw.get("nodes")), 2) + for _kw_child in _kw.get("nodes"): + self.assertEqual(_kw_child.get("tags")[0], 1) finally: - HierarchicalKeyword.objects.filter(slug__in=['a', 'a1', 'a2', 'b', 'b1']).delete() + HierarchicalKeyword.objects.filter(slug__in=["a", "a1", "a2", "b", "b1"]).delete() def test_tkeywords_list(self): """ Ensure we can access the list of thasaurus keywords. """ - url = reverse('tkeywords-list') + url = reverse("tkeywords-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], ThesaurusKeyword.objects.count()) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], ThesaurusKeyword.objects.count()) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) # response has uri to the response - self.assertTrue('uri' in response.data['tkeywords'][0].keys()) + self.assertTrue("uri" in response.data["tkeywords"][0].keys()) # Authenticated user - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['total'], ThesaurusKeyword.objects.count()) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) def test_rating_resource(self): resource = Dataset.objects.first() - url = reverse('base-resources-ratings', args=[resource.pk]) + url = reverse("base-resources-ratings", args=[resource.pk]) resource.set_permissions( - {'users': { - get_anonymous_user().username: ['base.view_resourcebase'], - "bobby": ['base.add_resourcebase'] - } - } + {"users": {get_anonymous_user().username: ["base.view_resourcebase"], "bobby": ["base.add_resourcebase"]}} ) - data = { - "rating": 3 - } + data = {"rating": 3} # Anonymous user response = self.client.get(url) - self.assertEqual(response.json()['rating'], 0) - self.assertEqual(response.json()['overall_rating'], 0) + self.assertEqual(response.json()["rating"], 0) + self.assertEqual(response.json()["overall_rating"], 0) self.assertEqual(response.status_code, 200) response = self.client.post(url, data=data) self.assertEqual(response.status_code, 403) # Authenticated user - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) response = self.client.get(url) - self.assertEqual(response.json()['rating'], 0) - self.assertEqual(response.json()['overall_rating'], 0) + self.assertEqual(response.json()["rating"], 0) + self.assertEqual(response.json()["overall_rating"], 0) self.assertEqual(response.status_code, 200) response = self.client.post(url, data=data) - self.assertEqual(response.json()['rating'], 3) - self.assertEqual(response.json()['overall_rating'], 3.0) + self.assertEqual(response.json()["rating"], 3) + self.assertEqual(response.json()["overall_rating"], 3.0) self.assertEqual(response.status_code, 200) # Authenticated user2 - self.assertTrue(self.client.login(username='bobby', password='bob')) + self.assertTrue(self.client.login(username="bobby", password="bob")) response = self.client.get(url) - self.assertEqual(response.json()['rating'], 0) - self.assertEqual(response.json()['overall_rating'], 3.0) + self.assertEqual(response.json()["rating"], 0) + self.assertEqual(response.json()["overall_rating"], 3.0) self.assertEqual(response.status_code, 200) - data['rating'] = 1 + data["rating"] = 1 response = self.client.post(url, data=data) - self.assertEqual(response.json()['rating'], 1) - self.assertEqual(response.json()['overall_rating'], 2.0) + self.assertEqual(response.json()["rating"], 1) + self.assertEqual(response.json()["overall_rating"], 2.0) self.assertEqual(response.status_code, 200) def test_set_resource_thumbnail(self): re_uuid = "[0-F]{8}-([0-F]{4}-){3}[0-F]{12}" resource = Dataset.objects.first() - url = reverse('base-resources-set_thumbnail', args=[resource.pk]) + url = reverse("base-resources-set_thumbnail", args=[resource.pk]) data = {"file": "http://localhost:8000/thumb.png"} # Anonymous user @@ -2041,45 +1865,57 @@ def test_set_resource_thumbnail(self): self.assertEqual(response.status_code, 403) # Authenticated user - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) response = self.client.put(url, data=data, format="json") self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()['thumbnail_url'], data['file']) - self.assertEqual(Dataset.objects.get(pk=resource.pk).thumbnail_url, data['file']) + self.assertEqual(response.json()["thumbnail_url"], data["file"]) + self.assertEqual(Dataset.objects.get(pk=resource.pk).thumbnail_url, data["file"]) # set with invalid image url data = {"file": "invali url"} response = self.client.put(url, data=data, format="json") self.assertEqual(response.status_code, 400) expected = { - 'success': False, - 'errors': ['file is either a file upload, ASCII byte string or a valid image url string'], - 'code': 'invalid' + "success": False, + "errors": ["file is either a file upload, ASCII byte string or a valid image url string"], + "code": "invalid", } self.assertEqual(response.json(), expected) # Test with non image url data = {"file": "http://localhost:8000/thumb.txt"} response = self.client.put(url, data=data, format="json") self.assertEqual(response.status_code, 400) - self.assertEqual(response.json(), 'The url must be of an image with format (png, jpeg or jpg)') + self.assertEqual(response.json(), "The url must be of an image with format (png, jpeg or jpg)") # using Base64 data as an ASCII byte string - data['file'] = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ + data[ + "file" + ] = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ fAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAANSURBVAiZYzAxMfkPAALYAZzx61+bAAAAAElFTkSuQmCC" with patch("geonode.base.models.is_monochromatic_image") as _mck: _mck.return_value = False response = self.client.put(url, data=data, format="json") self.assertEqual(response.status_code, 200) - self.assertIsNotNone(re.search(f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", Dataset.objects.get(pk=resource.pk).thumbnail_url, re.I)) + self.assertIsNotNone( + re.search( + f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", Dataset.objects.get(pk=resource.pk).thumbnail_url, re.I + ) + ) # File upload - with patch('PIL.Image.open') as _mck: + with patch("PIL.Image.open") as _mck: _mck.return_value = test_image # rest thumbnail_url to None resource.thumbnail_url = None resource.save() self.assertEqual(Dataset.objects.get(pk=resource.pk).thumbnail_url, None) - f = SimpleUploadedFile('test_image.png', BytesIO(test_image.tobytes()).read(), 'image/png') + f = SimpleUploadedFile("test_image.png", BytesIO(test_image.tobytes()).read(), "image/png") response = self.client.put(url, data={"file": f}) - self.assertIsNotNone(re.search(f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", Dataset.objects.get(pk=resource.pk).thumbnail_url, re.I)) + self.assertIsNotNone( + re.search( + f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", + Dataset.objects.get(pk=resource.pk).thumbnail_url, + re.I, + ) + ) self.assertEqual(response.status_code, 200) def test_set_thumbnail_from_bbox_from_Anonymous_user_raise_permission_error(self): @@ -2087,14 +1923,14 @@ def test_set_thumbnail_from_bbox_from_Anonymous_user_raise_permission_error(self Given a request with Anonymous user, should raise an authentication error. """ dataset_id = sys.maxsize - url = reverse('base-resources-set-thumb-from-bbox', args=[dataset_id]) + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) # Anonymous expected = { "success": False, "errors": ["Authentication credentials were not provided."], "code": "not_authenticated", } - response = self.client.post(url, format='json') + response = self.client.post(url, format="json") self.assertEqual(response.status_code, 403) self.assertEqual(expected, response.json()) @@ -2107,22 +1943,17 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset(self, moc # Admin self.client.login(username="admin", password="admin") dataset_id = Dataset.objects.first().resourcebase_ptr_id - url = reverse('base-resources-set-thumb-from-bbox', args=[dataset_id]) + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) payload = { - "bbox": [ - -9072629.904175375, - -9043966.018568434, - 1491839.8773032012, - 1507127.2829602365 - ], - "srid": "EPSG:3857" + "bbox": [-9072629.904175375, -9043966.018568434, 1491839.8773032012, 1507127.2829602365], + "srid": "EPSG:3857", } - response = self.client.post(url, data=payload, format='json') + response = self.client.post(url, data=payload, format="json") expected = { - 'message': 'Thumbnail correctly created.', - 'success': True, - 'thumbnail_url': 'http://localhost:8000/mocked_url.jpg' + "message": "Thumbnail correctly created.", + "success": True, + "thumbnail_url": "http://localhost:8000/mocked_url.jpg", } self.assertEqual(response.status_code, 200) self.assertEqual(expected, response.json()) @@ -2134,22 +1965,14 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_not_existing_dataset(self) # Admin self.client.login(username="admin", password="admin") dataset_id = sys.maxsize - url = reverse('base-resources-set-thumb-from-bbox', args=[dataset_id]) + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) payload = { - "bbox": [ - -9072629.904175375, - -9043966.018568434, - 1491839.8773032012, - 1507127.2829602365 - ], - "srid": "EPSG:3857" + "bbox": [-9072629.904175375, -9043966.018568434, 1491839.8773032012, 1507127.2829602365], + "srid": "EPSG:3857", } - response = self.client.post(url, data=payload, format='json') + response = self.client.post(url, data=payload, format="json") - expected = { - 'message': f'Resource selected with id {dataset_id} does not exists', - 'success': False - } + expected = {"message": f"Resource selected with id {dataset_id} does not exists", "success": False} self.assertEqual(response.status_code, 404) self.assertEqual(expected, response.json()) @@ -2160,39 +1983,29 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_doc(self): # Admin self.client.login(username="admin", password="admin") dataset_id = Document.objects.first().resourcebase_ptr_id - url = reverse('base-resources-set-thumb-from-bbox', args=[dataset_id]) - payload = { - "bbox": [], - "srid": "EPSG:3857" - } - response = self.client.post(url, data=payload, format='json') + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = {"bbox": [], "srid": "EPSG:3857"} + response = self.client.post(url, data=payload, format="json") - expected = { - 'message': 'Not implemented: Endpoint available only for Dataset and Maps', - 'success': False - } + expected = {"message": "Not implemented: Endpoint available only for Dataset and Maps", "success": False} self.assertEqual(response.status_code, 405) self.assertEqual(expected, response.json()) - @patch("geonode.base.api.views.create_thumbnail", side_effect=ThumbnailError('Some exception during thumb creation')) + @patch( + "geonode.base.api.views.create_thumbnail", side_effect=ThumbnailError("Some exception during thumb creation") + ) def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp(self, mock_exp): """ Given a logged User and an existing dataset, should raise a ThumbnailException. """ # Admin - self.assertTrue(self.client.login(username='admin', password='admin')) + self.assertTrue(self.client.login(username="admin", password="admin")) dataset_id = Dataset.objects.first().resourcebase_ptr_id - url = reverse('base-resources-set-thumb-from-bbox', args=[dataset_id]) - payload = { - "bbox": [], - "srid": "EPSG:3857" - } - response = self.client.post(url, data=payload, format='json') + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = {"bbox": [], "srid": "EPSG:3857"} + response = self.client.post(url, data=payload, format="json") - expected = { - 'message': 'Some exception during thumb creation', - 'success': False - } + expected = {"message": "Some exception during thumb creation", "success": False} self.assertEqual(response.status_code, 500) self.assertEqual(expected, response.json()) @@ -2201,37 +2014,38 @@ def test_manager_can_edit_map(self): REST API must not forbid saving maps and apps to non-admin and non-owners. """ from geonode.maps.models import Map - _map = Map.objects.filter(uuid__isnull=False, owner__username='admin').first() + + _map = Map.objects.filter(uuid__isnull=False, owner__username="admin").first() if not len(_map.uuid): _map.uuid = str(uuid4()) _map.save() resource = ResourceBase.objects.filter(uuid=_map.uuid).first() - bobby = get_user_model().objects.get(username='bobby') + bobby = get_user_model().objects.get(username="bobby") # Add perms to Bobby resource_perm_spec_patch = { - 'uuid': resource.uuid, - 'users': [ + "uuid": resource.uuid, + "users": [ { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': '', - 'permissions': 'manage' + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "", + "permissions": "manage", } - ] + ], } # Patch the resource perms - self.assertTrue(self.client.login(username='admin', password='admin')) - set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", 'permissions') - response = self.client.patch(set_perms_url, data=resource_perm_spec_patch, format='json') - self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('status')) - self.assertIsNotNone(response.data.get('status_url')) - status = response.data.get('status') - resp_js = json.loads(response.content.decode('utf-8')) + self.assertTrue(self.client.login(username="admin", password="admin")) + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + response = self.client.patch(set_perms_url, data=resource_perm_spec_patch, format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) status_url = resp_js.get("status_url", None) execution_id = resp_js.get("execution_id", "") self.assertIsNotNone(status_url) @@ -2239,7 +2053,7 @@ def test_manager_can_edit_map(self): for _cnt in range(0, 10): response = self.client.get(f"{status_url}") self.assertEqual(response.status_code, 200) - resp_js = json.loads(response.content.decode('utf-8')) + resp_js = json.loads(response.content.decode("utf-8")) logger.error(f"[{_cnt + 1}] ... {resp_js}") if resp_js.get("status", "") == "finished": status = resp_js.get("status", "") @@ -2250,183 +2064,156 @@ def test_manager_can_edit_map(self): self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) # Test "bobby" can access the "permissions" endpoint - resource_service_permissions_url = reverse('base-resources-perms-spec', kwargs={'pk': resource.pk}) - response = self.client.get(resource_service_permissions_url, format='json') + resource_service_permissions_url = reverse("base-resources-perms-spec", kwargs={"pk": resource.pk}) + response = self.client.get(resource_service_permissions_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertEqual( resource_perm_spec, { - 'users': [ - { - 'id': bobby.id, - 'username': 'bobby', - 'first_name': 'bobby', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'manage', - 'is_superuser': False, - 'is_staff': False + "users": [ + { + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "manage", + "is_superuser": False, + "is_staff": False, }, { - 'id': 1, - 'username': 'admin', - 'first_name': 'admin', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'permissions': 'owner', - 'is_superuser': True, - 'is_staff': True - } + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "permissions": "owner", + "is_superuser": True, + "is_staff": True, + }, + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, ], - 'organizations': [], - 'groups': [ - { - 'id': 3, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'view'}, - { - 'id': 2, - 'title': 'Registered Members', - 'name': 'registered-members', - 'permissions': 'none' - } - ] }, - resource_perm_spec + resource_perm_spec, ) # Test "bobby" can manage the resource permissions - get_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': _map.get_self_resource().pk})}/", 'permissions') - response = self.client.get(get_perms_url, format='json') + get_perms_url = urljoin( + f"{reverse('base-resources-detail', kwargs={'pk': _map.get_self_resource().pk})}/", "permissions" + ) + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertEqual( resource_perm_spec, { - 'users': [ - { - 'id': bobby.id, - 'username': 'bobby', - 'first_name': 'bobby', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'manage', - 'is_staff': False, - 'is_superuser': False, + "users": [ + { + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "manage", + "is_staff": False, + "is_superuser": False, }, { - 'id': 1, - 'username': 'admin', - 'first_name': 'admin', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'permissions': 'owner', - 'is_staff': True, - 'is_superuser': True, - } - ], - 'organizations': [], - 'groups': [ - { - 'id': 3, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'view' + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "permissions": "owner", + "is_staff": True, + "is_superuser": True, }, - { - 'id': 2, - 'title': 'Registered Members', - 'name': 'registered-members', - 'permissions': 'none' - } - ] + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, + ], }, - resource_perm_spec + resource_perm_spec, ) # Fetch the map perms as user "bobby" - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get(get_perms_url, format='json') + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data self.assertEqual( resource_perm_spec, { - 'users': [ - { - 'id': 1, - 'username': 'admin', - 'first_name': 'admin', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240', - 'permissions': 'owner', - 'is_staff': True, - 'is_superuser': True, + "users": [ + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "permissions": "owner", + "is_staff": True, + "is_superuser": True, }, { - 'id': bobby.id, - 'username': 'bobby', - 'first_name': 'bobby', - 'last_name': '', - 'avatar': 'https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240', - 'permissions': 'manage', - 'is_staff': False, - 'is_superuser': False, - } - ], - 'organizations': [], - 'groups': [ - { - 'id': 3, - 'title': 'anonymous', - 'name': 'anonymous', - 'permissions': 'view' + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240", + "permissions": "manage", + "is_staff": False, + "is_superuser": False, }, - { - 'id': 2, - 'title': 'Registered Members', - 'name': 'registered-members', - 'permissions': 'none' - } - ] + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, + ], }, - resource_perm_spec + resource_perm_spec, ) def test_resource_service_copy(self): files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") files_as_dict, _ = get_files(files) resource = Dataset.objects.create( - owner=get_user_model().objects.get(username='admin'), - name='test_copy', - store='geonode_data', + owner=get_user_model().objects.get(username="admin"), + name="test_copy", + store="geonode_data", subtype="vector", alternate="geonode:test_copy", uuid=str(uuid4()), - files=list(files_as_dict.values()) + files=list(files_as_dict.values()), ) - bobby = get_user_model().objects.get(username='bobby') - copy_url = reverse('base-resources-resource-service-copy', kwargs={'pk': resource.pk}) - response = self.client.put(copy_url, data={'title': 'cloned_resource'}) + bobby = get_user_model().objects.get(username="bobby") + copy_url = reverse("base-resources-resource-service-copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) self.assertEqual(response.status_code, 403) # set perms to enable user clone resource self.assertTrue(self.client.login(username="admin", password="admin")) perm_spec = { "uuid": resource.uuid, - 'users': [ + "users": [ { - 'id': bobby.id, - 'username': bobby.username, - 'first_name': bobby.first_name, - 'last_name': bobby.last_name, - 'avatar': '', - 'permissions': 'manage' + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "", + "permissions": "manage", } - ] + ], } - set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", 'permissions') + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") response = self.client.patch(set_perms_url, data=perm_spec, format="json") self.assertEqual(response.status_code, 200) @@ -2435,19 +2222,19 @@ def test_resource_service_copy(self): response = self.client.put(copy_url) self.assertEqual(response.status_code, 200) cloned_resource = Dataset.objects.last() - self.assertEqual(cloned_resource.owner.username, 'admin') + self.assertEqual(cloned_resource.owner.username, "admin") # clone dataset with invalid file - resource.files = ['/path/invalid_file.wrong'] + resource.files = ["/path/invalid_file.wrong"] resource.save() response = self.client.put(copy_url) self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['message'], 'Resource can not be cloned.') + self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clone dataset with no files resource.files = [] resource.save() response = self.client.put(copy_url) self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['message'], 'Resource can not be cloned.') + self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clean resource.delete() @@ -2455,49 +2242,43 @@ def test_resource_service_copy_with_perms_dataset(self): files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") files_as_dict, _ = get_files(files) resource = Dataset.objects.create( - owner=get_user_model().objects.get(username='admin'), - name='test_copy', - store='geonode_data', + owner=get_user_model().objects.get(username="admin"), + name="test_copy", + store="geonode_data", subtype="vector", alternate="geonode:test_copy", resource_type="dataset", uuid=str(uuid4()), - files=list(files_as_dict.values()) + files=list(files_as_dict.values()), ) self._assertCloningWithPerms(resource) @patch.dict(os.environ, {"ASYNC_SIGNALS": "False"}) @override_settings(ASYNC_SIGNALS=False) def test_resource_service_copy_with_perms_dataset_set_default_perms(self): - with self.settings( - ASYNC_SIGNALS=False - ): + with self.settings(ASYNC_SIGNALS=False): files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") files_as_dict, _ = get_files(files) resource = Dataset.objects.create( - owner=get_user_model().objects.get(username='admin'), - name='test_copy_with_perms', - store='geonode_data', + owner=get_user_model().objects.get(username="admin"), + name="test_copy_with_perms", + store="geonode_data", subtype="vector", alternate="geonode:test_copy_with_perms", resource_type="dataset", uuid=str(uuid4()), - files=list(files_as_dict.values()) + files=list(files_as_dict.values()), ) _perms = { - 'users': { - "bobby": ['base.add_resourcebase', 'base.download_resourcebase'] - }, - "groups": { - "anonymous": ["base.view_resourcebase", "base.download_resourcebae"] - } + "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebae"]}, } resource.set_permissions(_perms) # checking that bobby is in the original dataset perms list - self.assertTrue('bobby' in 'bobby' in [x.username for x in resource.get_all_level_info().get("users", [])]) + self.assertTrue("bobby" in "bobby" in [x.username for x in resource.get_all_level_info().get("users", [])]) # copying the resource, should remove the perms for bobby # only the default perms should be available - copy_url = reverse('base-resources-resource-service-copy', kwargs={'pk': resource.pk}) + copy_url = reverse("base-resources-resource-service-copy", kwargs={"pk": resource.pk}) self.assertTrue(self.client.login(username="admin", password="admin")) @@ -2506,22 +2287,22 @@ def test_resource_service_copy_with_perms_dataset_set_default_perms(self): resouce_service_dispatcher.apply((response.json().get("execution_id"),)) - self.assertEqual('finished', self.client.get(response.json().get("status_url")).json().get("status")) + self.assertEqual("finished", self.client.get(response.json().get("status_url")).json().get("status")) _resource = Dataset.objects.filter(title__icontains="test_copy_with_perms").last() self.assertIsNotNone(_resource) - self.assertFalse('bobby' in 'bobby' in [x.username for x in _resource.get_all_level_info().get("users", [])]) - self.assertTrue('admin' in 'admin' in [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertFalse("bobby" in "bobby" in [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertTrue("admin" in "admin" in [x.username for x in _resource.get_all_level_info().get("users", [])]) def test_resource_service_copy_with_perms_doc(self): files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( - owner=get_user_model().objects.get(username='admin'), + owner=get_user_model().objects.get(username="admin"), subtype="vector", alternate="geonode:test_copy", resource_type="document", uuid=str(uuid4()), - files=list(files_as_dict.values()) + files=list(files_as_dict.values()), ) self._assertCloningWithPerms(resource) @@ -2530,11 +2311,11 @@ def test_resource_service_copy_with_perms_map(self): files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( - owner=get_user_model().objects.get(username='admin'), + owner=get_user_model().objects.get(username="admin"), alternate="geonode:test_copy", resource_type="map", uuid=str(uuid4()), - files=list(files_as_dict.values()) + files=list(files_as_dict.values()), ) self._assertCloningWithPerms(resource) @@ -2544,31 +2325,20 @@ def _assertCloningWithPerms(self, resource): self.assertTrue(self.client.login(username="bobby", password="bob")) # bobby cannot copy the resource since he doesnt have all the perms needed - _perms = { - 'users': { - "bobby": ['base.add_resourcebase'] - }, - "groups": { - "anonymous": [] - } - } + _perms = {"users": {"bobby": ["base.add_resourcebase"]}, "groups": {"anonymous": []}} resource.set_permissions(_perms) - copy_url = reverse('base-resources-resource-service-copy', kwargs={'pk': resource.pk}) - response = self.client.put(copy_url, data={'title': 'cloned_resource'}) + copy_url = reverse("base-resources-resource-service-copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) self.assertEqual(response.status_code, 403) # set perms to enable user clone resource # bobby can copy the resource since he has all the perms needed _perms = { - 'users': { - "bobby": ['base.add_resourcebase', 'base.download_resourcebase'] - }, - "groups": { - "anonymous": ["base.view_resourcebase", "base.download_resourcebae"] - } + "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebae"]}, } resource.set_permissions(_perms) - copy_url = reverse('base-resources-resource-service-copy', kwargs={'pk': resource.pk}) - response = self.client.put(copy_url, data={'title': 'cloned_resource'}) + copy_url = reverse("base-resources-resource-service-copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) self.assertEqual(response.status_code, 200) resource.delete() @@ -2579,14 +2349,14 @@ def test_base_resources_return_download_link_if_document(self): doc = Document.objects.first() # From resource base API - url = reverse('base-resources-detail', args=[doc.id]) - response = self.client.get(url, format='json') - download_url = response.json().get('resource').get('download_url') + url = reverse("base-resources-detail", args=[doc.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_url") self.assertEqual(build_absolute_uri(doc.download_url), download_url) # from documents api - url = reverse('documents-detail', args=[doc.id]) - download_url = response.json().get('resource').get('download_url') + url = reverse("documents-detail", args=[doc.id]) + download_url = response.json().get("resource").get("download_url") self.assertEqual(build_absolute_uri(doc.download_url), download_url) def test_base_resources_return_download_link_if_dataset(self): @@ -2596,14 +2366,14 @@ def test_base_resources_return_download_link_if_dataset(self): _dataset = Dataset.objects.first() # From resource base API - url = reverse('base-resources-detail', args=[_dataset.id]) - response = self.client.get(url, format='json') - download_url = response.json().get('resource').get('download_url') + url = reverse("base-resources-detail", args=[_dataset.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_url") self.assertEqual(_dataset.download_url, download_url) # from dataset api - url = reverse('datasets-detail', args=[_dataset.id]) - download_url = response.json().get('resource').get('download_url') + url = reverse("datasets-detail", args=[_dataset.id]) + download_url = response.json().get("resource").get("download_url") self.assertEqual(_dataset.download_url, download_url) def test_base_resources_dont_return_download_link_if_map(self): @@ -2612,88 +2382,83 @@ def test_base_resources_dont_return_download_link_if_map(self): """ _map = Map.objects.first() # From resource base API - url = reverse('base-resources-detail', args=[_map.id]) - response = self.client.get(url, format='json') - download_url = response.json().get('resource').get('download_url', None) + url = reverse("base-resources-detail", args=[_map.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_url", None) self.assertIsNone(download_url) # from maps api - url = reverse('maps-detail', args=[_map.id]) - download_url = response.json().get('resource').get('download_url') + url = reverse("maps-detail", args=[_map.id]) + download_url = response.json().get("resource").get("download_url") self.assertIsNone(download_url) class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): def setUp(self): - self.layer = create_single_dataset('single_layer') + self.layer = create_single_dataset("single_layer") self.metadata = { "filter_header": "Foo Filter header", "field_name": "metadata-name", "field_label": "this is the help text", - "field_value": "foo" + "field_value": "foo", } - m = ExtraMetadata.objects.create( - resource=self.layer, - metadata=self.metadata - ) + m = ExtraMetadata.objects.create(resource=self.layer, metadata=self.metadata) self.layer.metadata.add(m) self.mdata = ExtraMetadata.objects.first() def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) - response = self.client.get(url, content_type='application/json') + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") self.assertTrue(200, response.status_code) - expected = [ - {**{"id": self.mdata.id}, **self.metadata} - ] + expected = [{**{"id": self.mdata.id}, **self.metadata}] self.assertEqual(expected, response.json()) def test_put_will_update_the_whole_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) input_metadata = { "id": self.mdata.id, "filter_header": "Foo Filter header", "field_name": "metadata-updated", "field_label": "this is the help text", - "field_value": "foo" + "field_value": "foo", } - response = self.client.put(url, data=[input_metadata], content_type='application/json') + response = self.client.put(url, data=[input_metadata], content_type="application/json") self.assertTrue(200, response.status_code) self.assertEqual([input_metadata], response.json()) def test_post_will_add_new_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) input_metadata = { "filter_header": "Foo Filter header", "field_name": "metadata-updated", "field_label": "this is the help text", - "field_value": "foo" + "field_value": "foo", } - response = self.client.post(url, data=[input_metadata], content_type='application/json') + response = self.client.post(url, data=[input_metadata], content_type="application/json") self.assertTrue(201, response.status_code) self.assertEqual(2, len(response.json())) def test_delete_will_delete_single_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) - response = self.client.delete(url, data=[self.mdata.id], content_type='application/json') + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.delete(url, data=[self.mdata.id], content_type="application/json") self.assertTrue(200, response.status_code) self.assertEqual([], response.json()) def test_user_without_view_perms_cannot_see_the_endpoint(self): from geonode.resource.manager import resource_manager - self.client.login(username='bobby', password='bob') + self.client.login(username="bobby", password="bob") resource_manager.remove_permissions(self.layer.uuid, instance=self.layer.get_self_resource()) - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) - response = self.client.get(url, content_type='application/json') + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") self.assertTrue(403, response.status_code) - perm_spec = {"users": {"bobby": ['view_resourcebase']}, "groups": {}} + perm_spec = {"users": {"bobby": ["view_resourcebase"]}, "groups": {}} self.layer.set_permissions(perm_spec) - url = reverse('base-resources-extra-metadata', args=[self.layer.id]) - response = self.client.get(url, content_type='application/json') + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") self.assertTrue(200, response.status_code) diff --git a/geonode/base/api/urls.py b/geonode/base/api/urls.py index 19a7336b1b7..b29044ef6fc 100644 --- a/geonode/base/api/urls.py +++ b/geonode/base/api/urls.py @@ -20,13 +20,13 @@ from . import views -router.register(r'users', views.UserViewSet, 'users') -router.register(r'groups', views.GroupViewSet, 'group-profiles') -router.register(r'resources', views.ResourceBaseViewSet, 'base-resources') -router.register(r'owners', views.OwnerViewSet, 'owners') -router.register(r'categories', views.TopicCategoryViewSet, 'categories') -router.register(r'keywords', views.HierarchicalKeywordViewSet, 'keywords') -router.register(r'tkeywords', views.ThesaurusKeywordViewSet, 'tkeywords') -router.register(r'regions', views.RegionViewSet, 'regions') +router.register(r"users", views.UserViewSet, "users") +router.register(r"groups", views.GroupViewSet, "group-profiles") +router.register(r"resources", views.ResourceBaseViewSet, "base-resources") +router.register(r"owners", views.OwnerViewSet, "owners") +router.register(r"categories", views.TopicCategoryViewSet, "categories") +router.register(r"keywords", views.HierarchicalKeywordViewSet, "keywords") +router.register(r"tkeywords", views.ThesaurusKeywordViewSet, "tkeywords") +router.register(r"regions", views.RegionViewSet, "regions") urlpatterns = [] diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index f4548d13df7..bc9eedc26c4 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -70,10 +70,7 @@ from geonode.groups.models import GroupProfile, GroupMember from geonode.people.utils import get_available_users from geonode.security.permissions import get_compact_perms_list, PermSpec, PermSpecCompact -from geonode.security.utils import ( - get_visible_resources, - get_resources_with_perms, - get_user_visible_groups) +from geonode.security.utils import get_visible_resources, get_resources_with_perms, get_user_visible_groups from geonode.resource.models import ExecutionRequest from geonode.resource.api.tasks import resouce_service_dispatcher @@ -100,7 +97,7 @@ TopicCategorySerializer, RegionSerializer, ThesaurusKeywordSerializer, - ExtraMetadataSerializer + ExtraMetadataSerializer, ) from .pagination import GeoNodeApiPagination from geonode.base.utils import validate_extra_metadata @@ -114,11 +111,13 @@ class UserViewSet(DynamicModelViewSet): """ API endpoint that allows users to be viewed or edited. """ + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [IsAuthenticated, IsSelfOrAdminOrReadOnly, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter + permission_classes = [ + IsAuthenticated, + IsSelfOrAdminOrReadOnly, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] serializer_class = UserSerializer pagination_class = GeoNodeApiPagination @@ -135,30 +134,37 @@ def get_queryset(self): queryset = self.get_serializer_class().setup_eager_loading(queryset) return queryset.order_by("username") - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the Resources visible to the user.") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the Resources visible to the user.", + ) + @action(detail=True, methods=["get"]) def resources(self, request, pk=None): user = self.get_object() - permitted = get_objects_for_user(user, 'base.view_resourcebase') - qs = ResourceBase.objects.all().filter(id__in=permitted).order_by('title') + permitted = get_objects_for_user(user, "base.view_resourcebase") + qs = ResourceBase.objects.all().filter(id__in=permitted).order_by("title") resources = get_visible_resources( qs, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) paginator = GeoNodeApiPagination() - paginator.page_size = request.GET.get('page_size', 10) + paginator.page_size = request.GET.get("page_size", 10) result_page = paginator.paginate_queryset(resources, request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True, context={"request": request}) return paginator.get_paginated_response({"resources": serializer.data}) - @extend_schema(methods=['get'], responses={200: GroupProfileSerializer(many=True)}, - description="API endpoint allowing to retrieve the Groups the user is member of.") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: GroupProfileSerializer(many=True)}, + description="API endpoint allowing to retrieve the Groups the user is member of.", + ) + @action(detail=True, methods=["get"]) def groups(self, request, pk=None): user = self.get_object() qs_ids = GroupMember.objects.filter(user=user).values_list("group", flat=True) @@ -170,11 +176,13 @@ class GroupViewSet(DynamicModelViewSet): """ API endpoint that allows gropus to be viewed or edited. """ + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [IsAuthenticatedOrReadOnly, IsManagerEditOrAdmin, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter + permission_classes = [ + IsAuthenticatedOrReadOnly, + IsManagerEditOrAdmin, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] serializer_class = GroupProfileSerializer pagination_class = GeoNodeApiPagination @@ -182,37 +190,45 @@ def get_queryset(self): """ Filters the public groups and private ones the current user is member of. """ - metadata_author_groups = get_user_visible_groups( - self.request.user, include_public_invite=True) + metadata_author_groups = get_user_visible_groups(self.request.user, include_public_invite=True) if not isinstance(metadata_author_groups, list): metadata_author_groups = list(metadata_author_groups.all()) queryset = GroupProfile.objects.filter(id__in=[_g.id for _g in metadata_author_groups]) return queryset.order_by("title") - @extend_schema(methods=['get'], responses={200: UserSerializer(many=True)}, - description="API endpoint allowing to retrieve the Group members.") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: UserSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group members.", + ) + @action(detail=True, methods=["get"]) def members(self, request, pk=None): group = self.get_object() members = get_user_model().objects.filter(id__in=group.member_queryset().values_list("user", flat=True)) return Response(UserSerializer(embed=True, many=True).to_representation(members)) - @extend_schema(methods=['get'], responses={200: UserSerializer(many=True)}, - description="API endpoint allowing to retrieve the Group managers.") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: UserSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group managers.", + ) + @action(detail=True, methods=["get"]) def managers(self, request, pk=None): group = self.get_object() managers = group.get_managers() return Response(UserSerializer(embed=True, many=True).to_representation(managers)) - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the Group specific resources.") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group specific resources.", + ) + @action(detail=True, methods=["get"]) def resources(self, request, pk=None): group = self.get_object() resources = group.resources() paginator = GeoNodeApiPagination() - paginator.page_size = request.GET.get('page_size', 10) + paginator.page_size = request.GET.get("page_size", 10) result_page = paginator.paginate_queryset(list(resources), request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True, context={"request": request}) return paginator.get_paginated_response({"resources": serializer.data}) @@ -222,10 +238,11 @@ class RegionViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, """ API endpoint that lists regions. """ - permission_classes = [AllowAny, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter + + permission_classes = [ + AllowAny, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter] queryset = Region.objects.all() serializer_class = RegionSerializer pagination_class = GeoNodeApiPagination @@ -241,18 +258,18 @@ def get_queryset(self): def _get_kw_hrefs(keywords, slugs: list = []): for obj in keywords: - if obj.get('tags', []): - slugs.append(obj.get('href')) - _get_kw_hrefs(obj.get('nodes', []), slugs) + if obj.get("tags", []): + slugs.append(obj.get("href")) + _get_kw_hrefs(obj.get("nodes", []), slugs) return slugs slugs = _get_kw_hrefs(resource_keywords) return HierarchicalKeyword.objects.filter(slug__in=slugs) - permission_classes = [AllowAny, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter + permission_classes = [ + AllowAny, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] serializer_class = HierarchicalKeywordSerializer pagination_class = GeoNodeApiPagination @@ -261,10 +278,11 @@ class ThesaurusKeywordViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveM """ API endpoint that lists Thesaurus keywords. """ - permission_classes = [AllowAny, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter + + permission_classes = [ + AllowAny, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] queryset = ThesaurusKeyword.objects.all() serializer_class = ThesaurusKeywordSerializer pagination_class = GeoNodeApiPagination @@ -274,10 +292,11 @@ class TopicCategoryViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveMode """ API endpoint that lists categories. """ - permission_classes = [AllowAny, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter + + permission_classes = [ + AllowAny, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter] queryset = TopicCategory.objects.all() serializer_class = TopicCategorySerializer pagination_class = GeoNodeApiPagination @@ -287,11 +306,12 @@ class OwnerViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, """ API endpoint that lists all possible owners. """ + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [AllowAny, ] - filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter + permission_classes = [ + AllowAny, ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] serializer_class = OwnerSerializer pagination_class = GeoNodeApiPagination @@ -303,11 +323,11 @@ def get_queryset(self): filter_options = {} if self.request.query_params: filter_options = { - 'type_filter': self.request.query_params.get('type'), - 'title_filter': self.request.query_params.get('title__icontains') + "type_filter": self.request.query_params.get("type"), + "title_filter": self.request.query_params.get("title__icontains"), } - queryset = queryset.filter(id__in=Subquery( - get_resources_with_perms(self.request.user, filter_options).values('owner')) + queryset = queryset.filter( + id__in=Subquery(get_resources_with_perms(self.request.user, filter_options).values("owner")) ) return queryset.order_by("username") @@ -316,61 +336,87 @@ class ResourceBaseViewSet(DynamicModelViewSet): """ API endpoint that allows base resources to be viewed or edited. """ + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, - ExtentFilter, ResourceBasePermissionsFilter, FavoriteFilter + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + ResourceBasePermissionsFilter, + FavoriteFilter, ] - queryset = ResourceBase.objects.all().order_by('-created') + queryset = ResourceBase.objects.all().order_by("-created") serializer_class = ResourceBaseSerializer pagination_class = GeoNodeApiPagination def _filtered(self, request, filter): paginator = GeoNodeApiPagination() - paginator.page_size = request.GET.get('page_size', 10) + paginator.page_size = request.GET.get("page_size", 10) resources = get_resources_with_perms(request.user).filter(**filter) result_page = paginator.paginate_queryset(resources, request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"resources": serializer.data}) - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the approved Resources.") - @action(detail=False, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the approved Resources.", + ) + @action(detail=False, methods=["get"]) def approved(self, request): return self._filtered(request, {"is_approved": True}) - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the published Resources.") - @action(detail=False, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the published Resources.", + ) + @action(detail=False, methods=["get"]) def published(self, request): return self._filtered(request, {"is_published": True}) - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the featured Resources.") - @action(detail=False, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the featured Resources.", + ) + @action(detail=False, methods=["get"]) def featured(self, request): return self._filtered(request, {"featured": True}) - @extend_schema(methods=['get'], responses={200: FavoriteSerializer(many=True)}, - description="API endpoint allowing to retrieve the favorite Resources.") - @action(detail=False, methods=['get'], permission_classes=[IsAuthenticated, ]) + @extend_schema( + methods=["get"], + responses={200: FavoriteSerializer(many=True)}, + description="API endpoint allowing to retrieve the favorite Resources.", + ) + @action( + detail=False, + methods=["get"], + permission_classes=[ + IsAuthenticated, + ], + ) def favorites(self, request, pk=None): paginator = GeoNodeApiPagination() - paginator.page_size = request.GET.get('page_size', 10) + paginator.page_size = request.GET.get("page_size", 10) favorites = Favorite.objects.favorites_for_user(user=request.user) result_page = paginator.paginate_queryset(favorites, request) serializer = FavoriteSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"favorites": serializer.data}) - @extend_schema(methods=['post', 'delete'], responses={200: FavoriteSerializer(many=True)}, - description="API endpoint allowing to retrieve the favorite Resources.") - @action(detail=True, methods=['post', 'delete'], permission_classes=[IsAuthenticated]) + @extend_schema( + methods=["post", "delete"], + responses={200: FavoriteSerializer(many=True)}, + description="API endpoint allowing to retrieve the favorite Resources.", + ) + @action(detail=True, methods=["post", "delete"], permission_classes=[IsAuthenticated]) def favorite(self, request, pk=None): resource = self.get_object() user = request.user - if request.method == 'POST': + if request.method == "POST": try: Favorite.objects.get(user=user, object_id=resource.pk) return Response({"message": "Resource is already in favorites"}, status=400) @@ -378,15 +424,17 @@ def favorite(self, request, pk=None): Favorite.objects.create_favorite(resource, user) return Response({"message": "Successfuly added resource to favorites"}, status=201) - if request.method == 'DELETE': + if request.method == "DELETE": try: Favorite.objects.get(user=user, object_id=resource.pk).delete() return Response({"message": "Successfuly removed resource from favorites"}, status=200) except Favorite.DoesNotExist: return Response({"message": "Resource not in favorites"}, status=404) - @extend_schema(methods=['get'], responses={200: ResourceBaseTypesSerializer()}, - description=""" + @extend_schema( + methods=["get"], + responses={200: ResourceBaseTypesSerializer()}, + description=""" Returns the list of available ResourceBase polymorphic_ctypes. the mapping looks like: @@ -412,22 +460,20 @@ def favorite(self, request, pk=None): ] } ``` - """) - @action(detail=False, methods=['get']) + """, + ) + @action(detail=False, methods=["get"]) def resource_types(self, request): - - def _to_compact_perms_list(allowed_perms: dict, resource_type: str, resource_subtype: str, compact_perms_labels: dict = {}) -> list: + def _to_compact_perms_list( + allowed_perms: dict, resource_type: str, resource_subtype: str, compact_perms_labels: dict = {} + ) -> list: _compact_perms_list = {} for _k, _v in allowed_perms.items(): _is_owner = _k not in ["anonymous", groups_settings.REGISTERED_MEMBERS_GROUP_NAME] _is_none_allowed = not _is_owner _compact_perms_list[_k] = get_compact_perms_list( - _v, - resource_type, - resource_subtype, - _is_owner, - _is_none_allowed, - compact_perms_labels) + _v, resource_type, resource_subtype, _is_owner, _is_none_allowed, compact_perms_labels + ) return _compact_perms_list resource_types = [] @@ -436,55 +482,67 @@ def _to_compact_perms_list(allowed_perms: dict, resource_type: str, resource_sub for _model in apps.get_models(): if _model.__name__ == "ResourceBase": for _m in _model.__subclasses__(): - if _m.__name__.lower() not in ['service']: + if _m.__name__.lower() not in ["service"]: _types.append(_m.__name__.lower()) _allowed_perms[_m.__name__.lower()] = { "perms": _m.allowed_permissions, "compact": _to_compact_perms_list( - _m.allowed_permissions, _m.__name__.lower(), _m.__name__.lower(), _m.compact_permission_labels) + _m.allowed_permissions, + _m.__name__.lower(), + _m.__name__.lower(), + _m.compact_permission_labels, + ), } - if settings.GEONODE_APPS_ENABLE and 'geoapp' in _types: - _types.remove('geoapp') - if hasattr(settings, 'CLIENT_APP_LIST') and settings.CLIENT_APP_LIST: + if settings.GEONODE_APPS_ENABLE and "geoapp" in _types: + _types.remove("geoapp") + if hasattr(settings, "CLIENT_APP_LIST") and settings.CLIENT_APP_LIST: _types += settings.CLIENT_APP_LIST else: from geonode.geoapps.models import GeoApp - geoapp_types = [x for x in GeoApp.objects.values_list('resource_type', flat=True).all().distinct()] + + geoapp_types = [x for x in GeoApp.objects.values_list("resource_type", flat=True).all().distinct()] _types += geoapp_types - if hasattr(settings, 'CLIENT_APP_ALLOWED_PERMS_LIST') and settings.CLIENT_APP_ALLOWED_PERMS_LIST: + if hasattr(settings, "CLIENT_APP_ALLOWED_PERMS_LIST") and settings.CLIENT_APP_ALLOWED_PERMS_LIST: for _type in settings.CLIENT_APP_ALLOWED_PERMS_LIST: for _type_name, _type_perms in _type.items(): _compact_permission_labels = {} - if hasattr(settings, 'CLIENT_APP_COMPACT_PERM_LABELS'): + if hasattr(settings, "CLIENT_APP_COMPACT_PERM_LABELS"): _compact_permission_labels = settings.CLIENT_APP_COMPACT_PERM_LABELS.get(_type_name, {}) _allowed_perms[_type_name] = { "perms": _type_perms, - "compact": _to_compact_perms_list(_type_perms, _type_name, _type_name, _compact_permission_labels) + "compact": _to_compact_perms_list( + _type_perms, _type_name, _type_name, _compact_permission_labels + ), } else: from geonode.geoapps.models import GeoApp + for _m in GeoApp.objects.filter(resource_type__in=_types).iterator(): - if hasattr(_m, 'resource_type') and _m.resource_type and _m.resource_type not in _allowed_perms: + if hasattr(_m, "resource_type") and _m.resource_type and _m.resource_type not in _allowed_perms: _allowed_perms[_m.resource_type] = { "perms": _m.allowed_permissions, "compact": _to_compact_perms_list( - _m.allowed_permissions, _m.resource_type, _m.subtype, _m.compact_permission_labels) + _m.allowed_permissions, _m.resource_type, _m.subtype, _m.compact_permission_labels + ), } for _type in _types: - resource_types.append({ - "name": _type, - "count": get_resources_with_perms(request.user).filter(resource_type=_type).count(), - "allowed_perms": _allowed_perms[_type] if _type in _allowed_perms else [] - }) + resource_types.append( + { + "name": _type, + "count": get_resources_with_perms(request.user).filter(resource_type=_type).count(), + "allowed_perms": _allowed_perms[_type] if _type in _allowed_perms else [], + } + ) return Response({"resource_types": resource_types}) - @extend_schema(methods=['get', 'put', 'patch', 'delete'], - request=PermSpecSerialiazer(), - responses={200: None}, - description=""" + @extend_schema( + methods=["get", "put", "patch", "delete"], + request=PermSpecSerialiazer(), + responses={200: None}, + description=""" Sets an object's the permission levels based on the perm_spec JSON. the mapping looks like: @@ -503,15 +561,15 @@ def _to_compact_perms_list(allowed_perms: dict, resource_type: str, resource_sub } } ``` - """) + """, + ) @action( detail=True, url_path="permissions", # noqa url_name="perms-spec", - methods=['get', 'put', 'patch', 'delete'], - permission_classes=[ - IsAuthenticated - ]) + methods=["get", "put", "patch", "delete"], + permission_classes=[IsAuthenticated], + ) def resource_service_permissions(self, request, pk): """Instructs the Async dispatcher to execute a 'DELETE' or 'UPDATE' on the permissions of a valid 'uuid' @@ -569,85 +627,90 @@ def resource_service_permissions(self, request, pk): """ config = Configuration.load() resource = get_object_or_404(ResourceBase, pk=pk) - _user_can_manage = request.user.has_perm('change_resourcebase_permissions', resource.get_self_resource()) - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated or \ - resource is None or not _user_can_manage: + _user_can_manage = request.user.has_perm("change_resourcebase_permissions", resource.get_self_resource()) + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not _user_can_manage + ): return Response(status=status.HTTP_403_FORBIDDEN) try: perms_spec = PermSpec(resource.get_all_level_info(), resource) request_params = request.data - if request.method == 'GET': + if request.method == "GET": return Response(perms_spec.compact) - elif request.method == 'DELETE': + elif request.method == "DELETE": _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='remove_permissions', + func_name="remove_permissions", geonode_resource=resource, action="permissions", - input_params={ - "uuid": request_params.get('uuid', resource.uuid) - } + input_params={"uuid": request_params.get("uuid", resource.uuid)}, ) - elif request.method == 'PUT': + elif request.method == "PUT": perms_spec_compact = PermSpecCompact(request.data, resource) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='set_permissions', + func_name="set_permissions", geonode_resource=resource, action="permissions", input_params={ - "uuid": request_params.get('uuid', resource.uuid), - "owner": request_params.get('owner', resource.owner.username), + "uuid": request_params.get("uuid", resource.uuid), + "owner": request_params.get("owner", resource.owner.username), "permissions": perms_spec_compact.extended, - "created": request_params.get('created', False) - } + "created": request_params.get("created", False), + }, ) - elif request.method == 'PATCH': + elif request.method == "PATCH": perms_spec_compact_patch = PermSpecCompact(request.data, resource) perms_spec_compact_resource = PermSpecCompact(perms_spec.compact, resource) perms_spec_compact_resource.merge(perms_spec_compact_patch) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='set_permissions', + func_name="set_permissions", geonode_resource=resource, action="permissions", input_params={ - "uuid": request_params.get('uuid', resource.uuid), - "owner": request_params.get('owner', resource.owner.username), + "uuid": request_params.get("uuid", resource.uuid), + "owner": request_params.get("owner", resource.owner.username), "permissions": perms_spec_compact_resource.extended, - "created": request_params.get('created', False) - } + "created": request_params.get("created", False), + }, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @extend_schema( - methods=["post"], responses={200}, description="API endpoint allowing to set the thumbnail url for an existing dataset." + methods=["post"], + responses={200}, + description="API endpoint allowing to set the thumbnail url for an existing dataset.", ) @action( detail=False, url_path="(?P\d+)/set_thumbnail_from_bbox", # noqa url_name="set-thumb-from-bbox", methods=["post"], - permission_classes=[ - IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}) - ]) + permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + ) def set_thumbnail_from_bbox(self, request, resource_id): import traceback from django.utils.datastructures import MultiValueDictKeyError + try: resource = ResourceBase.objects.get(id=ast.literal_eval(resource_id)) @@ -665,13 +728,20 @@ def set_thumbnail_from_bbox(self, request, resource_id): bbox = request_body["bbox"] + [request_body["srid"]] zoom = request_body.get("zoom", None) - thumbnail_url = create_thumbnail(resource.get_real_instance(), bbox=bbox, background_zoom=zoom, overwrite=True) - return Response({"message": "Thumbnail correctly created.", "success": True, "thumbnail_url": thumbnail_url}, status=200) + thumbnail_url = create_thumbnail( + resource.get_real_instance(), bbox=bbox, background_zoom=zoom, overwrite=True + ) + return Response( + {"message": "Thumbnail correctly created.", "success": True, "thumbnail_url": thumbnail_url}, status=200 + ) except ResourceBase.DoesNotExist: traceback.print_exc() logger.error(f"Resource selected with id {resource_id} does not exists") return Response( - data={"message": f"Resource selected with id {resource_id} does not exists", "success": False}, status=404, exception=True) + data={"message": f"Resource selected with id {resource_id} does not exists", "success": False}, + status=404, + exception=True, + ) except NotImplementedError as e: traceback.print_exc() logger.error(e) @@ -693,9 +763,8 @@ def set_thumbnail_from_bbox(self, request, resource_id): url_path="ingest/(?P\w+)", # noqa url_name="resource-service-ingest", methods=["post"], - permission_classes=[ - IsAuthenticated - ]) + permission_classes=[IsAuthenticated], + ) def resource_service_ingest(self, request, resource_type: str = None): """Instructs the Async dispatcher to execute a 'INGEST' operation @@ -748,38 +817,42 @@ def resource_service_ingest(self, request, resource_type: str = None): } """ config = Configuration.load() - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated \ - or not request.user.has_perm('base.add_resourcebase'): + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or not request.user.has_perm("base.add_resourcebase") + ): return Response(status=status.HTTP_403_FORBIDDEN) try: request_params = self._get_request_params(request) - uuid = request_params.get('uuid', str(uuid4())) + uuid = request_params.get("uuid", str(uuid4())) resource_filter = ResourceBase.objects.filter(uuid=uuid) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='ingest', + func_name="ingest", geonode_resource=resource_filter.get() if resource_filter.exists() else None, action="ingest", input_params={ "uuid": uuid, - "files": request_params.get('files', '[]'), + "files": request_params.get("files", "[]"), "resource_type": resource_type, - "defaults": request_params.get('defaults', f"{{\"owner\":\"{request.user.username}\"}}") - } + "defaults": request_params.get("defaults", f'{{"owner":"{request.user.username}"}}'), + }, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @@ -792,9 +865,8 @@ def resource_service_ingest(self, request, resource_type: str = None): url_path="create/(?P\w+)", # noqa url_name="resource-service-create", methods=["post"], - permission_classes=[ - IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}) - ]) + permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + ) def resource_service_create(self, request, resource_type: str = None): """Instructs the Async dispatcher to execute a 'CREATE' operation **WARNING**: This will create an empty dataset; if you need to upload a resource to GeoNode, consider using the endpoint "ingest" instead @@ -846,52 +918,57 @@ def resource_service_create(self, request, resource_type: str = None): } """ config = Configuration.load() - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated \ - or not request.user.has_perm('base.add_resourcebase'): + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or not request.user.has_perm("base.add_resourcebase") + ): return Response(status=status.HTTP_403_FORBIDDEN) try: request_params = self._get_request_params(request) - uuid = request_params.get('uuid', str(uuid4())) + uuid = request_params.get("uuid", str(uuid4())) resource_filter = ResourceBase.objects.filter(uuid=uuid) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='create', + func_name="create", geonode_resource=resource_filter.get() if resource_filter.exists() else None, action="create", input_params={ "uuid": uuid, "resource_type": resource_type, - "defaults": request_params.get('defaults', f"{{\"owner\":\"{request.user.username}\"}}") - } + "defaults": request_params.get("defaults", f'{{"owner":"{request.user.username}"}}'), + }, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @extend_schema( - methods=["delete"], responses={200}, description="Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid'." + methods=["delete"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid'.", ) @action( detail=True, url_path="delete", # noqa url_name="resource-service-delete", methods=["delete"], - permission_classes=[ - IsAuthenticated, UserHasPerms - ]) + permission_classes=[IsAuthenticated, UserHasPerms], + ) def resource_service_delete(self, request, pk): """Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid' @@ -932,46 +1009,50 @@ def resource_service_delete(self, request, pk): """ config = Configuration.load() resource = get_object_or_404(ResourceBase, pk=pk) - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated or \ - resource is None or not request.user.has_perm('delete_resourcebase', resource.get_self_resource()): + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("delete_resourcebase", resource.get_self_resource()) + ): return Response(status=status.HTTP_403_FORBIDDEN) try: _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='delete', + func_name="delete", action="delete", geonode_resource=resource, - input_params={ - "uuid": resource.uuid - } + input_params={"uuid": resource.uuid}, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @extend_schema( - methods=["put"], responses={200}, description="Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid'." + methods=["put"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid'.", ) @action( detail=True, url_path="update", # noqa url_name="resource-service-update", methods=["put"], - permission_classes=[ - IsAuthenticated, UserHasPerms - ]) + permission_classes=[IsAuthenticated, UserHasPerms], + ) def resource_service_update(self, request, pk): """Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid' @@ -1039,45 +1120,52 @@ def resource_service_update(self, request, pk): """ config = Configuration.load() resource = get_object_or_404(ResourceBase, pk=pk) - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated or \ - resource is None or not request.user.has_perm('change_resourcebase', resource.get_self_resource()): + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("change_resourcebase", resource.get_self_resource()) + ): return Response(status=status.HTTP_403_FORBIDDEN) try: request_params = self._get_request_params(request=request) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='update', + func_name="update", geonode_resource=resource, action="update", input_params={ - "uuid": request_params.get('uuid', resource.uuid), - "xml_file": request_params.get('xml_file', None), - "metadata_uploaded": request_params.get('metadata_uploaded', False), - "vals": request_params.get('vals', '{}'), - "regions": request_params.get('regions', '[]'), - "keywords": request_params.get('keywords', '[]'), - "custom": request_params.get('custom', '{}'), - "notify": request_params.get('notify', True) - } + "uuid": request_params.get("uuid", resource.uuid), + "xml_file": request_params.get("xml_file", None), + "metadata_uploaded": request_params.get("metadata_uploaded", False), + "vals": request_params.get("vals", "{}"), + "regions": request_params.get("regions", "[]"), + "keywords": request_params.get("keywords", "[]"), + "custom": request_params.get("custom", "{}"), + "notify": request_params.get("notify", True), + }, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @extend_schema( - methods=["put"], responses={200}, description="Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'uuid'." + methods=["put"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'uuid'.", ) @action( detail=True, @@ -1085,20 +1173,16 @@ def resource_service_update(self, request, pk): url_name="resource-service-copy", methods=["put"], permission_classes=[ - IsAuthenticated, UserHasPerms( + IsAuthenticated, + UserHasPerms( perms_dict={ - "dataset": { - "PUT": ['base.add_resourcebase', 'base.download_resourcebase'], "rule": all - }, - "document": { - "PUT": ['base.add_resourcebase', 'base.download_resourcebase'], "rule": all - }, - "default": { - "PUT": ['base.add_resourcebase'] - } + "dataset": {"PUT": ["base.add_resourcebase", "base.download_resourcebase"], "rule": all}, + "document": {"PUT": ["base.add_resourcebase", "base.download_resourcebase"], "rule": all}, + "default": {"PUT": ["base.add_resourcebase"]}, } - ) - ]) + ), + ], + ) def resource_service_copy(self, request, pk): """Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'pk' @@ -1150,8 +1234,14 @@ def resource_service_copy(self, request, pk): """ config = Configuration.load() resource = get_object_or_404(ResourceBase, pk=pk) - if config.read_only or config.maintenance or request.user.is_anonymous or not request.user.is_authenticated or \ - resource is None or not request.user.has_perm('view_resourcebase', resource.get_self_resource()): + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("view_resourcebase", resource.get_self_resource()) + ): return Response(status=status.HTTP_403_FORBIDDEN) if not resource.is_copyable: return Response({"message": "Resource can not be cloned."}, status=400) @@ -1159,48 +1249,50 @@ def resource_service_copy(self, request, pk): request_params = self._get_request_params(request) _exec_request = ExecutionRequest.objects.create( user=request.user, - func_name='copy', + func_name="copy", geonode_resource=resource, action="copy", input_params={ "instance": resource.id, - "owner": request_params.get('owner', request.user.username), - "defaults": request_params.get('defaults', '{}') - } + "owner": request_params.get("owner", request.user.username), + "defaults": request_params.get("defaults", "{}"), + }, ) resouce_service_dispatcher.apply_async((_exec_request.exec_id,)) return Response( { - 'status': _exec_request.status, - 'execution_id': _exec_request.exec_id, - 'status_url': - urljoin( - settings.SITEURL, - reverse('rs-execution-status', kwargs={'execution_id': _exec_request.exec_id}) - ) + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), }, - status=status.HTTP_200_OK) + status=status.HTTP_200_OK, + ) except Exception as e: logger.exception(e) return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) @extend_schema( - methods=['post', 'get'], + methods=["post", "get"], responses={200}, - description="API endpoint allowing to rate and get overall rating of the Resource.") + description="API endpoint allowing to rate and get overall rating of the Resource.", + ) @action( detail=True, url_path="ratings", url_name="ratings", - methods=['post', 'get'], + methods=["post", "get"], permission_classes=[ - IsAuthenticatedOrReadOnly, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}) - ]) + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ], + ) def ratings(self, request, pk): resource = get_object_or_404(ResourceBase, pk=pk) resource = resource.get_real_instance() ct = ContentType.objects.get_for_model(resource) - if request.method == 'POST': + if request.method == "POST": rating_input = int(request.data.get("rating")) category = resource._meta.object_name.lower() # check if category is configured in settings.PINAX_RATINGS_CATEGORY_CHOICES @@ -1208,60 +1300,38 @@ def ratings(self, request, pk): # Check for errors and bail early if category and cat_choice is None: - return HttpResponseForbidden( - "Invalid category. It must match a preconfigured setting" - ) + return HttpResponseForbidden("Invalid category. It must match a preconfigured setting") if rating_input not in range(NUM_OF_RATINGS + 1): - return HttpResponseForbidden( - f"Invalid rating. It must be a value between 0 and {NUM_OF_RATINGS}" - ) - Rating.update( - rating_object=resource, - user=request.user, - category=cat_choice, - rating=rating_input - ) + return HttpResponseForbidden(f"Invalid rating. It must be a value between 0 and {NUM_OF_RATINGS}") + Rating.update(rating_object=resource, user=request.user, category=cat_choice, rating=rating_input) user_rating = None if request.user.is_authenticated: - user_rating = Rating.objects.filter( - object_id=resource.pk, - content_type=ct, - user=request.user - ).first() - overall_rating = OverallRating.objects.filter( - object_id=resource.pk, - content_type=ct - ).aggregate(r=models.Avg("rating"))["r"] + user_rating = Rating.objects.filter(object_id=resource.pk, content_type=ct, user=request.user).first() + overall_rating = OverallRating.objects.filter(object_id=resource.pk, content_type=ct).aggregate( + r=models.Avg("rating") + )["r"] overall_rating = Decimal(str(overall_rating or "0")) - return Response( - { - "rating": user_rating.rating if user_rating else 0, - "overall_rating": overall_rating - } - ) + return Response({"rating": user_rating.rating if user_rating else 0, "overall_rating": overall_rating}) @extend_schema( - methods=['put'], - responses={200}, - description="API endpoint allowing to set thumbnail of the Resource.") + methods=["put"], responses={200}, description="API endpoint allowing to set thumbnail of the Resource." + ) @action( detail=True, url_path="set_thumbnail", url_name="set_thumbnail", - methods=['put'], - permission_classes=[ - IsAuthenticated, UserHasPerms - ], - parser_classes=[JSONParser, MultiPartParser] + methods=["put"], + permission_classes=[IsAuthenticated, UserHasPerms], + parser_classes=[JSONParser, MultiPartParser], ) def set_thumbnail(self, request, pk): resource = get_object_or_404(ResourceBase, pk=pk) - if not request.data.get('file'): + if not request.data.get("file"): raise ValidationError("Field file is required") - file_data = request.data['file'] + file_data = request.data["file"] if isinstance(file_data, str): if re.match(BASE64_PATTERN, file_data): @@ -1269,42 +1339,41 @@ def set_thumbnail(self, request, pk): thumbnail, _thumbnail_format = _decode_base64(file_data) except Exception: return Response( - 'The request body is not a valid base64 string or the image format is not PNG or JPEG', - status=status.HTTP_400_BAD_REQUEST + "The request body is not a valid base64 string or the image format is not PNG or JPEG", + status=status.HTTP_400_BAD_REQUEST, ) else: try: # Check if file_data is a valid url and set it as thumbail_url validate = URLValidator() validate(file_data) - if urlparse(file_data).path.rsplit('.')[-1] not in ['png', 'jpeg', 'jpg']: + if urlparse(file_data).path.rsplit(".")[-1] not in ["png", "jpeg", "jpg"]: return Response( - 'The url must be of an image with format (png, jpeg or jpg)', - status=status.HTTP_400_BAD_REQUEST + "The url must be of an image with format (png, jpeg or jpg)", + status=status.HTTP_400_BAD_REQUEST, ) resource.thumbnail_url = file_data resource.save() return Response({"thumbnail_url": resource.thumbnail_url}) except Exception: - raise ValidationError(detail='file is either a file upload, ASCII byte string or a valid image url string') + raise ValidationError( + detail="file is either a file upload, ASCII byte string or a valid image url string" + ) else: # Validate size if file_data.size > 1000000: - raise ValidationError(detail='File must not exceed 1MB') + raise ValidationError(detail="File must not exceed 1MB") thumbnail = file_data.read() try: file_data.seek(0) Image.open(file_data) except Exception: - raise ValidationError(detail='Invalid data provided') + raise ValidationError(detail="Invalid data provided") if thumbnail: resource_manager.set_thumbnail(resource.uuid, instance=resource, thumbnail=thumbnail) return Response({"thumbnail_url": resource.thumbnail_url}) - return Response( - 'Unable to set thumbnail', - status=status.HTTP_400_BAD_REQUEST - ) + return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST) @extend_schema( methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource" @@ -1312,9 +1381,7 @@ def set_thumbnail(self, request, pk): @action( detail=True, methods=["get", "put", "delete", "post"], - permission_classes=[ - IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}) - ], + permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], url_path=r"extra_metadata", # noqa url_name="extra-metadata", ) @@ -1335,7 +1402,7 @@ def extra_metadata(self, request, pk): return Response(status=500, data=e.args[0]) if request.method == "PUT": - ''' + """ update specific metadata. The ID of the metadata is required to perform the update [ { @@ -1348,25 +1415,25 @@ def extra_metadata(self, request, pk): "category": "object" } ] - ''' + """ for _m in extra_metadata: - _id = _m.pop('id') + _id = _m.pop("id") ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) logger.info("metadata updated for the selected resource") _obj.refresh_from_db() return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) elif request.method == "DELETE": # delete single metadata - ''' + """ Expect a payload with the IDs of the metadata that should be deleted. Payload be like: [4, 3] - ''' + """ ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() _obj.refresh_from_db() return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) elif request.method == "POST": # add new metadata - ''' + """ [ { "name": "foo_name", @@ -1377,12 +1444,9 @@ def extra_metadata(self, request, pk): "category": "object" } ] - ''' + """ for _m in extra_metadata: - new_m = ExtraMetadata.objects.create( - resource=_obj, - metadata=_m - ) + new_m = ExtraMetadata.objects.create(resource=_obj, metadata=_m) new_m.save() _obj.metadata.add(new_m) _obj.refresh_from_db() @@ -1390,12 +1454,16 @@ def extra_metadata(self, request, pk): def _get_request_params(self, request, encode=False): try: - return QueryDict(request.body, mutable=True, encoding="UTF-8") if encode else QueryDict(request.body, mutable=True) + return ( + QueryDict(request.body, mutable=True, encoding="UTF-8") + if encode + else QueryDict(request.body, mutable=True) + ) except Exception as e: - ''' + """ The request with the barer token access to the request.data during the token verification so in this case if the request.body cannot not access, we just re-access to the request.data to get the params needed - ''' + """ logger.debug(e) return request.data diff --git a/geonode/base/auth.py b/geonode/base/auth.py index 3faff79dc82..cfef16bd820 100644 --- a/geonode/base/auth.py +++ b/geonode/base/auth.py @@ -43,7 +43,7 @@ def extract_user_from_headers(request): user = token_header_authenticate_user(auth_header) if "apikey" in request.GET: - user = get_auth_user_from_token(request.GET.get('apikey')) + user = get_auth_user_from_token(request.GET.get("apikey")) return user @@ -65,7 +65,7 @@ def extract_headers(request): def make_token_expiration(seconds=86400): - _expire_seconds = getattr(settings, 'ACCESS_TOKEN_EXPIRE_SECONDS', seconds) + _expire_seconds = getattr(settings, "ACCESS_TOKEN_EXPIRE_SECONDS", seconds) _expire_time = datetime.datetime.now(timezone.get_current_timezone()) _expire_delta = datetime.timedelta(seconds=_expire_seconds) return _expire_time + _expire_delta @@ -79,10 +79,8 @@ def create_auth_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): Application = get_application_model() app = Application.objects.get(name=client) (access_token, created) = AccessToken.objects.get_or_create( - user=user, - application=app, - expires=expires, - token=generate_token()) + user=user, application=app, expires=expires, token=generate_token() + ) return access_token except Exception: tb = traceback.format_exc() @@ -108,7 +106,7 @@ def get_auth_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): try: Application = get_application_model() app = Application.objects.get(name=client) - access_token = AccessToken.objects.filter(user=user, application=app).order_by('-expires').first() + access_token = AccessToken.objects.filter(user=user, application=app).order_by("-expires").first() return access_token except Exception: tb = traceback.format_exc() @@ -121,7 +119,7 @@ def get_auth_user(access_token, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NA try: Application = get_application_model() app = Application.objects.get(name=client) - user = AccessToken.objects.filter(token=access_token, application=app).order_by('-expires').first().user + user = AccessToken.objects.filter(token=access_token, application=app).order_by("-expires").first().user return user except Exception: tb = traceback.format_exc() @@ -140,7 +138,7 @@ def get_or_create_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME # Let's create the new AUTH TOKEN existing_token = None try: - existing_token = AccessToken.objects.filter(user=user, application=app).order_by('-expires').first() + existing_token = AccessToken.objects.filter(user=user, application=app).order_by("-expires").first() if existing_token and existing_token.is_expired(): existing_token.delete() existing_token = None @@ -170,7 +168,7 @@ def delete_old_tokens(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): app = application.objects.get(name=client) # Lets delete the old one - old_tokens = AccessToken.objects.filter(user=user, application=app).order_by('-expires') + old_tokens = AccessToken.objects.filter(user=user, application=app).order_by("-expires") for old in old_tokens: if old.is_expired(): old.delete() @@ -181,40 +179,40 @@ def delete_old_tokens(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): def get_token_from_auth_header(auth_header, create_if_not_exists=False): - if re.search('Basic', auth_header, re.IGNORECASE): + if re.search("Basic", auth_header, re.IGNORECASE): user = basic_auth_authenticate_user(auth_header) if user and user.is_active: return get_auth_token(user) if not create_if_not_exists else get_or_create_token(user) - elif re.search('Bearer', auth_header, re.IGNORECASE): - return re.compile(re.escape('Bearer '), re.IGNORECASE).sub('', auth_header) + elif re.search("Bearer", auth_header, re.IGNORECASE): + return re.compile(re.escape("Bearer "), re.IGNORECASE).sub("", auth_header) return None def set_session_token(session, token): - session['access_token'] = str(token) + session["access_token"] = str(token) def get_session_token(session): - return session.get('access_token', None) + return session.get("access_token", None) def get_token_object_from_session(session): - if 'access_token' in session: + if "access_token" in session: try: return AccessToken.objects.get(token=get_session_token(session)) except Exception: tb = traceback.format_exc() if tb: logger.debug(tb) - del session['access_token'] + del session["access_token"] session.modified = True return None return None def remove_session_token(session): - if 'access_token' in session: - del session['access_token'] + if "access_token" in session: + del session["access_token"] def basic_auth_authenticate_user(auth_header: str): @@ -223,8 +221,8 @@ def basic_auth_authenticate_user(auth_header: str): :param auth_header: Authorization header of the request """ - encoded_credentials = auth_header.split(' ')[1] # Removes "Basic " to isolate credentials - decoded_credentials = base64.b64decode(encoded_credentials).decode("utf-8").split(':') + encoded_credentials = auth_header.split(" ")[1] # Removes "Basic " to isolate credentials + decoded_credentials = base64.b64decode(encoded_credentials).decode("utf-8").split(":") username = decoded_credentials[0] password = decoded_credentials[1] @@ -255,12 +253,12 @@ def token_header_authenticate_user(auth_header: str): def visitor_ip_address(request): - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: - ip = x_forwarded_for.split(',')[0] + ip = x_forwarded_for.split(",")[0] else: - ip = request.META.get('REMOTE_ADDR') + ip = request.META.get("REMOTE_ADDR") return ip diff --git a/geonode/base/bbox_utils.py b/geonode/base/bbox_utils.py index f3592a59d33..dc7ca8c4baf 100644 --- a/geonode/base/bbox_utils.py +++ b/geonode/base/bbox_utils.py @@ -25,11 +25,7 @@ from typing import Union, List, Generator from shapely import affinity from shapely.ops import split -from shapely.geometry import ( - mapping, - Polygon, - LineString, - GeometryCollection) +from shapely.geometry import mapping, Polygon, LineString, GeometryCollection from django.contrib.gis.geos import Polygon as DjangoPolygon @@ -93,7 +89,10 @@ def filter_bbox(queryset, bbox): for _bbox in bboxes: _bbox = list(map(Decimal, _bbox)) search_polygon = polygon_from_bbox((_bbox[0], _bbox[1], _bbox[2], _bbox[3])) - for search_polygon_dl in [DjangoPolygon.from_ewkt(_p.wkt) for _p in split_polygon(json.loads(search_polygon.json), output_format="polygons")]: + for search_polygon_dl in [ + DjangoPolygon.from_ewkt(_p.wkt) + for _p in split_polygon(json.loads(search_polygon.json), output_format="polygons") + ]: _qs = queryset.filter(ll_bbox_polygon__intersects=search_polygon_dl) search_queryset = _qs if search_queryset is None else search_queryset | _qs @@ -112,10 +111,9 @@ def check_crossing(lon1: float, lon2: float, validate: bool = False, dlon_thresh return abs(lon2 - lon1) > dlon_threshold -def translate_polygons(geometry_collection: GeometryCollection, - output_format: str = "geojson") -> Generator[ - Union[List[dict], List[Polygon]], None, None -]: +def translate_polygons( + geometry_collection: GeometryCollection, output_format: str = "geojson" +) -> Generator[Union[List[dict], List[Polygon]], None, None]: """ ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513 """ @@ -132,9 +130,9 @@ def translate_polygons(geometry_collection: GeometryCollection, yield json.dumps(mapping(geo_polygon)) if (yield_geojson) else geo_polygon -def split_polygon(geojson: dict, output_format: str = "geojson", validate: bool = False) -> Union[ - List[dict], List[Polygon], GeometryCollection -]: +def split_polygon( + geojson: dict, output_format: str = "geojson", validate: bool = False +) -> Union[List[dict], List[Polygon], GeometryCollection]: """ ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513 Given a GeoJSON representation of a Polygon, returns a collection of @@ -191,13 +189,13 @@ def split_polygon(geojson: dict, output_format: str = "geojson", validate: bool ring_maxx = x_shift # Ensure that any holes remain contained within the (translated) outer shell - if (ring_index == 0): # by GeoJSON definition, first ring is the outer shell + if ring_index == 0: # by GeoJSON definition, first ring is the outer shell shell_minx, shell_maxx = (ring_minx, ring_maxx) - elif (ring_minx < shell_minx): + elif ring_minx < shell_minx: ring_shift = [[x + 360, y] for (x, y) in coords_shift[ring_index]] coords_shift[ring_index] = ring_shift ring_minx, ring_maxx = (x + 360 for x in (ring_minx, ring_maxx)) - elif (ring_maxx > shell_maxx): + elif ring_maxx > shell_maxx: ring_shift = [[x - 360, y] for (x, y) in coords_shift[ring_index]] coords_shift[ring_index] = ring_shift ring_minx, ring_maxx = (x - 360 for x in (ring_minx, ring_maxx)) diff --git a/geonode/base/enumerations.py b/geonode/base/enumerations.py index ad9ebbc19ff..93e05d8b60f 100644 --- a/geonode/base/enumerations.py +++ b/geonode/base/enumerations.py @@ -19,457 +19,453 @@ from django.utils.translation import ugettext_lazy as _ -LINK_TYPES = ['original', 'data', 'image', 'metadata', 'html', - 'OGC:WMS', 'OGC:WFS', 'OGC:WCS'] +LINK_TYPES = ["original", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] HIERARCHY_LEVELS = ( - ('series', _('series')), - ('software', _('computer program or routine')), - ('featureType', _('feature type')), - ('model', _('copy or imitation of an existing or hypothetical object')), - ('collectionHardware', _('collection hardware')), - ('collectionSession', _('collection session')), - ('nonGeographicDataset', _('non-geographic data')), - ('propertyType', _('property type')), - ('fieldSession', _('field session')), - ('dataset', _('dataset')), - ('service', _('service interfaces')), - ('attribute', _('attribute class')), - ('attributeType', _('characteristic of a feature')), - ('tile', _('tile or spatial subset of geographic data')), - ('feature', _('feature')), - ('dimensionGroup', _('dimension group')), + ("series", _("series")), + ("software", _("computer program or routine")), + ("featureType", _("feature type")), + ("model", _("copy or imitation of an existing or hypothetical object")), + ("collectionHardware", _("collection hardware")), + ("collectionSession", _("collection session")), + ("nonGeographicDataset", _("non-geographic data")), + ("propertyType", _("property type")), + ("fieldSession", _("field session")), + ("dataset", _("dataset")), + ("service", _("service interfaces")), + ("attribute", _("attribute class")), + ("attributeType", _("characteristic of a feature")), + ("tile", _("tile or spatial subset of geographic data")), + ("feature", _("feature")), + ("dimensionGroup", _("dimension group")), ) UPDATE_FREQUENCIES = ( - ('unknown', _('frequency of maintenance for the data is not known')), - ('continual', _('data is repeatedly and frequently updated')), - ('notPlanned', _('there are no plans to update the data')), - ('daily', _('data is updated each day')), - ('annually', _('data is updated every year')), - ('asNeeded', _('data is updated as deemed necessary')), - ('monthly', _('data is updated each month')), - ('fortnightly', _('data is updated every two weeks')), - ('irregular', - _('data is updated in intervals that are uneven in duration')), - ('weekly', _('data is updated on a weekly basis')), - ('biannually', _('data is updated twice each year')), - ('quarterly', _('data is updated every three months')), + ("unknown", _("frequency of maintenance for the data is not known")), + ("continual", _("data is repeatedly and frequently updated")), + ("notPlanned", _("there are no plans to update the data")), + ("daily", _("data is updated each day")), + ("annually", _("data is updated every year")), + ("asNeeded", _("data is updated as deemed necessary")), + ("monthly", _("data is updated each month")), + ("fortnightly", _("data is updated every two weeks")), + ("irregular", _("data is updated in intervals that are uneven in duration")), + ("weekly", _("data is updated on a weekly basis")), + ("biannually", _("data is updated twice each year")), + ("quarterly", _("data is updated every three months")), ) CONTACT_FIELDS = [ - 'name', - 'organization', - 'position', - 'voice', - 'facsimile', - 'delivery_point', - 'city', - 'administrative_area', - 'postal_code', - 'country', - 'email', - 'role' + "name", + "organization", + "position", + "voice", + "facsimile", + "delivery_point", + "city", + "administrative_area", + "postal_code", + "country", + "email", + "role", ] -DEFAULT_SUPPLEMENTAL_INFORMATION = _( - _('No information provided') -) +DEFAULT_SUPPLEMENTAL_INFORMATION = _(_("No information provided")) COUNTRIES = ( - ('AFG', 'Afghanistan'), - ('ALA', 'Aland Islands'), - ('ALB', 'Albania'), - ('DZA', 'Algeria'), - ('ASM', 'American Samoa'), - ('AND', 'Andorra'), - ('AGO', 'Angola'), - ('AIA', 'Anguilla'), - ('ATG', 'Antigua and Barbuda'), - ('ARG', 'Argentina'), - ('ARM', 'Armenia'), - ('ABW', 'Aruba'), - ('AUS', 'Australia'), - ('AUT', 'Austria'), - ('AZE', 'Azerbaijan'), - ('BHS', 'Bahamas'), - ('BHR', 'Bahrain'), - ('BGD', 'Bangladesh'), - ('BRB', 'Barbados'), - ('BLR', 'Belarus'), - ('BEL', 'Belgium'), - ('BLZ', 'Belize'), - ('BEN', 'Benin'), - ('BMU', 'Bermuda'), - ('BTN', 'Bhutan'), - ('BOL', 'Bolivia'), - ('BIH', 'Bosnia and Herzegovina'), - ('BWA', 'Botswana'), - ('BRA', 'Brazil'), - ('VGB', 'British Virgin Islands'), - ('BRN', 'Brunei Darussalam'), - ('BGR', 'Bulgaria'), - ('BFA', 'Burkina Faso'), - ('BDI', 'Burundi'), - ('KHM', 'Cambodia'), - ('CMR', 'Cameroon'), - ('CAN', 'Canada'), - ('CPV', 'Cape Verde'), - ('CYM', 'Cayman Islands'), - ('CAF', 'Central African Republic'), - ('TCD', 'Chad'), - ('CIL', 'Channel Islands'), - ('CHL', 'Chile'), - ('CHN', 'China'), - ('HKG', 'China - Hong Kong'), - ('MAC', 'China - Macao'), - ('COL', 'Colombia'), - ('COM', 'Comoros'), - ('COG', 'Congo'), - ('COK', 'Cook Islands'), - ('CRI', 'Costa Rica'), - ('CIV', 'Cote d\'Ivoire'), - ('HRV', 'Croatia'), - ('CUB', 'Cuba'), - ('CYP', 'Cyprus'), - ('CZE', 'Czech Republic'), - ('PRK', 'Democratic People\'s Republic of Korea'), - ('COD', 'Democratic Republic of the Congo'), - ('DNK', 'Denmark'), - ('DJI', 'Djibouti'), - ('DMA', 'Dominica'), - ('DOM', 'Dominican Republic'), - ('ECU', 'Ecuador'), - ('EGY', 'Egypt'), - ('SLV', 'El Salvador'), - ('GNQ', 'Equatorial Guinea'), - ('ERI', 'Eritrea'), - ('EST', 'Estonia'), - ('ETH', 'Ethiopia'), - ('FRO', 'Faeroe Islands'), - ('FLK', 'Falkland Islands (Malvinas)'), - ('FJI', 'Fiji'), - ('FIN', 'Finland'), - ('FRA', 'France'), - ('GUF', 'French Guiana'), - ('PYF', 'French Polynesia'), - ('GAB', 'Gabon'), - ('GMB', 'Gambia'), - ('GEO', 'Georgia'), - ('DEU', 'Germany'), - ('GHA', 'Ghana'), - ('GIB', 'Gibraltar'), - ('GRC', 'Greece'), - ('GRL', 'Greenland'), - ('GRD', 'Grenada'), - ('GLP', 'Guadeloupe'), - ('GUM', 'Guam'), - ('GTM', 'Guatemala'), - ('GGY', 'Guernsey'), - ('GIN', 'Guinea'), - ('GNB', 'Guinea-Bissau'), - ('GUY', 'Guyana'), - ('HTI', 'Haiti'), - ('VAT', 'Holy See (Vatican City)'), - ('HND', 'Honduras'), - ('HUN', 'Hungary'), - ('ISL', 'Iceland'), - ('IND', 'India'), - ('IDN', 'Indonesia'), - ('IRN', 'Iran'), - ('IRQ', 'Iraq'), - ('IRL', 'Ireland'), - ('IMN', 'Isle of Man'), - ('ISR', 'Israel'), - ('ITA', 'Italy'), - ('JAM', 'Jamaica'), - ('JPN', 'Japan'), - ('JEY', 'Jersey'), - ('JOR', 'Jordan'), - ('KAZ', 'Kazakhstan'), - ('KEN', 'Kenya'), - ('KIR', 'Kiribati'), - ('KWT', 'Kuwait'), - ('KGZ', 'Kyrgyzstan'), - ('LAO', 'Lao People\'s Democratic Republic'), - ('LVA', 'Latvia'), - ('LBN', 'Lebanon'), - ('LSO', 'Lesotho'), - ('LBR', 'Liberia'), - ('LBY', 'Libyan Arab Jamahiriya'), - ('LIE', 'Liechtenstein'), - ('LTU', 'Lithuania'), - ('LUX', 'Luxembourg'), - ('MKD', 'Macedonia'), - ('MDG', 'Madagascar'), - ('MWI', 'Malawi'), - ('MYS', 'Malaysia'), - ('MDV', 'Maldives'), - ('MLI', 'Mali'), - ('MLT', 'Malta'), - ('MHL', 'Marshall Islands'), - ('MTQ', 'Martinique'), - ('MRT', 'Mauritania'), - ('MUS', 'Mauritius'), - ('MYT', 'Mayotte'), - ('MEX', 'Mexico'), - ('FSM', 'Micronesia, Federated States of'), - ('MCO', 'Monaco'), - ('MNG', 'Mongolia'), - ('MNE', 'Montenegro'), - ('MSR', 'Montserrat'), - ('MAR', 'Morocco'), - ('MOZ', 'Mozambique'), - ('MMR', 'Myanmar'), - ('NAM', 'Namibia'), - ('NRU', 'Nauru'), - ('NPL', 'Nepal'), - ('NLD', 'Netherlands'), - ('ANT', 'Netherlands Antilles'), - ('NCL', 'New Caledonia'), - ('NZL', 'New Zealand'), - ('NIC', 'Nicaragua'), - ('NER', 'Niger'), - ('NGA', 'Nigeria'), - ('NIU', 'Niue'), - ('NFK', 'Norfolk Island'), - ('MNP', 'Northern Mariana Islands'), - ('NOR', 'Norway'), - ('PSE', 'Occupied Palestinian Territory'), - ('OMN', 'Oman'), - ('PAK', 'Pakistan'), - ('PLW', 'Palau'), - ('PAN', 'Panama'), - ('PNG', 'Papua New Guinea'), - ('PRY', 'Paraguay'), - ('PER', 'Peru'), - ('PHL', 'Philippines'), - ('PCN', 'Pitcairn'), - ('POL', 'Poland'), - ('PRT', 'Portugal'), - ('PRI', 'Puerto Rico'), - ('QAT', 'Qatar'), - ('KOR', 'Republic of Korea'), - ('MDA', 'Republic of Moldova'), - ('REU', 'Reunion'), - ('ROU', 'Romania'), - ('RUS', 'Russian Federation'), - ('RWA', 'Rwanda'), - ('BLM', 'Saint-Barthelemy'), - ('SHN', 'Saint Helena'), - ('KNA', 'Saint Kitts and Nevis'), - ('LCA', 'Saint Lucia'), - ('MAF', 'Saint-Martin (French part)'), - ('SPM', 'Saint Pierre and Miquelon'), - ('VCT', 'Saint Vincent and the Grenadines'), - ('WSM', 'Samoa'), - ('SMR', 'San Marino'), - ('STP', 'Sao Tome and Principe'), - ('SAU', 'Saudi Arabia'), - ('SEN', 'Senegal'), - ('SRB', 'Serbia'), - ('SYC', 'Seychelles'), - ('SLE', 'Sierra Leone'), - ('SGP', 'Singapore'), - ('SVK', 'Slovakia'), - ('SVN', 'Slovenia'), - ('SLB', 'Solomon Islands'), - ('SOM', 'Somalia'), - ('ZAF', 'South Africa'), - ('SSD', 'South Sudan'), - ('ESP', 'Spain'), - ('LKA', 'Sri Lanka'), - ('SDN', 'Sudan'), - ('SUR', 'Suriname'), - ('SJM', 'Svalbard and Jan Mayen Islands'), - ('SWZ', 'Swaziland'), - ('SWE', 'Sweden'), - ('CHE', 'Switzerland'), - ('SYR', 'Syrian Arab Republic'), - ('TJK', 'Tajikistan'), - ('THA', 'Thailand'), - ('TLS', 'Timor-Leste'), - ('TGO', 'Togo'), - ('TKL', 'Tokelau'), - ('TON', 'Tonga'), - ('TTO', 'Trinidad and Tobago'), - ('TUN', 'Tunisia'), - ('TUR', 'Turkey'), - ('TKM', 'Turkmenistan'), - ('TCA', 'Turks and Caicos Islands'), - ('TUV', 'Tuvalu'), - ('UGA', 'Uganda'), - ('UKR', 'Ukraine'), - ('ARE', 'United Arab Emirates'), - ('GBR', 'United Kingdom'), - ('TZA', 'United Republic of Tanzania'), - ('USA', 'United States of America'), - ('VIR', 'United States Virgin Islands'), - ('URY', 'Uruguay'), - ('UZB', 'Uzbekistan'), - ('VUT', 'Vanuatu'), - ('VEN', 'Venezuela (Bolivarian Republic of)'), - ('VNM', 'Viet Nam'), - ('WLF', 'Wallis and Futuna Islands'), - ('ESH', 'Western Sahara'), - ('YEM', 'Yemen'), - ('ZMB', 'Zambia'), - ('ZWE', 'Zimbabwe'), + ("AFG", "Afghanistan"), + ("ALA", "Aland Islands"), + ("ALB", "Albania"), + ("DZA", "Algeria"), + ("ASM", "American Samoa"), + ("AND", "Andorra"), + ("AGO", "Angola"), + ("AIA", "Anguilla"), + ("ATG", "Antigua and Barbuda"), + ("ARG", "Argentina"), + ("ARM", "Armenia"), + ("ABW", "Aruba"), + ("AUS", "Australia"), + ("AUT", "Austria"), + ("AZE", "Azerbaijan"), + ("BHS", "Bahamas"), + ("BHR", "Bahrain"), + ("BGD", "Bangladesh"), + ("BRB", "Barbados"), + ("BLR", "Belarus"), + ("BEL", "Belgium"), + ("BLZ", "Belize"), + ("BEN", "Benin"), + ("BMU", "Bermuda"), + ("BTN", "Bhutan"), + ("BOL", "Bolivia"), + ("BIH", "Bosnia and Herzegovina"), + ("BWA", "Botswana"), + ("BRA", "Brazil"), + ("VGB", "British Virgin Islands"), + ("BRN", "Brunei Darussalam"), + ("BGR", "Bulgaria"), + ("BFA", "Burkina Faso"), + ("BDI", "Burundi"), + ("KHM", "Cambodia"), + ("CMR", "Cameroon"), + ("CAN", "Canada"), + ("CPV", "Cape Verde"), + ("CYM", "Cayman Islands"), + ("CAF", "Central African Republic"), + ("TCD", "Chad"), + ("CIL", "Channel Islands"), + ("CHL", "Chile"), + ("CHN", "China"), + ("HKG", "China - Hong Kong"), + ("MAC", "China - Macao"), + ("COL", "Colombia"), + ("COM", "Comoros"), + ("COG", "Congo"), + ("COK", "Cook Islands"), + ("CRI", "Costa Rica"), + ("CIV", "Cote d'Ivoire"), + ("HRV", "Croatia"), + ("CUB", "Cuba"), + ("CYP", "Cyprus"), + ("CZE", "Czech Republic"), + ("PRK", "Democratic People's Republic of Korea"), + ("COD", "Democratic Republic of the Congo"), + ("DNK", "Denmark"), + ("DJI", "Djibouti"), + ("DMA", "Dominica"), + ("DOM", "Dominican Republic"), + ("ECU", "Ecuador"), + ("EGY", "Egypt"), + ("SLV", "El Salvador"), + ("GNQ", "Equatorial Guinea"), + ("ERI", "Eritrea"), + ("EST", "Estonia"), + ("ETH", "Ethiopia"), + ("FRO", "Faeroe Islands"), + ("FLK", "Falkland Islands (Malvinas)"), + ("FJI", "Fiji"), + ("FIN", "Finland"), + ("FRA", "France"), + ("GUF", "French Guiana"), + ("PYF", "French Polynesia"), + ("GAB", "Gabon"), + ("GMB", "Gambia"), + ("GEO", "Georgia"), + ("DEU", "Germany"), + ("GHA", "Ghana"), + ("GIB", "Gibraltar"), + ("GRC", "Greece"), + ("GRL", "Greenland"), + ("GRD", "Grenada"), + ("GLP", "Guadeloupe"), + ("GUM", "Guam"), + ("GTM", "Guatemala"), + ("GGY", "Guernsey"), + ("GIN", "Guinea"), + ("GNB", "Guinea-Bissau"), + ("GUY", "Guyana"), + ("HTI", "Haiti"), + ("VAT", "Holy See (Vatican City)"), + ("HND", "Honduras"), + ("HUN", "Hungary"), + ("ISL", "Iceland"), + ("IND", "India"), + ("IDN", "Indonesia"), + ("IRN", "Iran"), + ("IRQ", "Iraq"), + ("IRL", "Ireland"), + ("IMN", "Isle of Man"), + ("ISR", "Israel"), + ("ITA", "Italy"), + ("JAM", "Jamaica"), + ("JPN", "Japan"), + ("JEY", "Jersey"), + ("JOR", "Jordan"), + ("KAZ", "Kazakhstan"), + ("KEN", "Kenya"), + ("KIR", "Kiribati"), + ("KWT", "Kuwait"), + ("KGZ", "Kyrgyzstan"), + ("LAO", "Lao People's Democratic Republic"), + ("LVA", "Latvia"), + ("LBN", "Lebanon"), + ("LSO", "Lesotho"), + ("LBR", "Liberia"), + ("LBY", "Libyan Arab Jamahiriya"), + ("LIE", "Liechtenstein"), + ("LTU", "Lithuania"), + ("LUX", "Luxembourg"), + ("MKD", "Macedonia"), + ("MDG", "Madagascar"), + ("MWI", "Malawi"), + ("MYS", "Malaysia"), + ("MDV", "Maldives"), + ("MLI", "Mali"), + ("MLT", "Malta"), + ("MHL", "Marshall Islands"), + ("MTQ", "Martinique"), + ("MRT", "Mauritania"), + ("MUS", "Mauritius"), + ("MYT", "Mayotte"), + ("MEX", "Mexico"), + ("FSM", "Micronesia, Federated States of"), + ("MCO", "Monaco"), + ("MNG", "Mongolia"), + ("MNE", "Montenegro"), + ("MSR", "Montserrat"), + ("MAR", "Morocco"), + ("MOZ", "Mozambique"), + ("MMR", "Myanmar"), + ("NAM", "Namibia"), + ("NRU", "Nauru"), + ("NPL", "Nepal"), + ("NLD", "Netherlands"), + ("ANT", "Netherlands Antilles"), + ("NCL", "New Caledonia"), + ("NZL", "New Zealand"), + ("NIC", "Nicaragua"), + ("NER", "Niger"), + ("NGA", "Nigeria"), + ("NIU", "Niue"), + ("NFK", "Norfolk Island"), + ("MNP", "Northern Mariana Islands"), + ("NOR", "Norway"), + ("PSE", "Occupied Palestinian Territory"), + ("OMN", "Oman"), + ("PAK", "Pakistan"), + ("PLW", "Palau"), + ("PAN", "Panama"), + ("PNG", "Papua New Guinea"), + ("PRY", "Paraguay"), + ("PER", "Peru"), + ("PHL", "Philippines"), + ("PCN", "Pitcairn"), + ("POL", "Poland"), + ("PRT", "Portugal"), + ("PRI", "Puerto Rico"), + ("QAT", "Qatar"), + ("KOR", "Republic of Korea"), + ("MDA", "Republic of Moldova"), + ("REU", "Reunion"), + ("ROU", "Romania"), + ("RUS", "Russian Federation"), + ("RWA", "Rwanda"), + ("BLM", "Saint-Barthelemy"), + ("SHN", "Saint Helena"), + ("KNA", "Saint Kitts and Nevis"), + ("LCA", "Saint Lucia"), + ("MAF", "Saint-Martin (French part)"), + ("SPM", "Saint Pierre and Miquelon"), + ("VCT", "Saint Vincent and the Grenadines"), + ("WSM", "Samoa"), + ("SMR", "San Marino"), + ("STP", "Sao Tome and Principe"), + ("SAU", "Saudi Arabia"), + ("SEN", "Senegal"), + ("SRB", "Serbia"), + ("SYC", "Seychelles"), + ("SLE", "Sierra Leone"), + ("SGP", "Singapore"), + ("SVK", "Slovakia"), + ("SVN", "Slovenia"), + ("SLB", "Solomon Islands"), + ("SOM", "Somalia"), + ("ZAF", "South Africa"), + ("SSD", "South Sudan"), + ("ESP", "Spain"), + ("LKA", "Sri Lanka"), + ("SDN", "Sudan"), + ("SUR", "Suriname"), + ("SJM", "Svalbard and Jan Mayen Islands"), + ("SWZ", "Swaziland"), + ("SWE", "Sweden"), + ("CHE", "Switzerland"), + ("SYR", "Syrian Arab Republic"), + ("TJK", "Tajikistan"), + ("THA", "Thailand"), + ("TLS", "Timor-Leste"), + ("TGO", "Togo"), + ("TKL", "Tokelau"), + ("TON", "Tonga"), + ("TTO", "Trinidad and Tobago"), + ("TUN", "Tunisia"), + ("TUR", "Turkey"), + ("TKM", "Turkmenistan"), + ("TCA", "Turks and Caicos Islands"), + ("TUV", "Tuvalu"), + ("UGA", "Uganda"), + ("UKR", "Ukraine"), + ("ARE", "United Arab Emirates"), + ("GBR", "United Kingdom"), + ("TZA", "United Republic of Tanzania"), + ("USA", "United States of America"), + ("VIR", "United States Virgin Islands"), + ("URY", "Uruguay"), + ("UZB", "Uzbekistan"), + ("VUT", "Vanuatu"), + ("VEN", "Venezuela (Bolivarian Republic of)"), + ("VNM", "Viet Nam"), + ("WLF", "Wallis and Futuna Islands"), + ("ESH", "Western Sahara"), + ("YEM", "Yemen"), + ("ZMB", "Zambia"), + ("ZWE", "Zimbabwe"), ) # Taken from http://www.w3.org/WAI/ER/IG/ert/iso639.htm ALL_LANGUAGES = ( - ('abk', 'Abkhazian'), - ('aar', 'Afar'), - ('afr', 'Afrikaans'), - ('amh', 'Amharic'), - ('ara', 'Arabic'), - ('asm', 'Assamese'), - ('aym', 'Aymara'), - ('aze', 'Azerbaijani'), - ('bak', 'Bashkir'), - ('ben', 'Bengali'), - ('bih', 'Bihari'), - ('bis', 'Bislama'), - ('bre', 'Breton'), - ('bul', 'Bulgarian'), - ('bel', 'Byelorussian'), - ('cat', 'Catalan'), - ('chi', 'Chinese'), - ('cos', 'Corsican'), - ('dan', 'Danish'), - ('dzo', 'Dzongkha'), - ('eng', 'English'), - ('fra', 'French'), - ('epo', 'Esperanto'), - ('est', 'Estonian'), - ('fao', 'Faroese'), - ('fij', 'Fijian'), - ('fin', 'Finnish'), - ('fry', 'Frisian'), - ('glg', 'Gallegan'), - ('ger', 'German'), - ('gre', 'Greek'), - ('kal', 'Greenlandic'), - ('grn', 'Guarani'), - ('guj', 'Gujarati'), - ('hau', 'Hausa'), - ('heb', 'Hebrew'), - ('hin', 'Hindi'), - ('hun', 'Hungarian'), - ('ind', 'Indonesian'), - ('ina', 'Interlingua (International Auxiliary language Association)'), - ('iku', 'Inuktitut'), - ('ipk', 'Inupiak'), - ('ita', 'Italian'), - ('jpn', 'Japanese'), - ('kan', 'Kannada'), - ('kas', 'Kashmiri'), - ('kaz', 'Kazakh'), - ('khm', 'Khmer'), - ('kin', 'Kinyarwanda'), - ('kir', 'Kirghiz'), - ('kor', 'Korean'), - ('kur', 'Kurdish'), - ('oci', 'Langue d \'Oc (post 1500)'), - ('lao', 'Lao'), - ('lat', 'Latin'), - ('lav', 'Latvian'), - ('lin', 'Lingala'), - ('lit', 'Lithuanian'), - ('mlg', 'Malagasy'), - ('mlt', 'Maltese'), - ('mar', 'Marathi'), - ('mol', 'Moldavian'), - ('mon', 'Mongolian'), - ('nau', 'Nauru'), - ('nep', 'Nepali'), - ('nor', 'Norwegian'), - ('ori', 'Oriya'), - ('orm', 'Oromo'), - ('pan', 'Panjabi'), - ('pol', 'Polish'), - ('por', 'Portuguese'), - ('pus', 'Pushto'), - ('que', 'Quechua'), - ('roh', 'Rhaeto-Romance'), - ('run', 'Rundi'), - ('rus', 'Russian'), - ('smo', 'Samoan'), - ('sag', 'Sango'), - ('san', 'Sanskrit'), - ('scr', 'Serbo-Croatian'), - ('sna', 'Shona'), - ('snd', 'Sindhi'), - ('sin', 'Singhalese'), - ('ssw', 'Siswant'), - ('slv', 'Slovenian'), - ('som', 'Somali'), - ('sot', 'Sotho'), - ('spa', 'Spanish'), - ('sun', 'Sudanese'), - ('swa', 'Swahili'), - ('tgl', 'Tagalog'), - ('tgk', 'Tajik'), - ('tam', 'Tamil'), - ('tat', 'Tatar'), - ('tel', 'Telugu'), - ('tha', 'Thai'), - ('tir', 'Tigrinya'), - ('tog', 'Tonga (Nyasa)'), - ('tso', 'Tsonga'), - ('tsn', 'Tswana'), - ('tur', 'Turkish'), - ('tuk', 'Turkmen'), - ('twi', 'Twi'), - ('uig', 'Uighur'), - ('ukr', 'Ukrainian'), - ('urd', 'Urdu'), - ('uzb', 'Uzbek'), - ('vie', 'Vietnamese'), - ('vol', 'Volapük'), - ('wol', 'Wolof'), - ('xho', 'Xhosa'), - ('yid', 'Yiddish'), - ('yor', 'Yoruba'), - ('zha', 'Zhuang'), - ('zul', 'Zulu'), + ("abk", "Abkhazian"), + ("aar", "Afar"), + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aym", "Aymara"), + ("aze", "Azerbaijani"), + ("bak", "Bashkir"), + ("ben", "Bengali"), + ("bih", "Bihari"), + ("bis", "Bislama"), + ("bre", "Breton"), + ("bul", "Bulgarian"), + ("bel", "Byelorussian"), + ("cat", "Catalan"), + ("chi", "Chinese"), + ("cos", "Corsican"), + ("dan", "Danish"), + ("dzo", "Dzongkha"), + ("eng", "English"), + ("fra", "French"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("fao", "Faroese"), + ("fij", "Fijian"), + ("fin", "Finnish"), + ("fry", "Frisian"), + ("glg", "Gallegan"), + ("ger", "German"), + ("gre", "Greek"), + ("kal", "Greenlandic"), + ("grn", "Guarani"), + ("guj", "Gujarati"), + ("hau", "Hausa"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hun", "Hungarian"), + ("ind", "Indonesian"), + ("ina", "Interlingua (International Auxiliary language Association)"), + ("iku", "Inuktitut"), + ("ipk", "Inupiak"), + ("ita", "Italian"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kas", "Kashmiri"), + ("kaz", "Kazakh"), + ("khm", "Khmer"), + ("kin", "Kinyarwanda"), + ("kir", "Kirghiz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("oci", "Langue d 'Oc (post 1500)"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lin", "Lingala"), + ("lit", "Lithuanian"), + ("mlg", "Malagasy"), + ("mlt", "Maltese"), + ("mar", "Marathi"), + ("mol", "Moldavian"), + ("mon", "Mongolian"), + ("nau", "Nauru"), + ("nep", "Nepali"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("orm", "Oromo"), + ("pan", "Panjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto"), + ("que", "Quechua"), + ("roh", "Rhaeto-Romance"), + ("run", "Rundi"), + ("rus", "Russian"), + ("smo", "Samoan"), + ("sag", "Sango"), + ("san", "Sanskrit"), + ("scr", "Serbo-Croatian"), + ("sna", "Shona"), + ("snd", "Sindhi"), + ("sin", "Singhalese"), + ("ssw", "Siswant"), + ("slv", "Slovenian"), + ("som", "Somali"), + ("sot", "Sotho"), + ("spa", "Spanish"), + ("sun", "Sudanese"), + ("swa", "Swahili"), + ("tgl", "Tagalog"), + ("tgk", "Tajik"), + ("tam", "Tamil"), + ("tat", "Tatar"), + ("tel", "Telugu"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tog", "Tonga (Nyasa)"), + ("tso", "Tsonga"), + ("tsn", "Tswana"), + ("tur", "Turkish"), + ("tuk", "Turkmen"), + ("twi", "Twi"), + ("uig", "Uighur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("vie", "Vietnamese"), + ("vol", "Volapük"), + ("wol", "Wolof"), + ("xho", "Xhosa"), + ("yid", "Yiddish"), + ("yor", "Yoruba"), + ("zha", "Zhuang"), + ("zul", "Zulu"), ) CHARSETS = ( - ('', 'None/Unknown'), - ('UTF-8', 'UTF-8/Unicode'), - ('ISO-8859-1', 'Latin1/ISO-8859-1'), - ('ISO-8859-2', 'Latin2/ISO-8859-2'), - ('ISO-8859-3', 'Latin3/ISO-8859-3'), - ('ISO-8859-4', 'Latin4/ISO-8859-4'), - ('ISO-8859-5', 'Latin5/ISO-8859-5'), - ('ISO-8859-6', 'Latin6/ISO-8859-6'), - ('ISO-8859-7', 'Latin7/ISO-8859-7'), - ('ISO-8859-8', 'Latin8/ISO-8859-8'), - ('ISO-8859-9', 'Latin9/ISO-8859-9'), - ('ISO-8859-10', 'Latin10/ISO-8859-10'), - ('ISO-8859-13', 'Latin13/ISO-8859-13'), - ('ISO-8859-14', 'Latin14/ISO-8859-14'), - ('ISO8859-15', 'Latin15/ISO-8859-15'), - ('Big5', 'BIG5'), - ('EUC-JP', 'EUC-JP'), - ('EUC-KR', 'EUC-KR'), - ('GBK', 'GBK'), - ('GB18030', 'GB18030'), - ('Shift_JIS', 'Shift_JIS'), - ('KOI8-R', 'KOI8-R'), - ('KOI8-U', 'KOI8-U'), - ('cp874', 'Windows CP874'), - ('windows-1250', 'Windows CP1250'), - ('windows-1251', 'Windows CP1251'), - ('windows-1252', 'Windows CP1252'), - ('windows-1253', 'Windows CP1253'), - ('windows-1254', 'Windows CP1254'), - ('windows-1255', 'Windows CP1255'), - ('windows-1256', 'Windows CP1256'), - ('windows-1257', 'Windows CP1257'), - ('windows-1258', 'Windows CP1258') + ("", "None/Unknown"), + ("UTF-8", "UTF-8/Unicode"), + ("ISO-8859-1", "Latin1/ISO-8859-1"), + ("ISO-8859-2", "Latin2/ISO-8859-2"), + ("ISO-8859-3", "Latin3/ISO-8859-3"), + ("ISO-8859-4", "Latin4/ISO-8859-4"), + ("ISO-8859-5", "Latin5/ISO-8859-5"), + ("ISO-8859-6", "Latin6/ISO-8859-6"), + ("ISO-8859-7", "Latin7/ISO-8859-7"), + ("ISO-8859-8", "Latin8/ISO-8859-8"), + ("ISO-8859-9", "Latin9/ISO-8859-9"), + ("ISO-8859-10", "Latin10/ISO-8859-10"), + ("ISO-8859-13", "Latin13/ISO-8859-13"), + ("ISO-8859-14", "Latin14/ISO-8859-14"), + ("ISO8859-15", "Latin15/ISO-8859-15"), + ("Big5", "BIG5"), + ("EUC-JP", "EUC-JP"), + ("EUC-KR", "EUC-KR"), + ("GBK", "GBK"), + ("GB18030", "GB18030"), + ("Shift_JIS", "Shift_JIS"), + ("KOI8-R", "KOI8-R"), + ("KOI8-U", "KOI8-U"), + ("cp874", "Windows CP874"), + ("windows-1250", "Windows CP1250"), + ("windows-1251", "Windows CP1251"), + ("windows-1252", "Windows CP1252"), + ("windows-1253", "Windows CP1253"), + ("windows-1254", "Windows CP1254"), + ("windows-1255", "Windows CP1255"), + ("windows-1256", "Windows CP1256"), + ("windows-1257", "Windows CP1257"), + ("windows-1258", "Windows CP1258"), ) STATE_READY = "READY" @@ -502,4 +498,4 @@ (SOURCE_TYPE_COPYREMOTE, "COPYREMOTE"), ) -LAYER_TYPES = ['vector', 'raster', 'remote', 'vector_time'] +LAYER_TYPES = ["vector", "raster", "remote", "vector_time"] diff --git a/geonode/base/fields.py b/geonode/base/fields.py index 728f23c62c8..d23241fa9ae 100644 --- a/geonode/base/fields.py +++ b/geonode/base/fields.py @@ -21,7 +21,6 @@ class MultiThesauriField(forms.ModelMultipleChoiceField): - def label_from_instance(self, obj): # Note: Not using .get() because filter()[0] is used in original # code. The hard-coded language is currently used throughout diff --git a/geonode/base/forms.py b/geonode/base/forms.py index a3146b6ecd9..cdd1a4f339a 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -43,10 +43,17 @@ from django.utils.translation import get_language from geonode.base.enumerations import ALL_LANGUAGES -from geonode.base.models import (HierarchicalKeyword, - License, Region, ResourceBase, Thesaurus, - ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel, - TopicCategory) +from geonode.base.models import ( + HierarchicalKeyword, + License, + Region, + ResourceBase, + Thesaurus, + ThesaurusKeyword, + ThesaurusKeywordLabel, + ThesaurusLabel, + TopicCategory, +) from geonode.base.widgets import TaggitSelect2Custom from geonode.base.fields import MultiThesauriField from geonode.documents.models import Document @@ -57,14 +64,11 @@ def get_tree_data(): - def rectree(parent, path): children_list_of_tuples = list() c = Region.objects.filter(parent=parent) for child in c: - children_list_of_tuples.append( - tuple((path + parent.name, tuple((child.id, child.name)))) - ) + children_list_of_tuples.append(tuple((path + parent.name, tuple((child.id, child.name))))) childrens = rectree(child, f"{parent.name}/") if childrens: children_list_of_tuples.extend(childrens) @@ -75,14 +79,10 @@ def rectree(parent, path): try: t = Region.objects.filter(Q(level=0) | Q(parent=None)) for toplevel in t: - data.append( - tuple((toplevel.id, toplevel.name)) - ) - childrens = rectree(toplevel, '') + data.append(tuple((toplevel.id, toplevel.name))) + childrens = rectree(toplevel, "") if childrens: - data.append( - tuple((toplevel.name, childrens)) - ) + data.append(tuple((toplevel.name, childrens))) except Exception: pass @@ -91,15 +91,12 @@ def rectree(parent, path): class AdvancedModelChoiceIterator(models.ModelChoiceIterator): def choice(self, obj): - return ( - self.field.prepare_value(obj), - self.field.label_from_instance(obj), - obj) + return (self.field.prepare_value(obj), self.field.label_from_instance(obj), obj) class CategoryChoiceField(forms.ModelChoiceField): def _get_choices(self): - if hasattr(self, '_choices'): + if hasattr(self, "_choices"): return self._choices return AdvancedModelChoiceIterator(self) @@ -107,22 +104,22 @@ def _get_choices(self): choices = property(_get_choices, ChoiceField._set_choices) def label_from_instance(self, obj): - return '' \ - '' \ - '' \ - '
' + obj.gn_description + '' + return ( + '' + '' + '' + "
" + obj.gn_description + "" + ) class RegionsMultipleChoiceField(forms.MultipleChoiceField): - def validate(self, value): """ Validates that the input is a list or tuple. """ if self.required and not value: - raise forms.ValidationError( - self.error_messages['required'], code='required') + raise forms.ValidationError(self.error_messages["required"], code="required") class RegionsSelect(forms.Select): @@ -133,15 +130,12 @@ def render(self, name, value, attrs=None, renderer=None): value = [] final_attrs = self.build_attrs(attrs) final_attrs["name"] = name - output = [ - format_html( - '', flatatt(final_attrs))] options = self.render_options(value) if options: output.append(options) - output.append('') - return mark_safe('\n'.join(output)) + output.append("") + return mark_safe("\n".join(output)) def value_from_datadict(self, data, files, name): try: @@ -150,47 +144,36 @@ def value_from_datadict(self, data, files, name): getter = data.get return getter(name) - def render_option_value( - self, - selected_choices, - option_value, - option_label, - data_section=None): + def render_option_value(self, selected_choices, option_value, option_label, data_section=None): if option_value is None: - option_value = '' + option_value = "" option_value = force_text(option_value) if option_value in selected_choices: - selected_html = mark_safe(' selected') + selected_html = mark_safe(" selected") if not self.allow_multiple_selected: # Only allow for a single selection. selected_choices.remove(option_value) else: - selected_html = '' + selected_html = "" label = force_text(option_label) if data_section is None: - data_section = '' + data_section = "" else: data_section = force_text(data_section) - if '/' in data_section: - label = format_html( - '{} [{}]', label, data_section.rsplit( - '/', 1)[1]) + if "/" in data_section: + label = format_html("{} [{}]", label, data_section.rsplit("/", 1)[1]) return format_html( - '', - data_section, - option_value, - selected_html, - label) + '', data_section, option_value, selected_html, label + ) def render_options(self, selected_choices): # Normalize to strings. def _region_id_from_choice(choice): - if isinstance(choice, int) or \ - (isinstance(choice, str) and choice.isdigit()): + if isinstance(choice, int) or (isinstance(choice, str) and choice.isdigit()): return int(choice) else: return choice.id @@ -198,56 +181,37 @@ def _region_id_from_choice(choice): selected_choices = {force_text(_region_id_from_choice(v)) for v in selected_choices} output = [] - output.append(format_html('', 'Global')) + output.append(format_html('', "Global")) for option_value, option_label in self.choices: - if not isinstance( - option_label, (list, tuple)) and isinstance( - option_label, str): - output.append( - self.render_option_value( - selected_choices, - option_value, - option_label)) - output.append('') + if not isinstance(option_label, (list, tuple)) and isinstance(option_label, str): + output.append(self.render_option_value(selected_choices, option_value, option_label)) + output.append("") for option_value, option_label in self.choices: - if isinstance( - option_label, (list, tuple)) and not isinstance( - option_label, str): - output.append( - format_html( - '', - force_text(option_value))) + if isinstance(option_label, (list, tuple)) and not isinstance(option_label, str): + output.append(format_html('', force_text(option_value))) for option in option_label: - if isinstance( - option, (list, tuple)) and not isinstance( - option, str): - if isinstance( - option[1][0], (list, tuple)) and not isinstance( - option[1][0], str): + if isinstance(option, (list, tuple)) and not isinstance(option, str): + if isinstance(option[1][0], (list, tuple)) and not isinstance(option[1][0], str): for option_child in option[1][0]: output.append( self.render_option_value( - selected_choices, - *option_child, - data_section=force_text( - option[1][0][0]))) + selected_choices, *option_child, data_section=force_text(option[1][0][0]) + ) + ) else: output.append( self.render_option_value( - selected_choices, - *option[1], - data_section=force_text( - option[0]))) + selected_choices, *option[1], data_section=force_text(option[0]) + ) + ) else: output.append( - self.render_option_value( - selected_choices, - *option, - data_section=force_text(option_value))) - output.append('') + self.render_option_value(selected_choices, *option, data_section=force_text(option_value)) + ) + output.append("") - return '\n'.join(output) + return "\n".join(output) class CategoryForm(forms.Form): @@ -255,14 +219,13 @@ class CategoryForm(forms.Form): required=False, label=f"*{_('Category')}", empty_label=None, - queryset=TopicCategory.objects.filter( - is_choice=True).extra( - order_by=['description'])) + queryset=TopicCategory.objects.filter(is_choice=True).extra(order_by=["description"]), + ) def clean(self): cleaned_data = self.data ccf_data = cleaned_data.get("category_choice_field") - category_mandatory = getattr(settings, 'TOPICCATEGORY_MANDATORY', False) + category_mandatory = getattr(settings, "TOPICCATEGORY_MANDATORY", False) if category_mandatory and not ccf_data: msg = _("Category is required.") self._errors = self.error_class([msg]) @@ -272,22 +235,24 @@ def clean(self): class TKeywordForm(forms.ModelForm): - prefix = 'tkeywords' + prefix = "tkeywords" class Meta: model = Document - fields = ['tkeywords'] + fields = ["tkeywords"] tkeywords = MultiThesauriField( queryset=ThesaurusKeyword.objects.prefetch_related( - Prefetch('keyword', queryset=ThesaurusKeywordLabel.objects.filter(lang='en')) + Prefetch("keyword", queryset=ThesaurusKeywordLabel.objects.filter(lang="en")) ), widget=autocomplete.ModelSelect2Multiple( - url='thesaurus_autocomplete', + url="thesaurus_autocomplete", ), label=_("Keywords from Thesaurus"), required=False, - help_text=_("List of keywords from Thesaurus", ), + help_text=_( + "List of keywords from Thesaurus", + ), ) @@ -302,7 +267,7 @@ class ThesaurusAvailableForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) lang = get_language() - for item in Thesaurus.objects.all().order_by('order', 'id'): + for item in Thesaurus.objects.all().order_by("order", "id"): tname = self._get_thesauro_title_label(item, lang) if item.card_max == 0: continue @@ -341,7 +306,8 @@ def _define_choicefield(self, item, required, tname, lang): label=f"{tname}", required=False, widget=forms.Select(attrs={"class": "treq" if required else ""}), - choices=self._get_thesauro_keyword_label(item, lang)) + choices=self._get_thesauro_keyword_label(item, lang), + ) @staticmethod def _get_thesauro_keyword_label(item, lang): @@ -349,14 +315,27 @@ def _get_thesauro_keyword_label(item, lang): keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item) # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again - qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") if len(qs_keyword_ids) == 0: lang = remove_country_from_languagecode(lang) - qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") - - not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + + not_qs_ids = ( + ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids) + .order_by("keyword_id") + .distinct("keyword_id") + .values("keyword_id") + ) - qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label")) + qs_local = list( + ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list( + "keyword_id", "label" + ) + ) qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label")) return [THESAURUS_RESULT_LIST_SEPERATOR] + qs_local + [THESAURUS_RESULT_LIST_SEPERATOR] + qs_non_local @@ -370,7 +349,6 @@ def _get_thesauro_title_label(item, lang): class ResourceBaseDateTimePicker(DateTimePicker): - def build_attrs(self, base_attrs=None, extra_attrs=None, **kwargs): "Helper function for building an attribute dictionary." if extra_attrs: @@ -383,145 +361,139 @@ def build_attrs(self, base_attrs=None, extra_attrs=None, **kwargs): class ResourceBaseForm(TranslationModelForm): """Base form for metadata, should be inherited by childres classes of ResourceBase""" - abstract = forms.CharField( - label=_("Abstract"), - required=False, - widget=TinyMCE()) - purpose = forms.CharField( - label=_("Purpose"), - required=False, - widget=TinyMCE()) + abstract = forms.CharField(label=_("Abstract"), required=False, widget=TinyMCE()) - constraints_other = forms.CharField( - label=_("Other constraints"), - required=False, - widget=TinyMCE()) + purpose = forms.CharField(label=_("Purpose"), required=False, widget=TinyMCE()) - supplemental_information = forms.CharField( - label=_('Supplemental information'), - required=False, - widget=TinyMCE()) + constraints_other = forms.CharField(label=_("Other constraints"), required=False, widget=TinyMCE()) + + supplemental_information = forms.CharField(label=_("Supplemental information"), required=False, widget=TinyMCE()) ptype = forms.CharField(required=False) sourcetype = forms.CharField(required=False) - data_quality_statement = forms.CharField( - label=_("Data quality statement"), - required=False, - widget=TinyMCE()) + data_quality_statement = forms.CharField(label=_("Data quality statement"), required=False, widget=TinyMCE()) owner = forms.ModelChoiceField( empty_label=_("Owner"), label=_("Owner"), required=True, - queryset=get_user_model().objects.exclude(username='AnonymousUser'), - widget=autocomplete.ModelSelect2(url='autocomplete_profile')) + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=autocomplete.ModelSelect2(url="autocomplete_profile"), + ) date = forms.DateTimeField( label=_("Date"), localize=True, - input_formats=['%Y-%m-%d %H:%M %p'], - widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}) + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), ) temporal_extent_start = forms.DateTimeField( label=_("temporal extent start"), required=False, localize=True, - input_formats=['%Y-%m-%d %H:%M %p'], - widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}) + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), ) temporal_extent_end = forms.DateTimeField( label=_("temporal extent end"), required=False, localize=True, - input_formats=['%Y-%m-%d %H:%M %p'], - widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}) + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), ) poc = forms.ModelChoiceField( empty_label=_("Person outside GeoNode (fill form)"), label=_("Point of Contact"), required=False, - queryset=get_user_model().objects.exclude( - username='AnonymousUser'), - widget=autocomplete.ModelSelect2(url='autocomplete_profile')) + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=autocomplete.ModelSelect2(url="autocomplete_profile"), + ) metadata_author = forms.ModelChoiceField( empty_label=_("Person outside GeoNode (fill form)"), label=_("Metadata Author"), required=False, - queryset=get_user_model().objects.exclude( - username='AnonymousUser'), - widget=autocomplete.ModelSelect2(url='autocomplete_profile')) + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=autocomplete.ModelSelect2(url="autocomplete_profile"), + ) keywords = TagField( label=_("Free-text Keywords"), required=False, help_text=_("A space or comma-separated list of keywords. Use the widget to select from Hierarchical tree."), # widget=TreeWidget(url='autocomplete_hierachical_keyword'), #Needs updating to work with select2 - widget=TaggitSelect2Custom(url='autocomplete_hierachical_keyword')) + widget=TaggitSelect2Custom(url="autocomplete_hierachical_keyword"), + ) - regions = RegionsMultipleChoiceField( - label=_("Regions"), - required=False, - widget=RegionsSelect) + regions = RegionsMultipleChoiceField(label=_("Regions"), required=False, widget=RegionsSelect) regions.widget.attrs = {"size": 20} extra_metadata = forms.CharField( required=False, widget=forms.Textarea, - help_text=_('Additional metadata, must be in format [\ + help_text=_( + 'Additional metadata, must be in format [\ {"metadata_key": "metadata_value"},\ {"metadata_key": "metadata_value"} \ - ]') + ]' + ), ) def __init__(self, *args, **kwargs): - self.user = kwargs.pop('user', None) + self.user = kwargs.pop("user", None) super().__init__(*args, **kwargs) - self.fields['regions'].choices = get_tree_data() + self.fields["regions"].choices = get_tree_data() self.can_change_perms = self.user and self.user.has_perm( - 'change_resourcebase_permissions', self.instance.get_self_resource() + "change_resourcebase_permissions", self.instance.get_self_resource() ) if self.instance and self.instance.id and self.instance.metadata.exists(): - self.fields['extra_metadata'].initial = [x.metadata for x in self.instance.metadata.all()] + self.fields["extra_metadata"].initial = [x.metadata for x in self.instance.metadata.all()] for field in self.fields: - if field == 'featured' and self.user and not self.user.is_superuser: + if field == "featured" and self.user and not self.user.is_superuser: self.fields[field].disabled = True help_text = self.fields[field].help_text - if help_text != '': + if help_text != "": self.fields[field].widget.attrs.update( { - 'class': 'has-popover', - 'data-content': help_text, - 'data-placement': 'right', - 'data-container': 'body', - 'data-html': 'true'}) + "class": "has-popover", + "data-content": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", + } + ) - if field in ['poc', 'owner'] and not self.can_change_perms: + if field in ["poc", "owner"] and not self.can_change_perms: self.fields[field].disabled = True def disable_keywords_widget_for_non_superuser(self, user): if settings.FREETEXT_KEYWORDS_READONLY and not user.is_superuser: - self['keywords'].field.disabled = True + self["keywords"].field.disabled = True def clean_keywords(self): - keywords = self.cleaned_data['keywords'] + keywords = self.cleaned_data["keywords"] _unsescaped_kwds = [] for k in keywords: - _k = ('%s' % re.sub(r'%([A-Z0-9]{2})', r'&#x\g<1>;', k.strip())).split(",") + _k = ("%s" % re.sub(r"%([A-Z0-9]{2})", r"&#x\g<1>;", k.strip())).split(",") if not isinstance(_k, str): for _kk in [html.unescape(x.strip()) for x in _k]: # Simulate JS Unescape - _kk = _kk.replace('%u', r'\u').encode('unicode-escape').replace( - b'\\\\u', - b'\\u').decode('unicode-escape') if '%u' in _kk else _kk - _hk = HierarchicalKeyword.objects.filter(name__iexact=f'{_kk.strip()}') + _kk = ( + _kk.replace("%u", r"\u") + .encode("unicode-escape") + .replace(b"\\\\u", b"\\u") + .decode("unicode-escape") + if "%u" in _kk + else _kk + ) + _hk = HierarchicalKeyword.objects.filter(name__iexact=f"{_kk.strip()}") if _hk and len(_hk) > 0: _unsescaped_kwds.append(str(_hk[0])) else: @@ -541,51 +513,50 @@ def clean_title(self): return title def clean_extra_metadata(self): - cleaned_data = self.cleaned_data.get('extra_metadata', []) + cleaned_data = self.cleaned_data.get("extra_metadata", []) return json.dumps(validate_extra_metadata(cleaned_data, self.instance), indent=4) class Meta: exclude = ( - 'contacts', - 'name', - 'uuid', - 'bbox_polygon', - 'll_bbox_polygon', - 'srid', - 'category', - 'csw_typename', - 'csw_schema', - 'csw_mdsource', - 'csw_type', - 'csw_wkt_geometry', - 'metadata_uploaded', - 'metadata_xml', - 'csw_anytext', - 'popular_count', - 'share_count', - 'thumbnail', - 'charset', - 'rating', - 'detail_url', - 'tkeywords', - 'users_geolimits', - 'groups_geolimits', - 'dirty_state', - 'state', - 'blob', - 'files', - 'was_approved', - 'was_published' + "contacts", + "name", + "uuid", + "bbox_polygon", + "ll_bbox_polygon", + "srid", + "category", + "csw_typename", + "csw_schema", + "csw_mdsource", + "csw_type", + "csw_wkt_geometry", + "metadata_uploaded", + "metadata_xml", + "csw_anytext", + "popular_count", + "share_count", + "thumbnail", + "charset", + "rating", + "detail_url", + "tkeywords", + "users_geolimits", + "groups_geolimits", + "dirty_state", + "state", + "blob", + "files", + "was_approved", + "was_published", ) class ValuesListField(forms.Field): - def to_python(self, value): if value in validators.EMPTY_VALUES: return [] - value = [item.strip() for item in value.split(',') if item.strip()] + value = [item.strip() for item in value.split(",") if item.strip()] return value @@ -597,32 +568,15 @@ def clean(self, value): class BatchEditForm(forms.Form): - LANGUAGES = (('', '--------'),) + ALL_LANGUAGES - group = forms.ModelChoiceField( - label=_('Group'), - queryset=Group.objects.all(), - required=False) - owner = forms.ModelChoiceField( - label=_('Owner'), - queryset=get_user_model().objects.all(), - required=False) - category = forms.ModelChoiceField( - label=_('Category'), - queryset=TopicCategory.objects.all(), - required=False) - license = forms.ModelChoiceField( - label=_('License'), - queryset=License.objects.all(), - required=False) - regions = forms.ModelChoiceField( - label=_('Regions'), - queryset=Region.objects.all(), - required=False) - date = forms.DateTimeField( - label=_('Date'), - required=False) + LANGUAGES = (("", "--------"),) + ALL_LANGUAGES + group = forms.ModelChoiceField(label=_("Group"), queryset=Group.objects.all(), required=False) + owner = forms.ModelChoiceField(label=_("Owner"), queryset=get_user_model().objects.all(), required=False) + category = forms.ModelChoiceField(label=_("Category"), queryset=TopicCategory.objects.all(), required=False) + license = forms.ModelChoiceField(label=_("License"), queryset=License.objects.all(), required=False) + regions = forms.ModelChoiceField(label=_("Regions"), queryset=Region.objects.all(), required=False) + date = forms.DateTimeField(label=_("Date"), required=False) language = forms.ChoiceField( - label=_('Language'), + label=_("Language"), required=False, choices=LANGUAGES, ) @@ -632,7 +586,7 @@ class BatchEditForm(forms.Form): def get_user_choices(): try: - return [(x.pk, x.title) for x in Dataset.objects.all().order_by('id')] + return [(x.pk, x.title) for x in Dataset.objects.all().order_by("id")] except Exception: return [] @@ -640,13 +594,11 @@ def get_user_choices(): class UserAndGroupPermissionsForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['layers'].label_from_instance = self.label_from_instance + self.fields["layers"].label_from_instance = self.label_from_instance layers = MultipleChoiceField( choices=get_user_choices(), - widget=autocomplete.Select2Multiple( - url='datasets_autocomplete' - ), + widget=autocomplete.Select2Multiple(url="datasets_autocomplete"), label="Datasets", required=False, ) @@ -655,17 +607,17 @@ def __init__(self, *args, **kwargs): required=True, widget=forms.RadioSelect, choices=( - ('view', 'View'), - ('download', 'Download'), - ('edit', 'Edit'), + ("view", "View"), + ("download", "Download"), + ("edit", "Edit"), ), ) mode = forms.ChoiceField( required=True, widget=forms.RadioSelect, choices=( - ('set', 'Set'), - ('unset', 'Unset'), + ("set", "Set"), + ("unset", "Unset"), ), ) ids = forms.CharField(required=False, widget=forms.HiddenInput()) @@ -677,17 +629,14 @@ def label_from_instance(obj): class OwnerRightsRequestForm(forms.Form): resource = forms.ModelChoiceField( - label=_('Resource'), - queryset=ResourceBase.objects.all(), - widget=forms.HiddenInput()) + label=_("Resource"), queryset=ResourceBase.objects.all(), widget=forms.HiddenInput() + ) reason = forms.CharField( - label=_('Reason'), - widget=forms.Textarea, - help_text=_('Short reasoning behind the request'), - required=True) + label=_("Reason"), widget=forms.Textarea, help_text=_("Short reasoning behind the request"), required=True + ) class Meta: - fields = ['reason', 'resource'] + fields = ["reason", "resource"] class ThesaurusImportForm(forms.Form): diff --git a/geonode/base/middleware.py b/geonode/base/middleware.py index 0c0f918a3cf..875a7ae8ae7 100644 --- a/geonode/base/middleware.py +++ b/geonode/base/middleware.py @@ -30,16 +30,14 @@ class ReadOnlyMiddleware: with an exception for whitelisted url names. """ - FORBIDDEN_HTTP_METHODS = [ - 'POST', 'PUT', 'DELETE' - ] + FORBIDDEN_HTTP_METHODS = ["POST", "PUT", "DELETE"] WHITELISTED_URL_NAMES = [ - 'login', - 'logout', - 'account_login', - 'account_logout', - 'ows_endpoint', + "login", + "logout", + "account_login", + "account_logout", + "ows_endpoint", ] def __init__(self, get_response): @@ -54,7 +52,7 @@ def process_view(self, request, view_func, view_args, view_kwargs): configuration_session_cache(session) # check if the Geonode instance is read-only - if session.get('config').get('configuration').get('read_only'): + if session.get("config").get("configuration").get("read_only"): # allow superadmin users to do whatever they want if not request.user.is_superuser or not request.user.is_active: # check if the request's method is forbidden in read-only instance @@ -62,7 +60,7 @@ def process_view(self, request, view_func, view_args, view_kwargs): # check if the request is not against whitelisted views (check by URL names) if request.resolver_match.url_name not in self.WHITELISTED_URL_NAMES: # return HttpResponse('Error: Instance in read-only mode', status=405) - return render(request, 'base/read_only_violation.html', status=405) + return render(request, "base/read_only_violation.html", status=405) class MaintenanceMiddleware: @@ -76,9 +74,9 @@ class MaintenanceMiddleware: # URL's enabling superuser to login/logout to/from admin panel WHITELISTED_URL_NAMES = [ - 'login', - 'logout', - 'index', + "login", + "logout", + "index", ] def __init__(self, get_response): @@ -93,9 +91,9 @@ def process_view(self, request, view_func, view_args, view_kwargs): configuration_session_cache(session) # check if the Geonode instance is in maintenance mode - if session.get('config').get('configuration').get('maintenance'): + if session.get("config").get("configuration").get("maintenance"): # allow superadmin users to do whatever they want if not request.user.is_superuser: # check if the request is not against whitelisted views (check by URL names) if request.resolver_match.url_name not in self.WHITELISTED_URL_NAMES: - return render(request, 'base/maintenance.html', status=503) + return render(request, "base/maintenance.html", status=503) diff --git a/geonode/base/models.py b/geonode/base/models.py index 31bb13c8439..823e1579520 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -65,27 +65,14 @@ from geonode.singleton import SingletonModel from geonode.groups.conf import settings as groups_settings from geonode.base.bbox_utils import BBOXHelper, polygon_from_bbox -from geonode.utils import ( - bbox_to_wkt, - find_by_attr, - bbox_to_projection, - get_allowed_extensions, - is_monochromatic_image) -from geonode.thumbs.utils import ( - thumb_size, - remove_thumbs, - get_unique_upload_path) +from geonode.utils import bbox_to_wkt, find_by_attr, bbox_to_projection, get_allowed_extensions, is_monochromatic_image +from geonode.thumbs.utils import thumb_size, remove_thumbs, get_unique_upload_path from geonode.groups.models import GroupProfile from geonode.security.utils import get_visible_resources, get_geoapp_subtypes from geonode.security.models import PermissionLevelMixin -from geonode.security.permissions import ( - VIEW_PERMISSIONS, - OWNER_PERMISSIONS -) - -from geonode.notifications_helper import ( - send_notification, - get_notification_recipients) +from geonode.security.permissions import VIEW_PERMISSIONS, OWNER_PERMISSIONS + +from geonode.notifications_helper import send_notification, get_notification_recipients from geonode.people.enumerations import ROLE_VALUES from urllib.parse import urlsplit, urljoin @@ -99,51 +86,43 @@ class ContactRole(models.Model): """ ContactRole is an intermediate model to bind Profiles as Contacts to Resources and apply roles. """ - resource = models.ForeignKey('ResourceBase', blank=False, null=False, on_delete=models.CASCADE) + + resource = models.ForeignKey("ResourceBase", blank=False, null=False, on_delete=models.CASCADE) contact = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) role = models.CharField( - choices=ROLE_VALUES, - max_length=255, - help_text=_( - 'function performed by the responsible ' - 'party')) + choices=ROLE_VALUES, max_length=255, help_text=_("function performed by the responsible " "party") + ) def clean(self): """ Make sure there is only one poc and author per resource """ - if not hasattr(self, 'resource'): + if not hasattr(self, "resource"): # The ModelForm will already raise a Validation error for a missing resource. # Re-raising an empty error here ensures the rest of this method isn't # executed. - raise ValidationError('') + raise ValidationError("") - if (self.role == self.resource.poc) or ( - self.role == self.resource.metadata_author): - contacts = self.resource.contacts.filter( - contactrole__role=self.role) + if (self.role == self.resource.poc) or (self.role == self.resource.metadata_author): + contacts = self.resource.contacts.filter(contactrole__role=self.role) if contacts.count() == 1: # only allow this if we are updating the same contact if self.contact != contacts.get(): - raise ValidationError( - f'There can be only one {self.role} for a given resource') + raise ValidationError(f"There can be only one {self.role} for a given resource") if self.contact is None: # verify that any unbound contact is only associated to one # resource bounds = ContactRole.objects.filter(contact=self.contact).count() if bounds > 1: - raise ValidationError( - 'There can be one and only one resource linked to an unbound contact' % - self.role) + raise ValidationError("There can be one and only one resource linked to an unbound contact" % self.role) elif bounds == 1: # verify that if there was one already, it corresponds to this # instance - if ContactRole.objects.filter( - contact=self.contact).get().id != self.id: + if ContactRole.objects.filter(contact=self.contact).get().id != self.id: raise ValidationError( - 'There can be one and only one resource linked to an unbound contact' % - self.role) + "There can be one and only one resource linked to an unbound contact" % self.role + ) class Meta: unique_together = (("contact", "resource", "role"),) @@ -156,19 +135,19 @@ class TopicCategory(models.Model): See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml """ - identifier = models.CharField(max_length=255, default='location') - description = models.TextField(default='') - gn_description = models.TextField( - 'GeoNode description', default='', null=True) + + identifier = models.CharField(max_length=255, default="location") + description = models.TextField(default="") + gn_description = models.TextField("GeoNode description", default="", null=True) is_choice = models.BooleanField(default=True) - fa_class = models.CharField(max_length=64, default='fa-times') + fa_class = models.CharField(max_length=64, default="fa-times") def __str__(self): return self.gn_description class Meta: ordering = ("identifier",) - verbose_name_plural = 'Metadata Topic Categories' + verbose_name_plural = "Metadata Topic Categories" class SpatialRepresentationType(models.Model): @@ -178,9 +157,10 @@ class SpatialRepresentationType(models.Model): See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml """ + identifier = models.CharField(max_length=255, editable=False) description = models.CharField(max_length=255, editable=False) - gn_description = models.CharField('GeoNode description', max_length=255) + gn_description = models.CharField("GeoNode description", max_length=255) is_choice = models.BooleanField(default=True) def __str__(self): @@ -188,47 +168,22 @@ def __str__(self): class Meta: ordering = ("identifier",) - verbose_name_plural = 'Metadata Spatial Representation Types' + verbose_name_plural = "Metadata Spatial Representation Types" class Region(MPTTModel): code = models.CharField(max_length=50, unique=True) name = models.CharField(max_length=255) - parent = TreeForeignKey( - 'self', - null=True, - blank=True, - on_delete=models.CASCADE, - related_name='children') + parent = TreeForeignKey("self", null=True, blank=True, on_delete=models.CASCADE, related_name="children") # Save bbox values in the database. # This is useful for spatial searches and for generating thumbnail images # and metadata records. - bbox_x0 = models.DecimalField( - max_digits=30, - decimal_places=15, - blank=True, - null=True) - bbox_x1 = models.DecimalField( - max_digits=30, - decimal_places=15, - blank=True, - null=True) - bbox_y0 = models.DecimalField( - max_digits=30, - decimal_places=15, - blank=True, - null=True) - bbox_y1 = models.DecimalField( - max_digits=30, - decimal_places=15, - blank=True, - null=True) - srid = models.CharField( - max_length=30, - blank=False, - null=False, - default='EPSG:4326') + bbox_x0 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_x1 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_y0 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_y1 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + srid = models.CharField(max_length=30, blank=False, null=False, default="EPSG:4326") def __str__(self): return str(self.name) @@ -236,35 +191,24 @@ def __str__(self): @property def bbox(self): """BBOX is in the format: [x0,x1,y0,y1].""" - return [ - self.bbox_x0, - self.bbox_x1, - self.bbox_y0, - self.bbox_y1, - self.srid] + return [self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, self.srid] @property def bbox_string(self): """BBOX is in the format: [x0,y0,x1,y1].""" - return ",".join([str(self.bbox_x0), str(self.bbox_y0), - str(self.bbox_x1), str(self.bbox_y1)]) + return ",".join([str(self.bbox_x0), str(self.bbox_y0), str(self.bbox_x1), str(self.bbox_y1)]) @property def geographic_bounding_box(self): """BBOX is in the format: [x0,x1,y0,y1].""" - return bbox_to_wkt( - self.bbox_x0, - self.bbox_x1, - self.bbox_y0, - self.bbox_y1, - srid=self.srid) + return bbox_to_wkt(self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, srid=self.srid) class Meta: ordering = ("name",) - verbose_name_plural = 'Metadata Regions' + verbose_name_plural = "Metadata Regions" class MPTTMeta: - order_insertion_by = ['name'] + order_insertion_by = ["name"] class RestrictionCodeType(models.Model): @@ -274,9 +218,10 @@ class RestrictionCodeType(models.Model): See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml """ + identifier = models.CharField(max_length=255, editable=False) description = models.TextField(max_length=255, editable=False) - gn_description = models.TextField('GeoNode description', max_length=255) + gn_description = models.TextField("GeoNode description", max_length=255) is_choice = models.BooleanField(default=True) def __str__(self): @@ -284,7 +229,7 @@ def __str__(self): class Meta: ordering = ("identifier",) - verbose_name_plural = 'Metadata Restriction Code Types' + verbose_name_plural = "Metadata Restriction Code Types" class License(models.Model): @@ -318,42 +263,38 @@ def description_bullets(self): class Meta: ordering = ("name",) - verbose_name_plural = 'Licenses' + verbose_name_plural = "Licenses" class HierarchicalKeywordQuerySet(MP_NodeQuerySet): """QuerySet to automatically create a root node if `depth` not given.""" def create(self, **kwargs): - if 'depth' not in kwargs: + if "depth" not in kwargs: return self.model.add_root(**kwargs) return super().create(**kwargs) class HierarchicalKeywordManager(MP_NodeManager): - def get_queryset(self): - return HierarchicalKeywordQuerySet(self.model).order_by('path') + return HierarchicalKeywordQuerySet(self.model).order_by("path") class HierarchicalKeyword(TagBase, MP_Node): - node_order_by = ['name'] + node_order_by = ["name"] objects = HierarchicalKeywordManager() @classmethod def resource_keywords_tree(cls, user, parent=None, resource_type=None, resource_name=None): - """ Returns resource keywords tree as a dict object. """ + """Returns resource keywords tree as a dict object.""" user = user or get_anonymous_user() - resource_types = [resource_type] if resource_type else ['dataset', 'map', 'document'] + get_geoapp_subtypes() + resource_types = [resource_type] if resource_type else ["dataset", "map", "document"] + get_geoapp_subtypes() qset = cls.get_tree(parent) if settings.SKIP_PERMS_FILTER: resources = ResourceBase.objects.all() else: - resources = get_objects_for_user( - user, - 'base.view_resourcebase' - ) + resources = get_objects_for_user(user, "base.view_resourcebase") resources = resources.filter( polymorphic_ctype__model__in=resource_types, @@ -367,32 +308,29 @@ def resource_keywords_tree(cls, user, parent=None, resource_type=None, resource_ user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) tree = {} - for hkw in qset.order_by('name'): + for hkw in qset.order_by("name"): slug = hkw.slug tags_count = 0 - tags_count = TaggedContentItem.objects.filter( - content_object__in=resources, - tag=hkw - ).count() + tags_count = TaggedContentItem.objects.filter(content_object__in=resources, tag=hkw).count() if tags_count > 0: - newobj = {"id": hkw.pk, "text": hkw.name, "href": slug, 'tags': [tags_count]} + newobj = {"id": hkw.pk, "text": hkw.name, "href": slug, "tags": [tags_count]} depth = hkw.depth or 1 # No use case, so purpose of 'parent' param is not clear. # So following first 'if' statement is left unchanged - if (not parent and depth == 1) or \ - (parent and depth == parent.depth): + if (not parent and depth == 1) or (parent and depth == parent.depth): if hkw.pk not in tree: tree[hkw.pk] = newobj tree[hkw.pk]["nodes"] = [] else: - tree[hkw.pk]['tags'] = [tags_count] + tree[hkw.pk]["tags"] = [tags_count] else: tree = cls._keywords_tree_of_a_child(hkw, tree, newobj) @@ -434,20 +372,20 @@ def _keywords_tree_of_a_child(cls, child, tree, newobj): class TaggedContentItem(ItemBase): - content_object = models.ForeignKey('ResourceBase', on_delete=models.CASCADE) - tag = models.ForeignKey('HierarchicalKeyword', related_name='keywords', on_delete=models.CASCADE) + content_object = models.ForeignKey("ResourceBase", on_delete=models.CASCADE) + tag = models.ForeignKey("HierarchicalKeyword", related_name="keywords", on_delete=models.CASCADE) # see https://github.com/alex/django-taggit/issues/101 @classmethod def tags_for(cls, model, instance=None, **extra_filters): kwargs = extra_filters or {} if instance is not None: - return cls.tag_model().objects.filter(**{ - f'{cls.tag_relname()}__content_object': instance - }, **kwargs) - return cls.tag_model().objects.filter(**{ - f'{cls.tag_relname()}__content_object__isnull': False - }, **kwargs).distinct() + return cls.tag_model().objects.filter(**{f"{cls.tag_relname()}__content_object": instance}, **kwargs) + return ( + cls.tag_model() + .objects.filter(**{f"{cls.tag_relname()}__content_object__isnull": False}, **kwargs) + .distinct() + ) class _HierarchicalTagManager(_TaggableManager): @@ -455,23 +393,17 @@ def add(self, *tags, through_defaults=None, tag_kwargs=None): if tag_kwargs is None: tag_kwargs = {} - str_tags = set([ - t - for t in tags - if not isinstance(t, self.through.tag_model()) - ]) + str_tags = set([t for t in tags if not isinstance(t, self.through.tag_model())]) tag_objs = set(tags) - str_tags # If str_tags has 0 elements Django actually optimizes that to not do a # query. Malcolm is very smart. - ''' + """ To avoid concurrency with the keyword in case of a massive upload. With the transaction block and the select_for_update, we can easily handle the concurrency. DOC: https://docs.djangoproject.com/en/3.2/ref/models/querysets/#select-for-update - ''' - existing = self.through.tag_model().objects.select_for_update().filter( - name__in=str_tags, **tag_kwargs - ) + """ + existing = self.through.tag_model().objects.select_for_update().filter(name__in=str_tags, **tag_kwargs) with transaction.atomic(): tag_objs.update(existing) new_ids = set() @@ -496,8 +428,7 @@ def add(self, *tags, through_defaults=None, tag_kwargs=None): for tag in tag_objs: try: - self.through.objects.get_or_create( - tag=tag, **self._lookup_kwargs(), defaults=through_defaults) + self.through.objects.get_or_create(tag=tag, **self._lookup_kwargs(), defaults=through_defaults) except Exception as e: logger.exception(e) @@ -515,26 +446,19 @@ class Thesaurus(models.Model): """ Loadable thesaurus containing keywords in different languages """ - id = models.AutoField( - null=False, - blank=False, - unique=True, - primary_key=True) - identifier = models.CharField( - max_length=255, - null=False, - blank=False, - unique=True) + id = models.AutoField(null=False, blank=False, unique=True, primary_key=True) + + identifier = models.CharField(max_length=255, null=False, blank=False, unique=True) # read from the RDF file title = models.CharField(max_length=255, null=False, blank=False) # read from the RDF file - date = models.CharField(max_length=20, default='') + date = models.CharField(max_length=20, default="") # read from the RDF file - description = models.TextField(max_length=255, default='') + description = models.TextField(max_length=255, default="") - slug = models.CharField(max_length=64, default='') + slug = models.CharField(max_length=64, default="") about = models.CharField(max_length=255, null=True, blank=True) @@ -548,7 +472,7 @@ def __str__(self): class Meta: ordering = ("identifier",) - verbose_name_plural = 'Thesauri' + verbose_name_plural = "Thesauri" class ThesaurusKeywordLabel(models.Model): @@ -562,14 +486,14 @@ class ThesaurusKeywordLabel(models.Model): label = models.CharField(max_length=255) # note = models.CharField(max_length=511) - keyword = models.ForeignKey('ThesaurusKeyword', related_name='keyword', on_delete=models.CASCADE) + keyword = models.ForeignKey("ThesaurusKeyword", related_name="keyword", on_delete=models.CASCADE) def __str__(self): return str(self.label) class Meta: ordering = ("keyword", "lang") - verbose_name_plural = 'Thesaurus Keyword Labels' + verbose_name_plural = "Thesaurus Keyword Labels" unique_together = (("keyword", "lang"),) @@ -577,16 +501,13 @@ class ThesaurusKeyword(models.Model): """ Loadable thesaurus containing keywords in different languages """ + # read from the RDF file about = models.CharField(max_length=255, null=True, blank=True) # read from the RDF file - alt_label = models.CharField( - max_length=255, - default='', - null=True, - blank=True) + alt_label = models.CharField(max_length=255, default="", null=True, blank=True) - thesaurus = models.ForeignKey('Thesaurus', related_name='thesaurus', on_delete=models.CASCADE) + thesaurus = models.ForeignKey("Thesaurus", related_name="thesaurus", on_delete=models.CASCADE) def __str__(self): return str(self.alt_label) @@ -597,7 +518,7 @@ def labels(self): class Meta: ordering = ("alt_label",) - verbose_name_plural = 'Thesaurus Keywords' + verbose_name_plural = "Thesaurus Keywords" unique_together = (("thesaurus", "alt_label"),) @@ -605,9 +526,9 @@ def generate_thesaurus_reference(instance, *args, **kwargs): if instance.about: return instance.about - prefix = instance.thesaurus.about or f'{settings.SITEURL}/thesaurus/{instance.thesaurus.identifier}' + prefix = instance.thesaurus.about or f"{settings.SITEURL}/thesaurus/{instance.thesaurus.identifier}" suffix = instance.alt_label or instance.id - instance.about = f'{prefix}#{suffix}' + instance.about = f"{prefix}#{suffix}" instance.save() return instance.about @@ -620,29 +541,29 @@ class ThesaurusLabel(models.Model): """ Contains localized version of the thesaurus title """ + # read from the RDF file lang = models.CharField(max_length=10) # read from the RDF file label = models.CharField(max_length=255) - thesaurus = models.ForeignKey('Thesaurus', related_name='rel_thesaurus', on_delete=models.CASCADE) + thesaurus = models.ForeignKey("Thesaurus", related_name="rel_thesaurus", on_delete=models.CASCADE) def __str__(self): return str(self.label) class Meta: ordering = ("lang",) - verbose_name_plural = 'Thesaurus Labels' + verbose_name_plural = "Thesaurus Labels" unique_together = (("thesaurus", "lang"),) class ResourceBaseManager(PolymorphicManager): def admin_contact(self): # this assumes there is at least one superuser - superusers = get_user_model().objects.filter(is_superuser=True).order_by('id') + superusers = get_user_model().objects.filter(is_superuser=True).order_by("id") if superusers.count() == 0: - raise RuntimeError( - 'GeoNode needs at least one admin/superuser set') + raise RuntimeError("GeoNode needs at least one admin/superuser set") return superusers[0] @@ -661,10 +582,10 @@ def upload_files(resource_id, files, force=False): if force: out.append(f) elif os.path.isfile(f) and os.path.exists(f): - with open(f, 'rb') as ff: + with open(f, "rb") as ff: folder = os.path.basename(os.path.dirname(f)) filename = os.path.basename(f) - file_uploaded_path = storage_manager.save(f'{folder}/{filename}', ff) + file_uploaded_path = storage_manager.save(f"{folder}/{filename}", ff) out.append(storage_manager.path(file_uploaded_path)) # making an update instead of save in order to avoid others @@ -703,8 +624,9 @@ def cleanup_uploaded_files(resource_id): remove_thumbs(filename) # Remove the uploaded sessions, if any - if 'geonode.upload' in settings.INSTALLED_APPS: + if "geonode.upload" in settings.INSTALLED_APPS: from geonode.upload.models import Upload + # Need to call delete one by one in order to invoke the # 'delete' overridden method for upload in Upload.objects.filter(resource_id=_resource.get_real_instance().id): @@ -715,181 +637,134 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): """ Base Resource Object loosely based on ISO 19115:2003 """ + BASE_PERMISSIONS = { - 'read': ['view_resourcebase'], - 'write': [ - 'change_resourcebase_metadata' + "read": ["view_resourcebase"], + "write": ["change_resourcebase_metadata"], + "download": ["download_resourcebase"], + "owner": [ + "change_resourcebase", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", ], - 'download': ['download_resourcebase'], - 'owner': [ - 'change_resourcebase', - 'delete_resourcebase', - 'change_resourcebase_permissions', - 'publish_resourcebase' - ] } PERMISSIONS = {} - VALID_DATE_TYPES = [(x.lower(), _(x)) - for x in ['Creation', 'Publication', 'Revision']] + VALID_DATE_TYPES = [(x.lower(), _(x)) for x in ["Creation", "Publication", "Revision"]] - abstract_help_text = _( - 'brief narrative summary of the content of the resource(s)') - date_help_text = _('reference date for the cited resource') - date_type_help_text = _('identification of when a given event occurred') - edition_help_text = _('version of the cited resource') + abstract_help_text = _("brief narrative summary of the content of the resource(s)") + date_help_text = _("reference date for the cited resource") + date_type_help_text = _("identification of when a given event occurred") + edition_help_text = _("version of the cited resource") attribution_help_text = _( - 'authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.') - doi_help_text = _( - 'a DOI will be added by Admin before publication.') - purpose_help_text = _( - 'summary of the intentions with which the resource(s) was developed') + "authority or function assigned, as to a ruler, legislative assembly, delegate, or the like." + ) + doi_help_text = _("a DOI will be added by Admin before publication.") + purpose_help_text = _("summary of the intentions with which the resource(s) was developed") maintenance_frequency_help_text = _( - 'frequency with which modifications and deletions are made to the data after ' - 'it is first produced') + "frequency with which modifications and deletions are made to the data after " "it is first produced" + ) keywords_help_text = _( - 'commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject ' - '(space or comma-separated)') + "commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject " + "(space or comma-separated)" + ) tkeywords_help_text = _( - 'formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject ' - '(space or comma-separated)') - regions_help_text = _('keyword identifies a location') - restriction_code_type_help_text = _( - 'limitation(s) placed upon the access or use of the data.') + "formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject " + "(space or comma-separated)" + ) + regions_help_text = _("keyword identifies a location") + restriction_code_type_help_text = _("limitation(s) placed upon the access or use of the data.") constraints_other_help_text = _( - 'other restrictions and legal prerequisites for accessing and using the resource or' - ' metadata') - license_help_text = _('license of the dataset') - language_help_text = _('language used within the dataset') + "other restrictions and legal prerequisites for accessing and using the resource or" " metadata" + ) + license_help_text = _("license of the dataset") + language_help_text = _("language used within the dataset") category_help_text = _( - 'high-level geographic data thematic classification to assist in the grouping and search of ' - 'available geographic data sets.') - spatial_representation_type_help_text = _( - 'method used to represent geographic information in the dataset.') - temporal_extent_start_help_text = _( - 'time period covered by the content of the dataset (start)') - temporal_extent_end_help_text = _( - 'time period covered by the content of the dataset (end)') + "high-level geographic data thematic classification to assist in the grouping and search of " + "available geographic data sets." + ) + spatial_representation_type_help_text = _("method used to represent geographic information in the dataset.") + temporal_extent_start_help_text = _("time period covered by the content of the dataset (start)") + temporal_extent_end_help_text = _("time period covered by the content of the dataset (end)") data_quality_statement_help_text = _( - 'general explanation of the data producer\'s knowledge about the lineage of a' - ' dataset') + "general explanation of the data producer's knowledge about the lineage of a" " dataset" + ) extra_metadata_help_text = _( - 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]') + 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]' + ) # internal fields uuid = models.CharField(max_length=36, unique=True, default=uuid.uuid4) - title = models.CharField(_('title'), max_length=255, help_text=_( - 'name by which the cited resource is known')) - abstract = models.TextField( - _('abstract'), - max_length=2000, - blank=True, - help_text=abstract_help_text) - purpose = models.TextField( - _('purpose'), - max_length=500, - null=True, - blank=True, - help_text=purpose_help_text) + title = models.CharField(_("title"), max_length=255, help_text=_("name by which the cited resource is known")) + abstract = models.TextField(_("abstract"), max_length=2000, blank=True, help_text=abstract_help_text) + purpose = models.TextField(_("purpose"), max_length=500, null=True, blank=True, help_text=purpose_help_text) owner = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name='owned_resource', - verbose_name=_("Owner"), - on_delete=models.PROTECT) - contacts = models.ManyToManyField( - settings.AUTH_USER_MODEL, - through='ContactRole') - alternate = models.CharField( - _('alternate'), - max_length=255, - null=True, - blank=True) - date = models.DateTimeField( - _('date'), - default=now, - help_text=date_help_text) + settings.AUTH_USER_MODEL, related_name="owned_resource", verbose_name=_("Owner"), on_delete=models.PROTECT + ) + contacts = models.ManyToManyField(settings.AUTH_USER_MODEL, through="ContactRole") + alternate = models.CharField(_("alternate"), max_length=255, null=True, blank=True) + date = models.DateTimeField(_("date"), default=now, help_text=date_help_text) date_type = models.CharField( - _('date type'), - max_length=255, - choices=VALID_DATE_TYPES, - default='publication', - help_text=date_type_help_text) - edition = models.CharField( - _('edition'), - max_length=255, - blank=True, - null=True, - help_text=edition_help_text) + _("date type"), max_length=255, choices=VALID_DATE_TYPES, default="publication", help_text=date_type_help_text + ) + edition = models.CharField(_("edition"), max_length=255, blank=True, null=True, help_text=edition_help_text) attribution = models.CharField( - _('Attribution'), - max_length=2048, - blank=True, - null=True, - help_text=attribution_help_text) - doi = models.CharField( - _('DOI'), - max_length=255, - blank=True, - null=True, - help_text=doi_help_text) + _("Attribution"), max_length=2048, blank=True, null=True, help_text=attribution_help_text + ) + doi = models.CharField(_("DOI"), max_length=255, blank=True, null=True, help_text=doi_help_text) maintenance_frequency = models.CharField( - _('maintenance frequency'), + _("maintenance frequency"), max_length=255, choices=enumerations.UPDATE_FREQUENCIES, blank=True, null=True, - help_text=maintenance_frequency_help_text) + help_text=maintenance_frequency_help_text, + ) keywords = TaggableManager( - _('keywords'), + _("keywords"), through=TaggedContentItem, blank=True, help_text=keywords_help_text, - manager=_HierarchicalTagManager) + manager=_HierarchicalTagManager, + ) tkeywords = models.ManyToManyField( - ThesaurusKeyword, - verbose_name=_('keywords'), - null=True, - blank=True, - help_text=tkeywords_help_text) + ThesaurusKeyword, verbose_name=_("keywords"), null=True, blank=True, help_text=tkeywords_help_text + ) regions = models.ManyToManyField( - Region, - verbose_name=_('keywords region'), - null=True, - blank=True, - help_text=regions_help_text) + Region, verbose_name=_("keywords region"), null=True, blank=True, help_text=regions_help_text + ) restriction_code_type = models.ForeignKey( RestrictionCodeType, - verbose_name=_('restrictions'), + verbose_name=_("restrictions"), help_text=restriction_code_type_help_text, null=True, blank=True, on_delete=models.SET_NULL, - limit_choices_to=Q(is_choice=True)) + limit_choices_to=Q(is_choice=True), + ) constraints_other = models.TextField( - _('restrictions other'), - blank=True, - null=True, - help_text=constraints_other_help_text) + _("restrictions other"), blank=True, null=True, help_text=constraints_other_help_text + ) license = models.ForeignKey( License, null=True, blank=True, verbose_name=_("License"), help_text=license_help_text, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL, + ) language = models.CharField( - _('language'), - max_length=3, - choices=enumerations.ALL_LANGUAGES, - default='eng', - help_text=language_help_text) + _("language"), max_length=3, choices=enumerations.ALL_LANGUAGES, default="eng", help_text=language_help_text + ) category = models.ForeignKey( TopicCategory, null=True, blank=True, on_delete=models.SET_NULL, limit_choices_to=Q(is_choice=True), - help_text=category_help_text) + help_text=category_help_text, + ) spatial_representation_type = models.ForeignKey( SpatialRepresentationType, null=True, @@ -897,37 +772,28 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): on_delete=models.SET_NULL, limit_choices_to=Q(is_choice=True), verbose_name=_("spatial representation type"), - help_text=spatial_representation_type_help_text) + help_text=spatial_representation_type_help_text, + ) # Section 5 temporal_extent_start = models.DateTimeField( - _('temporal extent start'), - blank=True, - null=True, - help_text=temporal_extent_start_help_text) + _("temporal extent start"), blank=True, null=True, help_text=temporal_extent_start_help_text + ) temporal_extent_end = models.DateTimeField( - _('temporal extent end'), - blank=True, - null=True, - help_text=temporal_extent_end_help_text) + _("temporal extent end"), blank=True, null=True, help_text=temporal_extent_end_help_text + ) supplemental_information = models.TextField( - _('supplemental information'), + _("supplemental information"), max_length=2000, default=enumerations.DEFAULT_SUPPLEMENTAL_INFORMATION, - help_text=_('any other descriptive information about the dataset')) + help_text=_("any other descriptive information about the dataset"), + ) # Section 8 data_quality_statement = models.TextField( - _('data quality statement'), - max_length=2000, - blank=True, - null=True, - help_text=data_quality_statement_help_text) - group = models.ForeignKey( - Group, - null=True, - blank=True, - on_delete=models.SET_NULL) + _("data quality statement"), max_length=2000, blank=True, null=True, help_text=data_quality_statement_help_text + ) + group = models.ForeignKey(Group, null=True, blank=True, on_delete=models.SET_NULL) # Section 9 # see metadata_author property definition below @@ -938,71 +804,42 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): bbox_polygon = PolygonField(null=True, blank=True) ll_bbox_polygon = PolygonField(null=True, blank=True) - srid = models.CharField( - max_length=30, - blank=False, - null=False, - default='EPSG:4326') + srid = models.CharField(max_length=30, blank=False, null=False, default="EPSG:4326") # CSW specific fields - csw_typename = models.CharField( - _('CSW typename'), - max_length=32, - default='gmd:MD_Metadata', - null=False) + csw_typename = models.CharField(_("CSW typename"), max_length=32, default="gmd:MD_Metadata", null=False) csw_schema = models.CharField( - _('CSW schema'), - max_length=64, - default='http://www.isotc211.org/2005/gmd', - null=False) - csw_mdsource = models.CharField( - _('CSW source'), - max_length=256, - default='local', - null=False) - csw_insert_date = models.DateTimeField( - _('CSW insert date'), auto_now_add=True, null=True) + _("CSW schema"), max_length=64, default="http://www.isotc211.org/2005/gmd", null=False + ) + csw_mdsource = models.CharField(_("CSW source"), max_length=256, default="local", null=False) + csw_insert_date = models.DateTimeField(_("CSW insert date"), auto_now_add=True, null=True) csw_type = models.CharField( - _('CSW type'), - max_length=32, - default='dataset', - null=False, - choices=enumerations.HIERARCHY_LEVELS) - csw_anytext = models.TextField(_('CSW anytext'), null=True, blank=True) + _("CSW type"), max_length=32, default="dataset", null=False, choices=enumerations.HIERARCHY_LEVELS + ) + csw_anytext = models.TextField(_("CSW anytext"), null=True, blank=True) csw_wkt_geometry = models.TextField( - _('CSW WKT geometry'), - null=False, - default='POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))') + _("CSW WKT geometry"), null=False, default="POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))" + ) # metadata XML specific fields metadata_uploaded = models.BooleanField(default=False) - metadata_uploaded_preserve = models.BooleanField(_('Metadata uploaded preserve'), default=False) + metadata_uploaded_preserve = models.BooleanField(_("Metadata uploaded preserve"), default=False) metadata_xml = models.TextField( - null=True, - default='', - blank=True) + null=True, default='', blank=True + ) popular_count = models.IntegerField(default=0) share_count = models.IntegerField(default=0) featured = models.BooleanField( - _("Featured"), - default=False, - help_text=_('Should this resource be advertised in home page?')) - was_published = models.BooleanField( - _("Was Published"), - default=True, - help_text=_('Previous Published state.')) + _("Featured"), default=False, help_text=_("Should this resource be advertised in home page?") + ) + was_published = models.BooleanField(_("Was Published"), default=True, help_text=_("Previous Published state.")) is_published = models.BooleanField( - _("Is Published"), - default=True, - help_text=_('Should this resource be published and searchable?')) - was_approved = models.BooleanField( - _("Was Approved"), - default=True, - help_text=_('Previous Approved state.')) + _("Is Published"), default=True, help_text=_("Should this resource be published and searchable?") + ) + was_approved = models.BooleanField(_("Was Approved"), default=True, help_text=_("Previous Approved state.")) is_approved = models.BooleanField( - _("Approved"), - default=True, - help_text=_('Is this resource validated from a publisher or editor?')) + _("Approved"), default=True, help_text=_("Is this resource validated from a publisher or editor?") + ) # fields necessary for the apis thumbnail_url = models.TextField(_("Thumbnail url"), null=True, blank=True) @@ -1018,7 +855,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): blank=False, default=enumerations.STATE_READY, choices=enumerations.PROCESSING_STATES, - help_text=_('Hold the resource processing state.')) + help_text=_("Hold the resource processing state."), + ) sourcetype = models.CharField( _("Source Type"), @@ -1027,43 +865,31 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): blank=False, default=enumerations.SOURCE_TYPE_LOCAL, choices=enumerations.SOURCE_TYPES, - help_text=_('The resource source type, which can be one of "LOCAL", "REMOTE" or "COPYREMOTE".')) + help_text=_('The resource source type, which can be one of "LOCAL", "REMOTE" or "COPYREMOTE".'), + ) remote_typename = models.CharField( - _('Remote Service Typename'), + _("Remote Service Typename"), null=True, blank=True, max_length=512, - help_text=_('Name of the Remote Service if any.')) + help_text=_("Name of the Remote Service if any."), + ) # fields controlling security state dirty_state = models.BooleanField( - _("Dirty State"), - default=False, - help_text=_('Security Rules Are Not Synched with GeoServer!')) + _("Dirty State"), default=False, help_text=_("Security Rules Are Not Synched with GeoServer!") + ) - users_geolimits = models.ManyToManyField( - "UserGeoLimit", - related_name="users_geolimits", - null=True, - blank=True) + users_geolimits = models.ManyToManyField("UserGeoLimit", related_name="users_geolimits", null=True, blank=True) - groups_geolimits = models.ManyToManyField( - "GroupGeoLimit", - related_name="groups_geolimits", - null=True, - blank=True) + groups_geolimits = models.ManyToManyField("GroupGeoLimit", related_name="groups_geolimits", null=True, blank=True) - resource_type = models.CharField( - _('Resource Type'), - max_length=1024, - blank=True, - null=True) + resource_type = models.CharField(_("Resource Type"), max_length=1024, blank=True, null=True) metadata_only = models.BooleanField( - _("Metadata"), - default=False, - help_text=_('If true, will be excluded from search')) + _("Metadata"), default=False, help_text=_("If true, will be excluded from search") + ) files = JSONField(null=True, default=list, blank=True) @@ -1072,11 +898,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): subtype = models.CharField(max_length=128, null=True, blank=True) metadata = models.ManyToManyField( - "ExtraMetadata", - verbose_name=_('Extra Metadata'), - null=True, - blank=True, - help_text=extra_metadata_help_text) + "ExtraMetadata", verbose_name=_("Extra Metadata"), null=True, blank=True, help_text=extra_metadata_help_text + ) objects = ResourceBaseManager() @@ -1085,19 +908,19 @@ class Meta: # add, change and delete are standard in django-guardian permissions = ( # ('view_resourcebase', 'Can view resource'), - ('change_resourcebase_permissions', 'Can change resource permissions'), - ('download_resourcebase', 'Can download resource'), - ('publish_resourcebase', 'Can publish resource'), - ('change_resourcebase_metadata', 'Can change resource metadata'), + ("change_resourcebase_permissions", "Can change resource permissions"), + ("download_resourcebase", "Can download resource"), + ("publish_resourcebase", "Can publish resource"), + ("change_resourcebase_metadata", "Can change resource metadata"), ) def __init__(self, *args, **kwargs): # Provide legacy support for bbox fields try: - bbox = [kwargs.pop(key, None) for key in ('bbox_x0', 'bbox_y0', 'bbox_x1', 'bbox_y1')] + bbox = [kwargs.pop(key, None) for key in ("bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1")] if all(bbox): - kwargs['bbox_polygon'] = Polygon.from_bbox(bbox) - kwargs['ll_bbox_polygon'] = Polygon.from_bbox(bbox) + kwargs["bbox_polygon"] = Polygon.from_bbox(bbox) + kwargs["ll_bbox_polygon"] = Polygon.from_bbox(bbox) except Exception as e: logger.exception(e) super().__init__(*args, **kwargs) @@ -1108,13 +931,13 @@ def __str__(self): def _remove_html_tags(self, attribute_str): _attribute_str = attribute_str try: - pattern = re.compile('<.*?>') + pattern = re.compile("<.*?>") _attribute_str = html.unescape( - re.sub(pattern, '', attribute_str).replace('\n', ' ').replace('\r', '').strip()) + re.sub(pattern, "", attribute_str).replace("\n", " ").replace("\r", "").strip() + ) except Exception: if attribute_str: - _attribute_str = html.unescape( - attribute_str.replace('\n', ' ').replace('\r', '').strip()) + _attribute_str = html.unescape(attribute_str.replace("\n", " ").replace("\r", "").strip()) return strip_tags(_attribute_str) @classproperty @@ -1122,7 +945,7 @@ def allowed_permissions(cls): return { "anonymous": VIEW_PERMISSIONS, "default": OWNER_PERMISSIONS, - groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS + groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS, } @classproperty @@ -1133,7 +956,7 @@ def compact_permission_labels(cls): "download": _("Download"), "edit": _("Edit"), "manage": _("Manage"), - "owner": _("Owner") + "owner": _("Owner"), } @property @@ -1169,8 +992,7 @@ def save(self, notify=False, *args, **kwargs): """ Send a notification when a resource is created or updated """ - if not self.resource_type and self.polymorphic_ctype and \ - self.polymorphic_ctype.model: + if not self.resource_type and self.polymorphic_ctype and self.polymorphic_ctype.model: self.resource_type = self.polymorphic_ctype.model.lower() # Resource Updated @@ -1178,14 +1000,14 @@ def save(self, notify=False, *args, **kwargs): _group_status_changed = False _approval_status_changed = False - if hasattr(self, 'class_name') and (self.pk is None or notify): - if self.pk is None and (self.title or getattr(self, 'name', None)): + if hasattr(self, "class_name") and (self.pk is None or notify): + if self.pk is None and (self.title or getattr(self, "name", None)): # Resource Created - if not self.title and getattr(self, 'name', None): - self.title = getattr(self, 'name', None) - notice_type_label = f'{self.class_name.lower()}_created' + if not self.title and getattr(self, "name", None): + self.title = getattr(self, "name", None) + notice_type_label = f"{self.class_name.lower()}_created" recipients = get_notification_recipients(notice_type_label, resource=self) - send_notification(recipients, notice_type_label, {'resource': self}) + send_notification(recipients, notice_type_label, {"resource": self}) elif self.pk: # Group has changed _group_status_changed = self.group != ResourceBase.objects.get(pk=self.get_self_resource().pk).group @@ -1194,9 +1016,9 @@ def save(self, notify=False, *args, **kwargs): if self.was_approved != self.is_approved: if not _notification_sent and not self.was_approved and self.is_approved: # Send "approved" notification - notice_type_label = f'{self.class_name.lower()}_approved' + notice_type_label = f"{self.class_name.lower()}_approved" recipients = get_notification_recipients(notice_type_label, resource=self) - send_notification(recipients, notice_type_label, {'resource': self}) + send_notification(recipients, notice_type_label, {"resource": self}) _notification_sent = True self.was_approved = self.is_approved _approval_status_changed = True @@ -1205,30 +1027,28 @@ def save(self, notify=False, *args, **kwargs): if self.was_published != self.is_published: if not _notification_sent and not self.was_published and self.is_published: # Send "published" notification - notice_type_label = f'{self.class_name.lower()}_published' + notice_type_label = f"{self.class_name.lower()}_published" recipients = get_notification_recipients(notice_type_label, resource=self) - send_notification(recipients, notice_type_label, {'resource': self}) + send_notification(recipients, notice_type_label, {"resource": self}) _notification_sent = True self.was_published = self.is_published _approval_status_changed = True # Updated Notifications Here if not _notification_sent: - notice_type_label = f'{self.class_name.lower()}_updated' + notice_type_label = f"{self.class_name.lower()}_updated" recipients = get_notification_recipients(notice_type_label, resource=self) - send_notification(recipients, notice_type_label, {'resource': self}) + send_notification(recipients, notice_type_label, {"resource": self}) if self.pk is None: - _initial_value = ResourceBase.objects.aggregate(Max("pk"))['pk__max'] + _initial_value = ResourceBase.objects.aggregate(Max("pk"))["pk__max"] if not _initial_value: _initial_value = 1 else: _initial_value += 1 - _next_value = get_next_value( - "ResourceBase", # type(self).__name__, - initial_value=_initial_value) + _next_value = get_next_value("ResourceBase", initial_value=_initial_value) # type(self).__name__, if _initial_value > _next_value: - Sequence.objects.filter(name='ResourceBase').update(last=_initial_value) + Sequence.objects.filter(name="ResourceBase").update(last=_initial_value) _next_value = _initial_value self.pk = self.id = _next_value @@ -1241,19 +1061,22 @@ def save(self, notify=False, *args, **kwargs): # Update workflow permissions if _approval_status_changed or _group_status_changed: - self.set_permissions(approval_status_changed=_approval_status_changed, group_status_changed=_group_status_changed) + self.set_permissions( + approval_status_changed=_approval_status_changed, group_status_changed=_group_status_changed + ) def delete(self, notify=True, *args, **kwargs): """ Send a notification when a layer, map or document is deleted """ from geonode.resource.manager import resource_manager + resource_manager.remove_permissions(self.uuid, instance=self.get_real_instance()) - if hasattr(self, 'class_name') and notify: - notice_type_label = f'{self.class_name.lower()}_deleted' + if hasattr(self, "class_name") and notify: + notice_type_label = f"{self.class_name.lower()}_deleted" recipients = get_notification_recipients(notice_type_label, resource=self) - send_notification(recipients, notice_type_label, {'resource': self}) + send_notification(recipients, notice_type_label, {"resource": self}) super().delete(*args, **kwargs) @@ -1294,7 +1117,7 @@ def topiccategory(self): @property def csw_crs(self): - return 'EPSG:4326' + return "EPSG:4326" @property def group_name(self): @@ -1304,8 +1127,8 @@ def group_name(self): def bbox(self): """BBOX is in the format: [x0, x1, y0, y1, srid].""" if self.bbox_polygon: - match = re.match(r'^(EPSG:)?(?P\d{4,6})$', self.srid) - srid = int(match.group('srid')) if match else 4326 + match = re.match(r"^(EPSG:)?(?P\d{4,6})$", self.srid) + srid = int(match.group("srid")) if match else 4326 bbox = BBOXHelper(self.bbox_polygon.extent) return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, f"EPSG:{srid}"] bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) @@ -1384,18 +1207,13 @@ def geographic_bounding_box(self): return str(bbox) else: bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) - return bbox_to_wkt( - bbox.xmin, - bbox.xmax, - bbox.ymin, - bbox.ymax, - srid='EPSG:4326') + return bbox_to_wkt(bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, srid="EPSG:4326") @property def license_light(self): a = [] if not self.license: - return '' + return "" if self.license.name is not None and (len(self.license.name) > 0): a.append(self.license.name) if self.license.url is not None and (len(self.license.url) > 0): @@ -1405,11 +1223,9 @@ def license_light(self): @property def license_verbose(self): a = [] - if self.license.name_long is not None and ( - len(self.license.name_long) > 0): + if self.license.name_long is not None and (len(self.license.name_long) > 0): a.append(f"{self.license.name_long}:") - if self.license.description is not None and ( - len(self.license.description) > 0): + if self.license.description is not None and (len(self.license.description) > 0): a.append(self.license.description) if self.license.url is not None and (len(self.license.url) > 0): a.append(f"({self.license.url})") @@ -1418,32 +1234,33 @@ def license_verbose(self): @property def metadata_completeness(self): required_fields = [ - 'abstract', - 'category', - 'data_quality_statement', - 'date', - 'date_type', - 'language', - 'license', - 'regions', - 'title'] - if self.restriction_code_type == 'otherRestrictions': - required_fields.append('constraints_other') + "abstract", + "category", + "data_quality_statement", + "date", + "date_type", + "language", + "license", + "regions", + "title", + ] + if self.restriction_code_type == "otherRestrictions": + required_fields.append("constraints_other") filled_fields = [] for required_field in required_fields: field = getattr(self, required_field, None) if field: - if required_field == 'license': - if field.name == 'Not Specified': + if required_field == "license": + if field.name == "Not Specified": continue - if required_field == 'regions': + if required_field == "regions": if not field.all(): continue - if required_field == 'category': + if required_field == "category": if not field.identifier: continue filled_fields.append(field) - return f'{len(filled_fields) * 100 / len(required_fields)}%' + return f"{len(filled_fields) * 100 / len(required_fields)}%" @property def instance_is_processed(self): @@ -1457,7 +1274,8 @@ def instance_is_processed(self): @property def is_copyable(self): from geonode.geoserver.helpers import select_relevant_files - if self.resource_type == 'dataset': + + if self.resource_type == "dataset": allowed_file = select_relevant_files(get_allowed_extensions(), self.files) return len(allowed_file) != 0 return True @@ -1472,13 +1290,13 @@ def region_name_list(self): return [region.name for region in self.regions.all()] def spatial_representation_type_string(self): - if hasattr(self.spatial_representation_type, 'identifier'): + if hasattr(self.spatial_representation_type, "identifier"): return self.spatial_representation_type.identifier else: - if hasattr(self, 'subtype'): - if self.subtype == 'raster': - return 'grid' - return 'vector' + if hasattr(self, "subtype"): + if self.subtype == "raster": + return "grid" + return "vector" else: return None @@ -1509,11 +1327,11 @@ def keyword_csv(self): try: keywords_qs = self.get_real_instance().keywords.all() if keywords_qs: - return ','.join(kw.name for kw in keywords_qs) + return ",".join(kw.name for kw in keywords_qs) else: - return '' + return "" except Exception: - return '' + return "" def get_absolute_url(self): try: @@ -1536,8 +1354,7 @@ def set_bbox_polygon(self, bbox, srid): self.srid = srid # This is a trick in order to avoid PostGIS reprojecting the bbox at save time # by assuming the default geometries have 'EPSG:4326' as srid. - ResourceBase.objects.filter(id=self.id).update( - bbox_polygon=self.bbox_polygon, srid=srid) + ResourceBase.objects.filter(id=self.id).update(bbox_polygon=self.bbox_polygon, srid=srid) finally: self.set_ll_bbox_polygon(bbox, srid=srid) @@ -1554,12 +1371,10 @@ def set_ll_bbox_polygon(self, bbox, srid="EPSG:4326"): if srid == 4326 or srid.upper() == "EPSG:4326": self.ll_bbox_polygon = bbox_polygon else: - match = re.match(r'^(EPSG:)?(?P\d{4,6})$', str(srid)) - bbox_polygon.srid = int(match.group('srid')) if match else 4326 - self.ll_bbox_polygon = Polygon.from_bbox( - bbox_to_projection(list(bbox_polygon.extent) + [srid])[:-1]) - ResourceBase.objects.filter(id=self.id).update( - ll_bbox_polygon=self.ll_bbox_polygon) + match = re.match(r"^(EPSG:)?(?P\d{4,6})$", str(srid)) + bbox_polygon.srid = int(match.group("srid")) if match else 4326 + self.ll_bbox_polygon = Polygon.from_bbox(bbox_to_projection(list(bbox_polygon.extent) + [srid])[:-1]) + ResourceBase.objects.filter(id=self.id).update(ll_bbox_polygon=self.ll_bbox_polygon) except Exception as e: raise GeoNodeException(e) @@ -1583,15 +1398,12 @@ def set_bounds_from_bbox(self, bbox, srid): return if not bbox or len(bbox) < 4: - raise ValidationError( - f'Bounding Box cannot be empty {self.name} for a given resource') + raise ValidationError(f"Bounding Box cannot be empty {self.name} for a given resource") if not srid: - raise ValidationError( - f'Projection cannot be empty {self.name} for a given resource') + raise ValidationError(f"Projection cannot be empty {self.name} for a given resource") self.srid = srid - self.set_bbox_polygon( - (bbox[0], bbox[2], bbox[1], bbox[3]), srid) + self.set_bbox_polygon((bbox[0], bbox[2], bbox[1], bbox[3]), srid) self.set_center_zoom() def set_center_zoom(self): @@ -1615,32 +1427,24 @@ def download_links(self): """assemble download links for pycsw""" links = [] for link in self.link_set.all(): - if link.link_type == 'metadata': # avoid recursion + if link.link_type == "metadata": # avoid recursion continue - if link.link_type == 'html': - links.append( - (self.title, - 'Web address (URL)', - 'WWW:LINK-1.0-http--link', - link.url)) - elif link.link_type in ('OGC:WMS', 'OGC:WFS', 'OGC:WCS'): + if link.link_type == "html": + links.append((self.title, "Web address (URL)", "WWW:LINK-1.0-http--link", link.url)) + elif link.link_type in ("OGC:WMS", "OGC:WFS", "OGC:WCS"): links.append((self.title, link.name, link.link_type, link.url)) else: - _link_type = 'WWW:DOWNLOAD-1.0-http--download' + _link_type = "WWW:DOWNLOAD-1.0-http--download" try: - _store_type = getattr(self.get_real_instance(), 'subtype', None) - if _store_type and _store_type in ['tileStore', 'remote'] and link.extension in ('html'): - _remote_service = getattr(self.get_real_instance(), '_remote_service', None) + _store_type = getattr(self.get_real_instance(), "subtype", None) + if _store_type and _store_type in ["tileStore", "remote"] and link.extension in ("html"): + _remote_service = getattr(self.get_real_instance(), "_remote_service", None) if _remote_service: - _link_type = f'WWW:DOWNLOAD-{_remote_service.type}' + _link_type = f"WWW:DOWNLOAD-{_remote_service.type}" except Exception as e: logger.exception(e) - description = f'{self.title} ({link.name} Format)' - links.append( - (self.title, - description, - _link_type, - link.url)) + description = f"{self.title} ({link.name} Format)" + links.append((self.title, description, _link_type, link.url)) return links @property @@ -1648,20 +1452,18 @@ def embed_url(self): return self.get_real_instance().embed_url def get_tiles_url(self): - """Return URL for Z/Y/X mapping clients or None if it does not exist. - """ + """Return URL for Z/Y/X mapping clients or None if it does not exist.""" try: - tiles_link = self.link_set.get(name='Tiles') + tiles_link = self.link_set.get(name="Tiles") except Link.DoesNotExist: return None else: return tiles_link.url def get_legend(self): - """Return Link for legend or None if it does not exist. - """ + """Return Link for legend or None if it does not exist.""" try: - legends_link = self.link_set.filter(name='Legend') + legends_link = self.link_set.filter(name="Legend") except Link.DoesNotExist: tb = traceback.format_exc() logger.debug(tb) @@ -1676,8 +1478,8 @@ def get_legend(self): def get_legend_url(self, style_name=None): """Return URL for legend or None if it does not exist. - The legend can be either an image (for Geoserver's WMS) - or a JSON object for ArcGIS. + The legend can be either an image (for Geoserver's WMS) + or a JSON object for ArcGIS. """ legend = self.get_legend() @@ -1694,10 +1496,9 @@ def get_legend_url(self, style_name=None): return None def get_ows_url(self): - """Return URL for OGC WMS server None if it does not exist. - """ + """Return URL for OGC WMS server None if it does not exist.""" try: - ows_link = self.link_set.get(name='OGC:WMS') + ows_link = self.link_set.get(name="OGC:WMS") except Link.DoesNotExist: return None else: @@ -1706,11 +1507,11 @@ def get_ows_url(self): def get_thumbnail_url(self): """Return a thumbnail url. - It could be a local one if it exists, a remote one (WMS GetImage) for example + It could be a local one if it exists, a remote one (WMS GetImage) for example """ _thumbnail_url = self.thumbnail_url - local_thumbnails = self.link_set.filter(name='Thumbnail') - remote_thumbnails = self.link_set.filter(name='Remote Thumbnail') + local_thumbnails = self.link_set.filter(name="Thumbnail") + remote_thumbnails = self.link_set.filter(name="Remote Thumbnail") if local_thumbnails.exists(): _thumbnail_url = local_thumbnails.first().url elif remote_thumbnails.exists(): @@ -1719,14 +1520,14 @@ def get_thumbnail_url(self): def has_thumbnail(self): """Determine if the thumbnail object exists and an image exists""" - return self.link_set.filter(name='Thumbnail').exists() + return self.link_set.filter(name="Thumbnail").exists() # Note - you should probably broadcast layer#post_save() events to ensure # that indexing (or other listeners) are notified def save_thumbnail(self, filename, image, **kwargs): upload_path = get_unique_upload_path(filename) # force convertion to JPEG output file - upload_path = f'{os.path.splitext(upload_path)[0]}.jpg' + upload_path = f"{os.path.splitext(upload_path)[0]}.jpg" try: # Check that the image is valid if is_monochromatic_image(None, image): @@ -1748,13 +1549,15 @@ def save_thumbnail(self, filename, image, **kwargs): _default_thumb_size = settings.THUMBNAIL_SIZE im = Image.open(storage_manager.open(actual_name)) centering = kwargs.get("centering", (0.5, 0.5)) - cover = ImageOps.fit(im, (_default_thumb_size['width'], _default_thumb_size['height']), centering=centering).convert("RGB") + cover = ImageOps.fit( + im, (_default_thumb_size["width"], _default_thumb_size["height"]), centering=centering + ).convert("RGB") # Saving the thumb into a temporary directory on file system tmp_location = os.path.abspath(f"{settings.MEDIA_ROOT}/{upload_path}") cover.save(tmp_location, quality="high") - with open(tmp_location, 'rb+') as img: + with open(tmp_location, "rb+") as img: # Saving the img via storage manager storage_manager.save(storage_manager.path(upload_path), img) @@ -1768,23 +1571,23 @@ def save_thumbnail(self, filename, image, **kwargs): parsed = urlsplit(url) if not parsed.netloc: # assuming is a relative path to current site - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL url = urljoin(site_url, url) if thumb_size(upload_path) == 0: raise Exception("Generated thumbnail image is zero size") # should only have one 'Thumbnail' link - Link.objects.filter(resource=self, name='Thumbnail').delete() + Link.objects.filter(resource=self, name="Thumbnail").delete() obj, _created = Link.objects.get_or_create( resource=self, - name='Thumbnail', + name="Thumbnail", defaults=dict( url=url, - extension='png', - mime='image/png', - link_type='image', - ) + extension="png", + mime="image/png", + link_type="image", + ), ) # Cleaning up the old stuff if self.thumbnail_path and storage_manager.exists(self.thumbnail_path): @@ -1794,26 +1597,19 @@ def save_thumbnail(self, filename, image, **kwargs): self.thumbnail_path = upload_path obj.url = url obj.save() - ResourceBase.objects.filter(id=self.id).update( - thumbnail_url=url, - thumbnail_path=upload_path - ) + ResourceBase.objects.filter(id=self.id).update(thumbnail_url=url, thumbnail_path=upload_path) except Exception as e: - logger.error( - f'Error when generating the thumbnail for resource {self.id}. ({e})' - ) + logger.error(f"Error when generating the thumbnail for resource {self.id}. ({e})") try: - Link.objects.filter(resource=self, name='Thumbnail').delete() + Link.objects.filter(resource=self, name="Thumbnail").delete() except Exception as e: - logger.error( - f'Error when generating the thumbnail for resource {self.id}. ({e})' - ) + logger.error(f"Error when generating the thumbnail for resource {self.id}. ({e})") def set_missing_info(self): """Set default permissions and point of contacts. - It is mandatory to call it from descendant classes - but hard to enforce technically via signals or save overriding. + It is mandatory to call it from descendant classes + but hard to enforce technically via signals or save overriding. """ user = None if self.owner: @@ -1831,16 +1627,15 @@ def set_missing_info(self): self.metadata_author = user from guardian.models import UserObjectPermission - logger.debug('Checking for permissions.') + + logger.debug("Checking for permissions.") # True if every key in the get_all_level_info dict is empty. no_custom_permissions = UserObjectPermission.objects.filter( - content_type=ContentType.objects.get_for_model( - self.get_self_resource()), object_pk=str( - self.pk)).exists() + content_type=ContentType.objects.get_for_model(self.get_self_resource()), object_pk=str(self.pk) + ).exists() if not no_custom_permissions: - logger.debug( - 'There are no permissions for this object, setting default perms.') + logger.debug("There are no permissions for this object, setting default perms.") self.set_default_permissions(owner=user) def maintenance_frequency_title(self): @@ -1851,19 +1646,13 @@ def language_title(self): def _set_poc(self, poc): # reset any poc assignation to this resource - ContactRole.objects.filter( - role='pointOfContact', - resource=self).delete() + ContactRole.objects.filter(role="pointOfContact", resource=self).delete() # create the new assignation - ContactRole.objects.create( - role='pointOfContact', - resource=self, - contact=poc) + ContactRole.objects.create(role="pointOfContact", resource=self, contact=poc) def _get_poc(self): try: - the_poc = ContactRole.objects.get( - role='pointOfContact', resource=self).contact + the_poc = ContactRole.objects.get(role="pointOfContact", resource=self).contact except ContactRole.DoesNotExist: the_poc = None return the_poc @@ -1872,17 +1661,13 @@ def _get_poc(self): def _set_metadata_author(self, metadata_author): # reset any metadata_author assignation to this resource - ContactRole.objects.filter(role='author', resource=self).delete() + ContactRole.objects.filter(role="author", resource=self).delete() # create the new assignation - ContactRole.objects.create( - role='author', - resource=self, - contact=metadata_author) + ContactRole.objects.create(role="author", resource=self, contact=metadata_author) def _get_metadata_author(self): try: - the_ma = ContactRole.objects.get( - role='author', resource=self).contact + the_ma = ContactRole.objects.get(role="author", resource=self).contact except ContactRole.DoesNotExist: the_ma = None return the_ma @@ -1900,59 +1685,48 @@ def add_missing_metadata_author_or_poc(self): class LinkManager(models.Manager): - """Helper class to access links grouped by type - """ + """Helper class to access links grouped by type""" def data(self): - return self.get_queryset().filter(link_type='data') + return self.get_queryset().filter(link_type="data") def image(self): - return self.get_queryset().filter(link_type='image') + return self.get_queryset().filter(link_type="image") def download(self): - return self.get_queryset().filter(link_type__in=['image', 'data', 'original']) + return self.get_queryset().filter(link_type__in=["image", "data", "original"]) def metadata(self): - return self.get_queryset().filter(link_type='metadata') + return self.get_queryset().filter(link_type="metadata") def original(self): - return self.get_queryset().filter(link_type='original') + return self.get_queryset().filter(link_type="original") def ows(self): - return self.get_queryset().filter( - link_type__in=['OGC:WMS', 'OGC:WFS', 'OGC:WCS']) + return self.get_queryset().filter(link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS"]) class Link(models.Model): """Auxiliary model for storing links for resources. - This helps avoiding the need for runtime lookups - to the OWS server or the CSW Catalogue. - - There are four types of links: - * original: For uploaded files (Shapefiles or GeoTIFFs) - * data: For WFS and WCS links that allow access to raw data - * image: For WMS and TMS links - * metadata: For CSW links - * OGC:WMS: for WMS service links - * OGC:WFS: for WFS service links - * OGC:WCS: for WCS service links + This helps avoiding the need for runtime lookups + to the OWS server or the CSW Catalogue. + + There are four types of links: + * original: For uploaded files (Shapefiles or GeoTIFFs) + * data: For WFS and WCS links that allow access to raw data + * image: For WMS and TMS links + * metadata: For CSW links + * OGC:WMS: for WMS service links + * OGC:WFS: for WFS service links + * OGC:WCS: for WCS service links """ - resource = models.ForeignKey( - ResourceBase, - blank=True, - null=True, - on_delete=models.CASCADE) - extension = models.CharField( - max_length=255, - help_text=_('For example "kml"')) - link_type = models.CharField( - max_length=255, choices=[ - (x, x) for x in enumerations.LINK_TYPES]) - name = models.CharField(max_length=255, help_text=_( - 'For example "View in Google Earth"')) - mime = models.CharField(max_length=255, - help_text=_('For example "text/xml"')) + + resource = models.ForeignKey(ResourceBase, blank=True, null=True, on_delete=models.CASCADE) + extension = models.CharField(max_length=255, help_text=_('For example "kml"')) + link_type = models.CharField(max_length=255, choices=[(x, x) for x in enumerations.LINK_TYPES]) + name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"')) + mime = models.CharField(max_length=255, help_text=_('For example "text/xml"')) url = models.TextField(max_length=1000) objects = LinkManager() @@ -1962,28 +1736,15 @@ def __str__(self): class MenuPlaceholder(models.Model): - name = models.CharField( - max_length=255, - null=False, - blank=False, - unique=True - ) + name = models.CharField(max_length=255, null=False, blank=False, unique=True) def __str__(self): return str(self.name) class Menu(models.Model): - title = models.CharField( - max_length=255, - null=False, - blank=False - ) - placeholder = models.ForeignKey( - to='MenuPlaceholder', - on_delete=models.CASCADE, - null=False - ) + title = models.CharField(max_length=255, null=False, blank=False) + placeholder = models.ForeignKey(to="MenuPlaceholder", on_delete=models.CASCADE, null=False) order = models.IntegerField( null=False, ) @@ -1993,32 +1754,18 @@ def __str__(self): class Meta: unique_together = ( - ('placeholder', 'order'), - ('placeholder', 'title'), + ("placeholder", "order"), + ("placeholder", "title"), ) - ordering = ['order'] + ordering = ["order"] class MenuItem(models.Model): - title = models.CharField( - max_length=255, - null=False, - blank=False - ) - menu = models.ForeignKey( - to='Menu', - null=False, - on_delete=models.CASCADE - ) - order = models.IntegerField( - null=False - ) + title = models.CharField(max_length=255, null=False, blank=False) + menu = models.ForeignKey(to="Menu", null=False, on_delete=models.CASCADE) + order = models.IntegerField(null=False) blank_target = models.BooleanField() - url = models.CharField( - max_length=2000, - null=False, - blank=False - ) + url = models.CharField(max_length=2000, null=False, blank=False) def __eq__(self, other): return self.order == other.order @@ -2046,10 +1793,10 @@ def __str__(self): class Meta: unique_together = ( - ('menu', 'order'), - ('menu', 'title'), + ("menu", "order"), + ("menu", "title"), ) - ordering = ['order'] + ordering = ["order"] class Configuration(SingletonModel): @@ -2061,64 +1808,39 @@ class Configuration(SingletonModel): from geonode.base.models import Configuration config = Configuration.load() """ + read_only = models.BooleanField(default=False) maintenance = models.BooleanField(default=False) class Meta: - verbose_name_plural = 'Configuration' + verbose_name_plural = "Configuration" def __str__(self): - return 'Configuration' + return "Configuration" class UserGeoLimit(models.Model): - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - null=False, - blank=False, - on_delete=models.CASCADE) - resource = models.ForeignKey( - ResourceBase, - null=False, - blank=False, - on_delete=models.CASCADE) - wkt = models.TextField( - db_column='wkt', - blank=True) + user = models.ForeignKey(settings.AUTH_USER_MODEL, null=False, blank=False, on_delete=models.CASCADE) + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) + wkt = models.TextField(db_column="wkt", blank=True) class GroupGeoLimit(models.Model): - group = models.ForeignKey( - GroupProfile, - null=False, - blank=False, - on_delete=models.CASCADE) - resource = models.ForeignKey( - ResourceBase, - null=False, - blank=False, - on_delete=models.CASCADE) - wkt = models.TextField( - db_column='wkt', - blank=True) + group = models.ForeignKey(GroupProfile, null=False, blank=False, on_delete=models.CASCADE) + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) + wkt = models.TextField(db_column="wkt", blank=True) def rating_post_save(instance, *args, **kwargs): """ Used to fill the average rating field on OverallRating change. """ - ResourceBase.objects.filter( - id=instance.object_id).update( - rating=instance.rating) + ResourceBase.objects.filter(id=instance.object_id).update(rating=instance.rating) signals.post_save.connect(rating_post_save, sender=OverallRating) class ExtraMetadata(models.Model): - resource = models.ForeignKey( - ResourceBase, - null=False, - blank=False, - on_delete=models.CASCADE) + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) metadata = JSONField(null=True, default=dict, blank=True) diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index 34771fc0a4c..3b0be582bf0 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -50,15 +50,14 @@ logger = logging.getLogger(__name__) imgfile = BytesIO( - b'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' - b'\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;' + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" ) -f = SimpleUploadedFile('test_img_file.gif', imgfile.read(), 'image/gif') +f = SimpleUploadedFile("test_img_file.gif", imgfile.read(), "image/gif") dfile = [f"{settings.MEDIA_ROOT}/img.gif"] def all_public(): - '''ensure all layers, maps and documents are publicly available''' + """ensure all layers, maps and documents are publicly available""" for lyr in Dataset.objects.all(): lyr.set_default_permissions() lyr.clear_dirty_state() @@ -75,38 +74,38 @@ def all_public(): def create_fixtures(): - biota = TopicCategory.objects.get(identifier='biota') - location = TopicCategory.objects.get(identifier='location') - elevation = TopicCategory.objects.get(identifier='elevation') - farming = TopicCategory.objects.get(identifier='farming') + biota = TopicCategory.objects.get(identifier="biota") + location = TopicCategory.objects.get(identifier="location") + elevation = TopicCategory.objects.get(identifier="elevation") + farming = TopicCategory.objects.get(identifier="farming") world_extent = [-180, 180, -90, 90] map_data = [ - ('GeoNode Default Map', 'GeoNode default map abstract', ('populartag',), world_extent, biota), - ('ipsum lorem', 'common ipsum lorem', ('populartag', 'maptagunique'), world_extent, biota), - ('lorem1 ipsum1', 'common abstract1', ('populartag',), world_extent, biota), - ('ipsum foo', 'common bar lorem', ('populartag',), world_extent, location), - ('map one', 'common this is a unique thing', ('populartag',), [0, 1, 0, 1], location), - ('quux', 'common double thing', ('populartag',), [0, 5, 0, 5], location), - ('morx', 'common thing double', ('populartag',), [0, 10, 0, 10], elevation), - ('titledupe something else ', 'whatever common', ('populartag',), [0, 10, 0, 10], elevation), - ('something titledupe else ', 'bar common', ('populartag',), [0, 50, 0, 50], elevation), - ('map metadata true', 'map metadata true', ('populartag',), [0, 22, 0, 22], farming), + ("GeoNode Default Map", "GeoNode default map abstract", ("populartag",), world_extent, biota), + ("ipsum lorem", "common ipsum lorem", ("populartag", "maptagunique"), world_extent, biota), + ("lorem1 ipsum1", "common abstract1", ("populartag",), world_extent, biota), + ("ipsum foo", "common bar lorem", ("populartag",), world_extent, location), + ("map one", "common this is a unique thing", ("populartag",), [0, 1, 0, 1], location), + ("quux", "common double thing", ("populartag",), [0, 5, 0, 5], location), + ("morx", "common thing double", ("populartag",), [0, 10, 0, 10], elevation), + ("titledupe something else ", "whatever common", ("populartag",), [0, 10, 0, 10], elevation), + ("something titledupe else ", "bar common", ("populartag",), [0, 50, 0, 50], elevation), + ("map metadata true", "map metadata true", ("populartag",), [0, 22, 0, 22], farming), ] user_data = [ - ('bobby', 'bob', 'bobby', ''), - ('norman', 'norman', 'norman', ''), - ('user1', 'pass', 'uniquefirst', 'foo'), - ('user2', 'pass', 'foo', 'uniquelast'), - ('unique_username', 'pass', 'foo', 'uniquelast'), - ('jblaze', 'pass', 'johnny', 'blaze'), - ('foo', 'pass', 'bar', 'baz'), + ("bobby", "bob", "bobby", ""), + ("norman", "norman", "norman", ""), + ("user1", "pass", "uniquefirst", "foo"), + ("user2", "pass", "foo", "uniquelast"), + ("unique_username", "pass", "foo", "uniquelast"), + ("jblaze", "pass", "johnny", "blaze"), + ("foo", "pass", "bar", "baz"), ] people_data = [ - ('this contains all my interesting profile information',), - ('some other information goes here',), + ("this contains all my interesting profile information",), + ("some other information goes here",), ] now = datetime.now(timezone.get_current_timezone()) step = timedelta(days=60) @@ -117,37 +116,75 @@ def it(): while True: yield current current = current - step + itinst = it() def callable(): return next(itinst) + return callable next_date = get_test_date() dataset_data = [ - ('CA', 'abstract1', 'CA', 'geonode:CA', world_extent, next_date(), ('populartag', 'here'), elevation), - ('layer2', 'abstract2', 'layer2', 'geonode:layer2', world_extent, next_date(), ('populartag',), elevation), - ('uniquetitle', 'something here', 'mylayer', 'geonode:mylayer', world_extent, next_date(), ('populartag',), elevation), - ('common blar', 'lorem ipsum', 'foo', 'geonode:foo', world_extent, next_date(), ('populartag', 'layertagunique'), location), - ('common double it', 'whatever', 'whatever', 'geonode:whatever', [0, 1, 0, 1], next_date(), ('populartag',), location), - ('common double time', 'else', 'fooey', 'geonode:fooey', [0, 5, 0, 5], next_date(), ('populartag',), location), - ('common bar', 'uniqueabstract', 'quux', 'geonode:quux', [0, 10, 0, 10], next_date(), ('populartag',), biota), - ('common morx', 'lorem ipsum', 'fleem', 'geonode:fleem', [0, 50, 0, 50], next_date(), ('populartag',), biota), - ('dataset metadata true', 'lorem ipsum', 'fleem', 'geonode:metadatatrue', [0, 22, 0, 22], next_date(), ('populartag',), farming) + ("CA", "abstract1", "CA", "geonode:CA", world_extent, next_date(), ("populartag", "here"), elevation), + ("layer2", "abstract2", "layer2", "geonode:layer2", world_extent, next_date(), ("populartag",), elevation), + ( + "uniquetitle", + "something here", + "mylayer", + "geonode:mylayer", + world_extent, + next_date(), + ("populartag",), + elevation, + ), + ( + "common blar", + "lorem ipsum", + "foo", + "geonode:foo", + world_extent, + next_date(), + ("populartag", "layertagunique"), + location, + ), + ( + "common double it", + "whatever", + "whatever", + "geonode:whatever", + [0, 1, 0, 1], + next_date(), + ("populartag",), + location, + ), + ("common double time", "else", "fooey", "geonode:fooey", [0, 5, 0, 5], next_date(), ("populartag",), location), + ("common bar", "uniqueabstract", "quux", "geonode:quux", [0, 10, 0, 10], next_date(), ("populartag",), biota), + ("common morx", "lorem ipsum", "fleem", "geonode:fleem", [0, 50, 0, 50], next_date(), ("populartag",), biota), + ( + "dataset metadata true", + "lorem ipsum", + "fleem", + "geonode:metadatatrue", + [0, 22, 0, 22], + next_date(), + ("populartag",), + farming, + ), ] document_data = [ - ('lorem ipsum', 'common lorem ipsum', ('populartag',), world_extent, biota), - ('ipsum lorem', 'common ipsum lorem', ('populartag', 'doctagunique'), world_extent, biota), - ('lorem1 ipsum1', 'common abstract1', ('populartag',), world_extent, biota), - ('ipsum foo', 'common bar lorem', ('populartag',), world_extent, location), - ('doc one', 'common this is a unique thing', ('populartag',), [0, 1, 0, 1], location), - ('quux', 'common double thing', ('populartag',), [0, 5, 0, 5], location), - ('morx', 'common thing double', ('populartag',), [0, 10, 0, 10], elevation), - ('titledupe something else ', 'whatever common', ('populartag',), [0, 10, 0, 10], elevation), - ('something titledupe else ', 'bar common', ('populartag',), [0, 50, 0, 50], elevation), - ('doc metadata true', 'doc metadata true', ('populartag',), [0, 22, 0, 22], farming) + ("lorem ipsum", "common lorem ipsum", ("populartag",), world_extent, biota), + ("ipsum lorem", "common ipsum lorem", ("populartag", "doctagunique"), world_extent, biota), + ("lorem1 ipsum1", "common abstract1", ("populartag",), world_extent, biota), + ("ipsum foo", "common bar lorem", ("populartag",), world_extent, location), + ("doc one", "common this is a unique thing", ("populartag",), [0, 1, 0, 1], location), + ("quux", "common double thing", ("populartag",), [0, 5, 0, 5], location), + ("morx", "common thing double", ("populartag",), [0, 10, 0, 10], elevation), + ("titledupe something else ", "whatever common", ("populartag",), [0, 10, 0, 10], elevation), + ("something titledupe else ", "bar common", ("populartag",), [0, 50, 0, 50], elevation), + ("doc metadata true", "doc metadata true", ("populartag",), [0, 22, 0, 22], farming), ] return map_data, user_data, people_data, dataset_data, document_data @@ -158,16 +195,16 @@ def create_models(type=None, integration=False): obj_ids = [] with transaction.atomic(): map_data, user_data, people_data, dataset_data, document_data = create_fixtures() - registeredmembers_group, _ = Group.objects.get_or_create(name='registered-members') - anonymous_group, _ = Group.objects.get_or_create(name='anonymous') - cont_group, _ = Group.objects.get_or_create(name='contributors') - perm = Permission.objects.get(codename='add_resourcebase') + registeredmembers_group, _ = Group.objects.get_or_create(name="registered-members") + anonymous_group, _ = Group.objects.get_or_create(name="anonymous") + cont_group, _ = Group.objects.get_or_create(name="contributors") + perm = Permission.objects.get(codename="add_resourcebase") cont_group.permissions.add(perm) logger.debug("[SetUp] Get or create user admin") - u, _ = get_user_model().objects.get_or_create(username='admin') - u.set_password('admin') + u, _ = get_user_model().objects.get_or_create(username="admin") + u.set_password("admin") u.is_superuser = True - u.first_name = 'admin' + u.first_name = "admin" u.save() u.groups.add(anonymous_group) users.append(u) @@ -187,11 +224,12 @@ def create_models(type=None, integration=False): users.append(u) logger.debug(f"[SetUp] Add group {anonymous_group}") - get_user_model().objects.get(username='AnonymousUser').groups.add(anonymous_group) + get_user_model().objects.get(username="AnonymousUser").groups.add(anonymous_group) from geonode.utils import DisableDjangoSignals + with DisableDjangoSignals(skip=integration): - if not type or ensure_string(type) == 'map': + if not type or ensure_string(type) == "map": for md, user in zip(map_data, cycle(users)): title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = md logger.debug(f"[SetUp] Add map {title}") @@ -203,10 +241,10 @@ def create_models(type=None, integration=False): owner=user, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", category=category, - metadata_only=title == 'map metadata true' - ) + metadata_only=title == "map metadata true", + ), ) m.set_default_permissions(owner=user) m.clear_dirty_state() @@ -216,7 +254,7 @@ def create_models(type=None, integration=False): m.keywords.add(kw) m.save() - if not type or ensure_string(type) == 'document': + if not type or ensure_string(type) == "document": for dd, user in zip(document_data, cycle(users)): title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = dd logger.debug(f"[SetUp] Add document {title}") @@ -228,11 +266,11 @@ def create_models(type=None, integration=False): owner=user, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", files=dfile, extension="gif", - metadata_only=title == 'doc metadata true' - ) + metadata_only=title == "doc metadata true", + ), ) m.set_default_permissions(owner=user) m.clear_dirty_state() @@ -242,8 +280,8 @@ def create_models(type=None, integration=False): m.keywords.add(kw) m.save() - if not type or ensure_string(type) == 'dataset': - for ld, user, subtype in zip(dataset_data, cycle(users), cycle(('raster', 'vector'))): + if not type or ensure_string(type) == "dataset": + for ld, user, subtype in zip(dataset_data, cycle(users), cycle(("raster", "vector"))): title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ld end = start + timedelta(days=365) logger.debug(f"[SetUp] Add dataset {title}") @@ -256,7 +294,7 @@ def create_models(type=None, integration=False): name=name, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", uuid=str(uuid4()), owner=user, temporal_extent_start=start, @@ -264,8 +302,8 @@ def create_models(type=None, integration=False): date=start, subtype=subtype, category=category, - metadata_only=title == 'dataset metadata true' - ) + metadata_only=title == "dataset metadata true", + ), ) dataset.set_default_permissions(owner=user) dataset.clear_dirty_state() @@ -281,12 +319,13 @@ def create_models(type=None, integration=False): def remove_models(obj_ids, type=None, integration=False): from geonode.utils import DisableDjangoSignals + with DisableDjangoSignals(skip=integration): if not type: - remove_models(None, type=b'map') - remove_models(None, type=b'dataset') - remove_models(None, type=b'document') - if type == 'map': + remove_models(None, type=b"map") + remove_models(None, type=b"dataset") + remove_models(None, type=b"document") + if type == "map": try: m_ids = obj_ids or [mp.id for mp in Map.objects.all()] for id in m_ids: @@ -294,7 +333,7 @@ def remove_models(obj_ids, type=None, integration=False): m.delete() except Exception: pass - elif type == 'dataset': + elif type == "dataset": try: l_ids = obj_ids or [lyr.id for lyr in Dataset.objects.all()] for id in l_ids: @@ -302,7 +341,7 @@ def remove_models(obj_ids, type=None, integration=False): dataset.delete() except Exception: pass - elif type == 'document': + elif type == "document": try: d_ids = obj_ids or [doc.id for doc in Document.objects.all()] for id in d_ids: @@ -313,31 +352,41 @@ def remove_models(obj_ids, type=None, integration=False): def dump_models(path=None): - result = serialize("json", sum([list(x) for x in - [get_user_model().objects.all(), - Dataset.objects.all(), - Map.objects.all(), - Document.objects.all(), - Tag.objects.all(), - TaggedItem.objects.all(), - ]], []), indent=2, use_natural_keys=True) + result = serialize( + "json", + sum( + [ + list(x) + for x in [ + get_user_model().objects.all(), + Dataset.objects.all(), + Map.objects.all(), + Document.objects.all(), + Tag.objects.all(), + TaggedItem.objects.all(), + ] + ], + [], + ), + indent=2, + use_natural_keys=True, + ) if path is None: parent, _ = os.path.split(__file__) - path = os.path.join(parent, 'fixtures', 'search_testdata.json') - with open(path, 'w') as f: + path = os.path.join(parent, "fixtures", "search_testdata.json") + with open(path, "w") as f: f.write(result) def create_single_dataset(name, keywords=None, owner=None, group=None, **kwargs): - admin, created = get_user_model().objects.get_or_create(username='admin') + admin, created = get_user_model().objects.get_or_create(username="admin") if created: admin.is_superuser = True - admin.first_name = 'admin' - admin.set_password('admin') + admin.first_name = "admin" + admin.set_password("admin") admin.save() - test_datetime = datetime.strptime('2020-01-01', '%Y-%m-%d') - ll = (name, 'lorem ipsum', name, f'geonode:{name}', [ - 0, 22, 0, 22], test_datetime, ('populartag',), "farming") + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + ll = (name, "lorem ipsum", name, f"geonode:{name}", [0, 22, 0, 22], test_datetime, ("populartag",), "farming") title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ll try: dataset, _ = Dataset.objects.get_or_create( @@ -348,7 +397,7 @@ def create_single_dataset(name, keywords=None, owner=None, group=None, **kwargs) name=name, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", uuid=str(uuid4()), owner=owner or admin, temporal_extent_start=test_datetime, @@ -358,8 +407,8 @@ def create_single_dataset(name, keywords=None, owner=None, group=None, **kwargs) resource_type="dataset", typename=f"geonode:{title}", group=group, - **kwargs - ) + **kwargs, + ), ) if isinstance(keywords, list): @@ -374,15 +423,14 @@ def create_single_dataset(name, keywords=None, owner=None, group=None, **kwargs) def create_single_map(name, owner=None, **kwargs): - admin, created = get_user_model().objects.get_or_create(username='admin') + admin, created = get_user_model().objects.get_or_create(username="admin") if created: admin.is_superuser = True - admin.first_name = 'admin' - admin.set_password('admin') + admin.first_name = "admin" + admin.set_password("admin") admin.save() - test_datetime = datetime.strptime('2020-01-01', '%Y-%m-%d') - ll = (name, 'lorem ipsum', name, f'{name}', [ - 0, 22, 0, 22], test_datetime, ('populartag',)) + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + ll = (name, "lorem ipsum", name, f"{name}", [0, 22, 0, 22], test_datetime, ("populartag",)) title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws = ll m, _ = Map.objects.get_or_create( title=title, @@ -392,10 +440,10 @@ def create_single_map(name, owner=None, **kwargs): owner=owner or admin, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", resource_type="map", - **kwargs - ) + **kwargs, + ), ) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() @@ -404,15 +452,14 @@ def create_single_map(name, owner=None, **kwargs): def create_single_doc(name, owner=None, **kwargs): - admin, created = get_user_model().objects.get_or_create(username='admin') + admin, created = get_user_model().objects.get_or_create(username="admin") if created: admin.is_superuser = True - admin.first_name = 'admin' - admin.set_password('admin') + admin.first_name = "admin" + admin.set_password("admin") admin.save() - test_datetime = datetime.strptime('2020-01-01', '%Y-%m-%d') - dd = (name, 'lorem ipsum', name, f'{name}', [ - 0, 22, 0, 22], test_datetime, ('populartag',)) + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + dd = (name, "lorem ipsum", name, f"{name}", [0, 22, 0, 22], test_datetime, ("populartag",)) title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws = dd logger.debug(f"[SetUp] Add document {title}") m, _ = Document.objects.get_or_create( @@ -423,11 +470,11 @@ def create_single_doc(name, owner=None, **kwargs): owner=owner or admin, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), - srid='EPSG:4326', + srid="EPSG:4326", files=dfile, resource_type="document", - **kwargs - ) + **kwargs, + ), ) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() @@ -442,6 +489,6 @@ def add_keywords_to_resource(resource, keywords): return resource -if __name__ == '__main__': +if __name__ == "__main__": create_models() dump_models() diff --git a/geonode/base/templatetags/base_tags.py b/geonode/base/templatetags/base_tags.py index 24b51253c11..2306f411d2c 100644 --- a/geonode/base/templatetags/base_tags.py +++ b/geonode/base/templatetags/base_tags.py @@ -35,24 +35,22 @@ from geonode.documents.models import Document from geonode.groups.models import GroupProfile from geonode.base.bbox_utils import filter_bbox -from geonode.base.models import ( - HierarchicalKeyword, Menu, MenuItem -) +from geonode.base.models import HierarchicalKeyword, Menu, MenuItem from geonode.security.utils import get_visible_resources from collections import OrderedDict register = template.Library() FACETS = { - 'raster': _('Raster Dataset'), - 'vector': _('Vector Dataset'), - 'vector_time': _('Vector Temporal Serie'), - 'remote': _('Remote Dataset'), - 'wms': _('WMS Cascade Dataset') + "raster": _("Raster Dataset"), + "vector": _("Vector Dataset"), + "vector_time": _("Vector Temporal Serie"), + "remote": _("Remote Dataset"), + "wms": _("WMS Cascade Dataset"), } -@register.filter(name='template_trans') +@register.filter(name="template_trans") def template_trans(text): try: return ugettext(text) @@ -68,42 +66,43 @@ def num_ratings(obj): @register.simple_tag(takes_context=True) def facets(context): - request = context['request'] - title_filter = request.GET.get('title__icontains', '') - abstract_filter = request.GET.get('abstract__icontains', '') - purpose_filter = request.GET.get('purpose__icontains', '') - extent_filter = request.GET.get('extent', None) - keywords_filter = request.GET.getlist('keywords__slug__in', None) - category_filter = request.GET.getlist('category__identifier__in', None) - regions_filter = request.GET.getlist('regions__name__in', None) - owner_filter = request.GET.getlist('owner__username__in', None) - date_gte_filter = request.GET.get('date__gte', None) - date_lte_filter = request.GET.get('date__lte', None) - date_range_filter = request.GET.get('date__range', None) - - facet_type = context.get('facet_type', 'all') + request = context["request"] + title_filter = request.GET.get("title__icontains", "") + abstract_filter = request.GET.get("abstract__icontains", "") + purpose_filter = request.GET.get("purpose__icontains", "") + extent_filter = request.GET.get("extent", None) + keywords_filter = request.GET.getlist("keywords__slug__in", None) + category_filter = request.GET.getlist("category__identifier__in", None) + regions_filter = request.GET.getlist("regions__name__in", None) + owner_filter = request.GET.getlist("owner__username__in", None) + date_gte_filter = request.GET.get("date__gte", None) + date_lte_filter = request.GET.get("date__lte", None) + date_range_filter = request.GET.get("date__range", None) + + facet_type = context.get("facet_type", "all") if not settings.SKIP_PERMS_FILTER: authorized = [] try: - authorized = get_objects_for_user( - request.user, 'base.view_resourcebase').values('id') + authorized = get_objects_for_user(request.user, "base.view_resourcebase").values("id") except Exception: pass - if facet_type == 'geoapps': + if facet_type == "geoapps": facets = {} from django.apps import apps + for label, app in apps.app_configs.items(): - if hasattr(app, 'type') and app.type == 'GEONODE_APP': - if hasattr(app, 'default_model'): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): geoapps = get_visible_resources( apps.get_model(label, app.default_model).objects.all(), request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) if category_filter: geoapps = geoapps.filter(category__identifier__in=category_filter) @@ -116,7 +115,7 @@ def facets(context): if date_lte_filter: geoapps = geoapps.filter(date__lte=date_lte_filter) if date_range_filter: - geoapps = geoapps.filter(date__range=date_range_filter.split(',')) + geoapps = geoapps.filter(date__range=date_range_filter.split(",")) if extent_filter: geoapps = filter_bbox(geoapps, extent_filter) @@ -139,7 +138,7 @@ def facets(context): facets[app.default_model] = geoapps.count() return facets - elif facet_type == 'documents': + elif facet_type == "documents": documents = Document.objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) @@ -152,14 +151,15 @@ def facets(context): if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: - documents = documents.filter(date__range=date_range_filter.split(',')) + documents = documents.filter(date__range=date_range_filter.split(",")) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() @@ -177,15 +177,15 @@ def facets(context): if not settings.SKIP_PERMS_FILTER: documents = documents.filter(id__in=authorized) - counts = documents.values('subtype').annotate(count=Count('subtype')) - facets = {count['subtype']: count['count'] for count in counts} + counts = documents.values("subtype").annotate(count=Count("subtype")) + facets = {count["subtype"]: count["count"] for count in counts} return facets else: layers = Dataset.objects.filter( - Q(title__icontains=title_filter) | - Q(abstract__icontains=abstract_filter) | - Q(purpose__icontains=purpose_filter) + Q(title__icontains=title_filter) + | Q(abstract__icontains=abstract_filter) + | Q(purpose__icontains=purpose_filter) ) if category_filter: layers = layers.filter(category__identifier__in=category_filter) @@ -198,14 +198,15 @@ def facets(context): if date_lte_filter: layers = layers.filter(date__lte=date_lte_filter) if date_range_filter: - layers = layers.filter(date__range=date_range_filter.split(',')) + layers = layers.filter(date__range=date_range_filter.split(",")) layers = get_visible_resources( layers, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) if extent_filter: layers = filter_bbox(layers, extent_filter) @@ -226,33 +227,34 @@ def facets(context): if not settings.SKIP_PERMS_FILTER: layers = layers.filter(id__in=authorized) - counts = layers.values('subtype').annotate(count=Count('subtype')) + counts = layers.values("subtype").annotate(count=Count("subtype")) counts_array = [] try: for count in counts: - counts_array.append((count['subtype'], count['count'])) + counts_array.append((count["subtype"], count["count"])) except Exception: pass count_dict = dict(counts_array) - vector_time_series = layers.exclude(has_time=False).filter(subtype='vector'). \ - values('subtype').annotate(count=Count('subtype')) + vector_time_series = ( + layers.exclude(has_time=False).filter(subtype="vector").values("subtype").annotate(count=Count("subtype")) + ) if vector_time_series: - count_dict['vectorTimeSeries'] = vector_time_series[0]['count'] + count_dict["vectorTimeSeries"] = vector_time_series[0]["count"] facets = { - 'raster': count_dict.get('raster', 0), - 'vector': count_dict.get('vector', 0), - 'vector_time': count_dict.get('vectorTimeSeries', 0), - 'remote': count_dict.get('remote', 0), - 'wms': count_dict.get('wmsStore', 0), + "raster": count_dict.get("raster", 0), + "vector": count_dict.get("vector", 0), + "vector_time": count_dict.get("vectorTimeSeries", 0), + "remote": count_dict.get("remote", 0), + "wms": count_dict.get("wmsStore", 0), } # Break early if only_datasets is set. - if facet_type == 'datasets': + if facet_type == "datasets": return facets maps = Map.objects.filter(title__icontains=title_filter) @@ -274,21 +276,23 @@ def facets(context): maps = maps.filter(date__lte=date_lte_filter) documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: - maps = maps.filter(date__range=date_range_filter.split(',')) - documents = documents.filter(date__range=date_range_filter.split(',')) + maps = maps.filter(date__range=date_range_filter.split(",")) + documents = documents.filter(date__range=date_range_filter.split(",")) maps = get_visible_resources( maps, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) if extent_filter: documents = filter_bbox(documents, extent_filter) @@ -311,103 +315,98 @@ def facets(context): maps = maps.filter(id__in=authorized) documents = documents.filter(id__in=authorized) - facets['map'] = maps.count() - facets['document'] = documents.count() + facets["map"] = maps.count() + facets["document"] = documents.count() - if facet_type == 'home': - facets['user'] = get_user_model().objects.exclude( - username='AnonymousUser').count() + if facet_type == "home": + facets["user"] = get_user_model().objects.exclude(username="AnonymousUser").count() - facets['group'] = GroupProfile.objects.exclude( - access="private").count() + facets["group"] = GroupProfile.objects.exclude(access="private").count() - facets['dataset'] = facets['raster'] + facets['vector'] + facets['remote'] + facets['wms'] + facets["dataset"] = facets["raster"] + facets["vector"] + facets["remote"] + facets["wms"] return facets @register.simple_tag(takes_context=True) def get_current_path(context): - request = context['request'] + request = context["request"] return request.get_full_path() @register.simple_tag(takes_context=True) def get_context_resourcetype(context): c_path = get_current_path(context) - resource_types = ['datasets', 'maps', 'geoapps', 'documents', 'search', 'people', - 'groups/categories', 'groups'] + resource_types = ["datasets", "maps", "geoapps", "documents", "search", "people", "groups/categories", "groups"] for resource_type in resource_types: if f"/{resource_type}/" in c_path: return resource_type - return 'error' + return "error" @register.simple_tag(takes_context=True) def fullurl(context, url): if not url: - return '' - r = context['request'] + return "" + r = context["request"] return r.build_absolute_uri(url) @register.simple_tag def get_menu(placeholder_name): menus = { - m: MenuItem.objects.filter(menu=m).order_by('order') + m: MenuItem.objects.filter(menu=m).order_by("order") for m in Menu.objects.filter(placeholder__name=placeholder_name) } return OrderedDict(menus.items()) -@register.inclusion_tag(filename='base/menu.html') +@register.inclusion_tag(filename="base/menu.html") def render_nav_menu(placeholder_name): menus = {} try: menus = { - m: MenuItem.objects.filter(menu=m).order_by('order') + m: MenuItem.objects.filter(menu=m).order_by("order") for m in Menu.objects.filter(placeholder__name=placeholder_name) } except Exception: pass - return {'menus': OrderedDict(menus.items())} + return {"menus": OrderedDict(menus.items())} -@register.inclusion_tag(filename='base/iso_categories.html') +@register.inclusion_tag(filename="base/iso_categories.html") def get_visibile_resources(user): - categories = get_objects_for_user(user, 'view_resourcebase', klass=ResourceBase, any_perm=False)\ - .filter(category__isnull=False).values('category__gn_description', - 'category__fa_class', 'category__description', 'category__identifier')\ - .annotate(count=Count('category')) + categories = ( + get_objects_for_user(user, "view_resourcebase", klass=ResourceBase, any_perm=False) + .filter(category__isnull=False) + .values("category__gn_description", "category__fa_class", "category__description", "category__identifier") + .annotate(count=Count("category")) + ) - return { - 'iso_formats': categories - } + return {"iso_formats": categories} @register.simple_tag def display_edit_request_button(resource, user, perms): def _has_owner_his_permissions(): _owner_perms = set( - resource.BASE_PERMISSIONS.get('owner') + - resource.BASE_PERMISSIONS.get('read') + - resource.BASE_PERMISSIONS.get('write') + resource.BASE_PERMISSIONS.get("owner") + + resource.BASE_PERMISSIONS.get("read") + + resource.BASE_PERMISSIONS.get("write") ) - if resource.resource_type in ['dataset', 'document']: - ''' + if resource.resource_type in ["dataset", "document"]: + """ The download resource permission should be available only if the resource is a datasets or Documents. You cant download maps - ''' - _owner_perms = _owner_perms.union(set(resource.BASE_PERMISSIONS.get('download'))) + """ + _owner_perms = _owner_perms.union(set(resource.BASE_PERMISSIONS.get("download"))) _owner_set = _owner_perms.difference(set(perms)) - return _owner_set == set() or \ - _owner_set == {'change_resourcebase_permissions', 'publish_resourcebase'} + return _owner_set == set() or _owner_set == {"change_resourcebase_permissions", "publish_resourcebase"} - if not _has_owner_his_permissions() and \ - (user.is_superuser or resource.owner.pk == user.pk): + if not _has_owner_his_permissions() and (user.is_superuser or resource.owner.pk == user.pk): return True return False @@ -418,9 +417,9 @@ def display_change_perms_button(resource, user, perms): from geonode.geoserver.helpers import ogc_server_settings except Exception: return False - if not getattr(ogc_server_settings, 'GEONODE_SECURITY_ENABLED', False): + if not getattr(ogc_server_settings, "GEONODE_SECURITY_ENABLED", False): return False - elif user.is_superuser or 'change_resourcebase_permissions' in set(perms): + elif user.is_superuser or "change_resourcebase_permissions" in set(perms): return True else: - return not getattr(settings, 'ADMIN_MODERATE_UPLOADS', False) + return not getattr(settings, "ADMIN_MODERATE_UPLOADS", False) diff --git a/geonode/base/templatetags/thesaurus.py b/geonode/base/templatetags/thesaurus.py index 032f8d47f5a..2b9c7e05371 100644 --- a/geonode/base/templatetags/thesaurus.py +++ b/geonode/base/templatetags/thesaurus.py @@ -49,9 +49,7 @@ def get_thesaurus_translation_by_id(id): def get_thesaurus_localized_label(tkeyword): lang = get_language() translation = ( - ThesaurusKeywordLabel.objects.values_list("label", flat=True) - .filter(keyword__id=tkeyword.id) - .filter(lang=lang) + ThesaurusKeywordLabel.objects.values_list("label", flat=True).filter(keyword__id=tkeyword.id).filter(lang=lang) ) if not translation: return tkeyword.alt_label diff --git a/geonode/base/templatetags/user_messages.py b/geonode/base/templatetags/user_messages.py index e690b0bb80a..92c381ed8c9 100644 --- a/geonode/base/templatetags/user_messages.py +++ b/geonode/base/templatetags/user_messages.py @@ -28,8 +28,10 @@ @register.simple_tag def is_unread(thread, user): - if thread.userthread_set.filter(user=user, unread=True).exists() or \ - thread.groupmemberthread_set.filter(user=user, unread=True).exists(): + if ( + thread.userthread_set.filter(user=user, unread=True).exists() + or thread.groupmemberthread_set.filter(user=user, unread=True).exists() + ): return True return False @@ -42,30 +44,30 @@ def random_uuid(): @register.simple_tag def format_senders(thread, current_user): User = get_user_model() - users = User.objects.filter(sent_messages__thread=thread).annotate(Sum('pk')) - sender_string = '' + users = User.objects.filter(sent_messages__thread=thread).annotate(Sum("pk")) + sender_string = "" u_count = users.count() if u_count < 3: for user in users: if user == current_user: - user_repr = _('me') + user_repr = _("me") else: - user_repr = f'{user.full_name_or_nick}' - sender_string += f'{user_repr}, ' + user_repr = f"{user.full_name_or_nick}" + sender_string += f"{user_repr}, " sender_string = sender_string[:-2] elif u_count == 3: for user in users: if user == current_user: - user_repr = _('me') + user_repr = _("me") else: - user_repr = f'{user.first_name_or_nick}' - sender_string += f'{user_repr}, ' + user_repr = f"{user.first_name_or_nick}" + sender_string += f"{user_repr}, " sender_string = sender_string[:-2] else: first_sender = thread.first_message.sender last_sender = thread.latest_message.sender - sender_string = f'{first_sender.first_name_or_nick} .. {last_sender.first_name_or_nick}' - return f'{sender_string}' + sender_string = f"{first_sender.first_name_or_nick} .. {last_sender.first_name_or_nick}" + return f"{sender_string}" @register.simple_tag @@ -75,8 +77,7 @@ def get_item(dictionary, key): @register.simple_tag def show_notification(notice_type_label, current_user): - adms_notice_types = getattr(settings, 'ADMINS_ONLY_NOTICE_TYPES', []) - if not current_user.is_superuser and adms_notice_types and \ - notice_type_label in adms_notice_types: + adms_notice_types = getattr(settings, "ADMINS_ONLY_NOTICE_TYPES", []) + if not current_user.is_superuser and adms_notice_types and notice_type_label in adms_notice_types: return False return True diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 7be78f2ea6a..9beb01b31e8 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -48,7 +48,7 @@ TopicCategory, Thesaurus, ThesaurusKeyword, - generate_thesaurus_reference + generate_thesaurus_reference, ) from django.conf import settings from django.contrib.gis.geos import Polygon @@ -62,7 +62,10 @@ from geonode.base.middleware import ReadOnlyMiddleware, MaintenanceMiddleware from geonode.base.templatetags.base_tags import get_visibile_resources, facets from geonode.base.templatetags.thesaurus import ( - get_name_translation, get_thesaurus_localized_label, get_thesaurus_translation_by_id, get_unique_thesaurus_set, + get_name_translation, + get_thesaurus_localized_label, + get_thesaurus_translation_by_id, + get_unique_thesaurus_set, get_thesaurus_title, get_thesaurus_date, ) @@ -76,14 +79,13 @@ from geonode.base.forms import ThesaurusAvailableForm, THESAURUS_RESULT_LIST_SEPERATOR -test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) +test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) class ThumbnailTests(GeoNodeBaseTestSupport): - def setUp(self): super().setUp() - self.rb = ResourceBase.objects.create(uuid=str(uuid4()), owner=get_user_model().objects.get(username='admin')) + self.rb = ResourceBase.objects.create(uuid=str(uuid4()), owner=get_user_model().objects.get(username="admin")) def tearDown(self): super().tearDown() @@ -101,15 +103,15 @@ def test_empty_image(self): Tests that an empty image does not change the current resource thumbnail. """ current = self.rb.get_thumbnail_url() - self.rb.save_thumbnail('test-thumb', None) + self.rb.save_thumbnail("test-thumb", None) self.assertEqual(current, self.rb.get_thumbnail_url()) - @patch('PIL.Image.open', return_value=test_image) + @patch("PIL.Image.open", return_value=test_image) def test_monochromatic_image(self, image): """ Tests that an monochromatic image does not change the current resource thumbnail. """ - filename = 'test-thumb' + filename = "test-thumb" current = self.rb.get_thumbnail_url() self.rb.save_thumbnail(filename, image) @@ -119,12 +121,12 @@ def test_monochromatic_image(self, image): thumb_utils.remove_thumbs(filename) self.assertFalse(thumb_utils.thumb_exists(filename)) - @patch('PIL.Image.open', return_value=test_image) + @patch("PIL.Image.open", return_value=test_image) def test_thumb_utils_methods(self, image): """ Bunch of tests on thumb_utils helpers. """ - filename = 'test-thumb' + filename = "test-thumb" upload_path = thumb_utils.thumb_path(filename) self.assertEqual(upload_path, os.path.join(settings.THUMBNAIL_LOCATION, filename)) thumb_utils.remove_thumbs(filename) @@ -141,25 +143,23 @@ def test_thumb_utils_methods(self, image): class TestThumbnailUrl(GeoNodeBaseTestSupport): - def setUp(self): super().setUp() f = BytesIO(test_image.tobytes()) - f.name = 'test_image.jpeg' + f.name = "test_image.jpeg" class TestCreationOfMissingMetadataAuthorsOrPOC(ThumbnailTests): - def test_add_missing_metadata_author_or_poc(self): """ Test that calling add_missing_metadata_author_or_poc resource method sets a missing metadata_author and/or point of contact (poc) to resource owner """ - user = get_user_model().objects.create(username='zlatan_i') + user = get_user_model().objects.create(username="zlatan_i") self.rb.owner = user self.rb.add_missing_metadata_author_or_poc() - self.assertEqual(self.rb.metadata_author.username, 'zlatan_i') - self.assertEqual(self.rb.poc.username, 'zlatan_i') + self.assertEqual(self.rb.metadata_author.username, "zlatan_i") + self.assertEqual(self.rb.poc.username, "zlatan_i") class RenderMenuTagTest(GeoNodeBaseTestSupport): @@ -169,385 +169,303 @@ class RenderMenuTagTest(GeoNodeBaseTestSupport): def setUp(self): super().setUp() - self.placeholder_0 = MenuPlaceholder.objects.create( - name='test_menu_placeholder_0' - ) - self.placeholder_1 = MenuPlaceholder.objects.create( - name='test_unicode_äöü_menu_placeholder_1' - ) - self.menu_0_0 = Menu.objects.create( - title='test_menu_0_0', - order=0, - placeholder=self.placeholder_0 - - ) - self.menu_0_1 = Menu.objects.create( - title='test_menu_0_1', - order=1, - placeholder=self.placeholder_0 - - ) - self.menu_1_0 = Menu.objects.create( - title='test_unicode_äöü_menu_1_0', - order=0, - placeholder=self.placeholder_1 - - ) + self.placeholder_0 = MenuPlaceholder.objects.create(name="test_menu_placeholder_0") + self.placeholder_1 = MenuPlaceholder.objects.create(name="test_unicode_äöü_menu_placeholder_1") + self.menu_0_0 = Menu.objects.create(title="test_menu_0_0", order=0, placeholder=self.placeholder_0) + self.menu_0_1 = Menu.objects.create(title="test_menu_0_1", order=1, placeholder=self.placeholder_0) + self.menu_1_0 = Menu.objects.create(title="test_unicode_äöü_menu_1_0", order=0, placeholder=self.placeholder_1) self.menu_item_0_0_0 = MenuItem.objects.create( - title='test_menu_item_0_0_0', - order=0, - blank_target=False, - url='/about', - menu=self.menu_0_0 + title="test_menu_item_0_0_0", order=0, blank_target=False, url="/about", menu=self.menu_0_0 ) self.menu_item_0_0_1 = MenuItem.objects.create( - title='test_menu_item_0_0_1', - order=1, - blank_target=False, - url='/about', - menu=self.menu_0_0 + title="test_menu_item_0_0_1", order=1, blank_target=False, url="/about", menu=self.menu_0_0 ) self.menu_item_0_1_0 = MenuItem.objects.create( - title='test_menu_item_0_1_0', - order=0, - blank_target=False, - url='/about', - menu=self.menu_0_1 + title="test_menu_item_0_1_0", order=0, blank_target=False, url="/about", menu=self.menu_0_1 ) self.menu_item_0_1_1 = MenuItem.objects.create( - title='test_menu_item_0_1_1', - order=1, - blank_target=False, - url='/about', - menu=self.menu_0_1 + title="test_menu_item_0_1_1", order=1, blank_target=False, url="/about", menu=self.menu_0_1 ) self.menu_item_0_1_2 = MenuItem.objects.create( - title='test_menu_item_0_1_2', - order=2, - blank_target=False, - url='/about', - menu=self.menu_0_1 + title="test_menu_item_0_1_2", order=2, blank_target=False, url="/about", menu=self.menu_0_1 ) self.menu_item_1_0_0 = MenuItem.objects.create( - title='test_unicode_äöü_menu_item_1_0_0', - order=0, - blank_target=False, - url='/about', - menu=self.menu_1_0 + title="test_unicode_äöü_menu_item_1_0_0", order=0, blank_target=False, url="/about", menu=self.menu_1_0 ) self.menu_item_1_0_1 = MenuItem.objects.create( - title='test_unicode_äöü_menu_item_1_0_1', - order=1, - blank_target=False, - url='/about', - menu=self.menu_1_0 + title="test_unicode_äöü_menu_item_1_0_1", order=1, blank_target=False, url="/about", menu=self.menu_1_0 ) def test_get_menu_placeholder_0(self): - template = Template( - "{% load base_tags %} {% get_menu 'test_menu_placeholder_0' %}" - ) + template = Template("{% load base_tags %} {% get_menu 'test_menu_placeholder_0' %}") rendered = template.render(Context({})) # menu_placeholder_0 # first menu with ascii chars self.assertIn( - self.menu_0_0.title, - rendered, - f'Expected "{self.menu_0_0.title}" string in the rendered template' + self.menu_0_0.title, rendered, f'Expected "{self.menu_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_0.title, rendered, - f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template' + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_0_1.title, rendered, - f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template' + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template', ) # second menu self.assertIn( - self.menu_0_1.title, - rendered, - f'Expected "{self.menu_0_1.title}" string in the rendered template' + self.menu_0_1.title, rendered, f'Expected "{self.menu_0_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_0.title, rendered, - f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_1_1.title, rendered, - f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_1_2.title, rendered, - f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template', ) # menu_placeholder_1 # first menu # unicode self.assertNotIn( - self.menu_1_0.title, - rendered, - f'No "{self.menu_1_0.title}" string expected in the rendered template' + self.menu_1_0.title, rendered, f'No "{self.menu_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_0.title, rendered, - f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template' + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_1_0_1.title, rendered, - f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template' + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template', ) def test_get_menu_placeholder_1(self): - template = Template( - "{% load base_tags %} {% get_menu 'test_unicode_äöü_menu_placeholder_1' %}" - ) + template = Template("{% load base_tags %} {% get_menu 'test_unicode_äöü_menu_placeholder_1' %}") rendered = template.render(Context({})) # menu_placeholder_0 # first menu self.assertNotIn( - self.menu_0_0.title, - rendered, - f'No "{self.menu_0_0.title}" string expected in the rendered template' + self.menu_0_0.title, rendered, f'No "{self.menu_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_0.title, rendered, - f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template' + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_0_1.title, rendered, - f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template' + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template', ) # second menu self.assertNotIn( - self.menu_0_1.title, - rendered, - f'No "{self.menu_0_1.title}" string expected in the rendered template' + self.menu_0_1.title, rendered, f'No "{self.menu_0_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_0.title, rendered, - f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_1_1.title, rendered, - f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_1_2.title, rendered, - f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template', ) # menu_placeholder_1 # first menu # unicode self.assertIn( - self.menu_1_0.title, - rendered, - f'Expected "{self.menu_1_0.title}" string in the rendered template' + self.menu_1_0.title, rendered, f'Expected "{self.menu_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_0.title, rendered, - f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template' + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_1_0_1.title, rendered, - f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template' + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template', ) def test_render_nav_menu_placeholder_0(self): - template = Template( - "{% load base_tags %} {% render_nav_menu 'test_menu_placeholder_0' %}" - ) + template = Template("{% load base_tags %} {% render_nav_menu 'test_menu_placeholder_0' %}") rendered = template.render(Context({})) # menu_placeholder_0 # first menu self.assertIn( - self.menu_0_0.title, - rendered, - f'Expected "{self.menu_0_0.title}" string in the rendered template' + self.menu_0_0.title, rendered, f'Expected "{self.menu_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_0.title, rendered, - f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template' + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_0_1.title, rendered, - f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template' + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template', ) # second menu self.assertIn( - self.menu_0_1.title, - rendered, - f'Expected "{self.menu_0_1.title}" string in the rendered template' + self.menu_0_1.title, rendered, f'Expected "{self.menu_0_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_0.title, rendered, - f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_1_1.title, rendered, - f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template', ) self.assertIn( self.menu_item_0_1_2.title, rendered, - f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template' + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template', ) # menu_placeholder_1 # first menu # unicode self.assertNotIn( - self.menu_1_0.title, - rendered, - f'No "{self.menu_1_0.title}" string expected in the rendered template' + self.menu_1_0.title, rendered, f'No "{self.menu_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_0.title, rendered, - f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template' + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_1_0_1.title, rendered, - f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template' + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template', ) def test_render_nav_menu_placeholder_1(self): - template = Template( - "{% load base_tags %} {% render_nav_menu 'test_unicode_äöü_menu_placeholder_1' %}" - ) + template = Template("{% load base_tags %} {% render_nav_menu 'test_unicode_äöü_menu_placeholder_1' %}") rendered = template.render(Context({})) # menu_placeholder_0 # first menu self.assertNotIn( - self.menu_0_0.title, - rendered, - f'No "{self.menu_0_0.title}" string expected in the rendered template' + self.menu_0_0.title, rendered, f'No "{self.menu_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_0.title, rendered, - f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template' + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_0_1.title, rendered, - f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template' + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template', ) # second menu self.assertNotIn( - self.menu_0_1.title, - rendered, - f'No "{self.menu_0_1.title}" string expected in the rendered template' + self.menu_0_1.title, rendered, f'No "{self.menu_0_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_0.title, rendered, - f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_1_1.title, rendered, - f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template', ) self.assertNotIn( self.menu_item_0_1_2.title, rendered, - f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template' + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template', ) # menu_placeholder_1 # first menu # unicode self.assertIn( - self.menu_1_0.title, - rendered, - f'Expected "{self.menu_1_0.title}" string in the rendered template' + self.menu_1_0.title, rendered, f'Expected "{self.menu_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_0.title, rendered, - f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template' + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template', ) self.assertIn( self.menu_item_1_0_1.title, rendered, - f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template' + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template', ) class DeleteResourcesCommandTests(GeoNodeBaseTestSupport): - def test_delete_resources_no_arguments(self): args = [] kwargs = {} with self.assertRaises(CommandError) as exception: - call_command('delete_resources', *args, **kwargs) + call_command("delete_resources", *args, **kwargs) self.assertIn( - 'No configuration provided', - exception.exception.args[0], - '"No configuration" exception expected.' + "No configuration provided", exception.exception.args[0], '"No configuration" exception expected.' ) def test_delete_resources_too_many_arguments(self): args = [] - kwargs = {'config_path': '/example/config.txt', 'map_filters': "*"} + kwargs = {"config_path": "/example/config.txt", "map_filters": "*"} with self.assertRaises(CommandError) as exception: - call_command('delete_resources', *args, **kwargs) + call_command("delete_resources", *args, **kwargs) self.assertIn( - 'Too many configuration options provided', + "Too many configuration options provided", exception.exception.args[0], - '"Too many configuration options provided" exception expected.' + '"Too many configuration options provided" exception expected.', ) def test_delete_resource_config_file_not_existing(self): args = [] - kwargs = {'config_path': '/example/config.json'} + kwargs = {"config_path": "/example/config.json"} with self.assertRaises(CommandError) as exception: - call_command('delete_resources', *args, **kwargs) + call_command("delete_resources", *args, **kwargs) self.assertIn( - 'Specified configuration file does not exist', + "Specified configuration file does not exist", exception.exception.args[0], - '"Specified configuration file does not exist" exception expected.' + '"Specified configuration file does not exist" exception expected.', ) def test_delete_resource_config_file_empty(self): # create an empty config file - config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'delete_resources_config.json') - open(config_file_path, 'a').close() + config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "delete_resources_config.json") + open(config_file_path, "a").close() args = [] - kwargs = {'config_path': config_file_path} + kwargs = {"config_path": config_file_path} with self.assertRaises(CommandError) as exception: - call_command('delete_resources', *args, **kwargs) + call_command("delete_resources", *args, **kwargs) self.assertIn( - 'Specified configuration file is empty', + "Specified configuration file is empty", exception.exception.args[0], - '"Specified configuration file is empty" exception expected.' + '"Specified configuration file is empty" exception expected.', ) # delete the config file @@ -555,7 +473,6 @@ def test_delete_resource_config_file_empty(self): class ConfigurationTest(GeoNodeBaseTestSupport): - @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_read_only_whitelist(self): web_client = Client() @@ -568,18 +485,18 @@ def test_read_only_whitelist(self): # post to whitelisted URLs as AnonymousUser for url_name in ReadOnlyMiddleware.WHITELISTED_URL_NAMES: - if url_name == 'login': - response = web_client.post(reverse('admin:login')) - elif url_name == 'logout': - response = web_client.post(reverse('admin:logout')) + if url_name == "login": + response = web_client.post(reverse("admin:login")) + elif url_name == "logout": + response = web_client.post(reverse("admin:logout")) else: response = web_client.post(reverse(url_name)) - self.assertNotEqual(response.status_code, 405, 'Whitelisted URL is not available.') + self.assertNotEqual(response.status_code, 405, "Whitelisted URL is not available.") def test_read_only_casual_user_privileges(self): web_client = Client() - url_name = 'autocomplete_region' + url_name = "autocomplete_region" # set read-only flag config = Configuration.load() @@ -588,13 +505,13 @@ def test_read_only_casual_user_privileges(self): config.save() # get user - user, _ = get_user_model().objects.get_or_create(username='user1') + user, _ = get_user_model().objects.get_or_create(username="user1") web_client.force_login(user) # post not whitelisted URL as superuser response = web_client.post(reverse(url_name)) - self.assertEqual(response.status_code, 405, 'User is allowed to post to forbidden URL') + self.assertEqual(response.status_code, 405, "User is allowed to post to forbidden URL") def test_maintenance_whitelist(self): @@ -608,17 +525,17 @@ def test_maintenance_whitelist(self): # post to whitelisted URLs as AnonymousUser for url_name in MaintenanceMiddleware.WHITELISTED_URL_NAMES: - if url_name == 'login': - response = web_client.get(reverse('admin:login')) - elif url_name == 'logout': - response = web_client.get(reverse('admin:logout')) - elif url_name == 'index': + if url_name == "login": + response = web_client.get(reverse("admin:login")) + elif url_name == "logout": + response = web_client.get(reverse("admin:logout")) + elif url_name == "index": # url needed in the middleware only for admin panel login redirection continue else: response = web_client.get(reverse(url_name)) - self.assertNotEqual(response.status_code, 503, 'Whitelisted URL is not available.') + self.assertNotEqual(response.status_code, 503, "Whitelisted URL is not available.") def test_maintenance_false(self): web_client = Client() @@ -630,9 +547,9 @@ def test_maintenance_false(self): config.save() # post not whitelisted URL as superuser - response = web_client.get('/') + response = web_client.get("/") - self.assertNotEqual(response.status_code, 503, 'User is allowed to get index page') + self.assertNotEqual(response.status_code, 503, "User is allowed to get index page") def test_maintenance_true(self): web_client = Client() @@ -644,38 +561,37 @@ def test_maintenance_true(self): config.save() # post not whitelisted URL as superuser - response = web_client.get('/') + response = web_client.get("/") - self.assertEqual(response.status_code, 503, 'User is allowed to get index page') + self.assertEqual(response.status_code, 503, "User is allowed to get index page") class TestOwnerRightsRequestUtils(TestCase): - def setUp(self): User = get_user_model() - self.user = User.objects.create(username='test', email='test@test.com') - self.admin = User.objects.create(username='admin', email='test@test.com', is_superuser=True) - self.d = Document.objects.create(uuid=str(uuid4()), owner=self.user, title='test', is_approved=True) - self.la = Dataset.objects.create(uuid=str(uuid4()), owner=self.user, title='test', is_approved=True) - self.s = Service.objects.create(uuid=str(uuid4()), owner=self.user, title='test', is_approved=True) - self.m = Map.objects.create(uuid=str(uuid4()), owner=self.user, title='test', is_approved=True) + self.user = User.objects.create(username="test", email="test@test.com") + self.admin = User.objects.create(username="admin", email="test@test.com", is_superuser=True) + self.d = Document.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.la = Dataset.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.s = Service.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.m = Map.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) def test_get_concrete_resource(self): - self.assertTrue(isinstance( - OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.d.id)), Document - )) + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.d.id)), Document) + ) - self.assertTrue(isinstance( - OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.la.id)), Dataset - )) + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.la.id)), Dataset) + ) - self.assertTrue(isinstance( - OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.s.id)), Service - )) + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.s.id)), Service) + ) - self.assertTrue(isinstance( - OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.m.id)), Map - )) + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.m.id)), Map) + ) @override_settings(ADMIN_MODERATE_UPLOADS=True) def test_msg_recipients_admin_mode(self): @@ -703,10 +619,9 @@ def test_display_change_perms_button_tag_standard(self): class TestGetVisibleResource(TestCase): - def setUp(self): - self.user = get_user_model().objects.create(username='mikel_arteta') - self.category = TopicCategory.objects.create(identifier='biota') + self.user = get_user_model().objects.create(username="mikel_arteta") + self.category = TopicCategory.objects.create(identifier="biota") self.rb = ResourceBase.objects.create(uuid=str(uuid4()), category=self.category, owner=self.user) def test_category_data_not_shown_for_missing_resourcebase_permissions(self): @@ -715,23 +630,23 @@ def test_category_data_not_shown_for_missing_resourcebase_permissions(self): ISO category format data of the ISO category """ categories = get_visibile_resources(self.user) - self.assertEqual(categories['iso_formats'].count(), 0) + self.assertEqual(categories["iso_formats"].count(), 0) def test_category_data_shown_for_with_resourcebase_permissions(self): """ Test that a user with view permissions of a resource base can see ISO format data of the ISO category """ - assign_perm('view_resourcebase', self.user, self.rb) + assign_perm("view_resourcebase", self.user, self.rb) categories = get_visibile_resources(self.user) - self.assertEqual(categories['iso_formats'].count(), 1) + self.assertEqual(categories["iso_formats"].count(), 1) def test_visible_notifications(self): """ Test that a standard user won't be able to show ADMINS_ONLY_NOTICE_TYPES """ - self.assertFalse(show_notification('monitoring_alert', self.user)) - self.assertTrue(show_notification('request_download_resourcebase', self.user)) + self.assertFalse(show_notification("monitoring_alert", self.user)) + self.assertTrue(show_notification("request_download_resourcebase", self.user)) def test_extent_filter_crossing_dateline(self): from .bbox_utils import filter_bbox @@ -742,16 +657,18 @@ def test_extent_filter_crossing_dateline(self): _ll = Dataset.objects.create( uuid=str(uuid4()), owner=self.user, - name='test_extent_filter_crossing_dateline', - title='test_extent_filter_crossing_dateline', - alternate='geonode:test_extent_filter_crossing_dateline', + name="test_extent_filter_crossing_dateline", + title="test_extent_filter_crossing_dateline", + alternate="geonode:test_extent_filter_crossing_dateline", is_approved=True, is_published=True, - ll_bbox_polygon=Polygon.from_bbox(bbox) + ll_bbox_polygon=Polygon.from_bbox(bbox), ) self.assertListEqual(list(_ll.ll_bbox_polygon.extent), bbox, _ll.ll_bbox_polygon.extent) self.assertTrue(Dataset.objects.filter(title=_ll.title).exists(), Dataset.objects.all()) - _qs = filter_bbox(Dataset.objects.all(), '-180.0000,-39.7790,-164.2456,9.2702,134.0552,-39.7790,180.0000,9.2702') + _qs = filter_bbox( + Dataset.objects.all(), "-180.0000,-39.7790,-164.2456,9.2702,134.0552,-39.7790,180.0000,9.2702" + ) self.assertTrue(_qs.filter(title=_ll.title), Dataset.objects.all() | _qs.all()) finally: if _ll: @@ -759,7 +676,6 @@ def test_extent_filter_crossing_dateline(self): class TestHtmlTagRemoval(SimpleTestCase): - def test_not_tags_in_attribute(self): attribute_target_value = "This is not a templated text" r = ResourceBase() @@ -799,9 +715,7 @@ def test_converted_html_in_tags_with_with_multiple_tags(self): class TestTagThesaurus(TestCase): # loading test thesausurs - fixtures = [ - "test_thesaurus.json" - ] + fixtures = ["test_thesaurus.json"] def setUp(self): self.sut = Thesaurus( @@ -830,34 +744,34 @@ def test_get_thesaurus_date(self): def test_get_name_translation_raise_exception_if_identifier_does_not_exists(self): with self.assertRaises(ObjectDoesNotExist): - get_name_translation('foo_indentifier') + get_name_translation("foo_indentifier") - @patch('geonode.base.templatetags.thesaurus.get_language') + @patch("geonode.base.templatetags.thesaurus.get_language") def test_get_name_translation_return_thesauro_title_if_label_for_selected_language_does_not_exists(self, lang): - lang.return_value = 'ke' - actual = get_name_translation('inspire-theme') + lang.return_value = "ke" + actual = get_name_translation("inspire-theme") expected = "GEMET - INSPIRE themes, version 1.0" self.assertEqual(expected, actual) - @patch('geonode.base.templatetags.thesaurus.get_language') + @patch("geonode.base.templatetags.thesaurus.get_language") def test_get_thesaurus_translation_by_id(self, lang): - lang.return_value = 'it' + lang.return_value = "it" actual = get_thesaurus_translation_by_id(1) expected = "Tema GEMET - INSPIRE, versione 1.0" self.assertEqual(expected, actual) - @patch('geonode.base.templatetags.thesaurus.get_language') + @patch("geonode.base.templatetags.thesaurus.get_language") def test_get_thesaurus_localized_label(self, lang): - lang.return_value = 'de' + lang.return_value = "de" keyword = ThesaurusKeyword.objects.get(id=1) actual = get_thesaurus_localized_label(keyword) expected = "Adressen" self.assertEqual(expected, actual) - @patch('geonode.base.templatetags.thesaurus.get_language') + @patch("geonode.base.templatetags.thesaurus.get_language") def test_get_name_translation_return_label_title_if_label_for_selected_language_exists(self, lang): - lang.return_value = 'it' - actual = get_name_translation('inspire-theme') + lang.return_value = "it" + actual = get_name_translation("inspire-theme") expected = "Tema GEMET - INSPIRE, versione 1.0" self.assertEqual(expected, actual) @@ -870,9 +784,7 @@ def __get_last_thesaurus(): class TestThesaurusAvailableForm(TestCase): # loading test thesausurs - fixtures = [ - "test_thesaurus.json" - ] + fixtures = ["test_thesaurus.json"] def setUp(self): self.sut = ThesaurusAvailableForm @@ -893,35 +805,35 @@ def test_form_is_valid_if_fileds_send_expected_values(self): def test_field_class_treq_is_correctly_set_when_field_is_required(self): actual = self.sut(data={"1": 1}) - required = actual.fields.get('1') - obj_class = required.widget.attrs.get('class') - self.assertTrue(obj_class == 'treq') + required = actual.fields.get("1") + obj_class = required.widget.attrs.get("class") + self.assertTrue(obj_class == "treq") def test_field_class_treq_is_not_set_when_field_is_optional(self): actual = self.sut(data={"1": 1}) - required = actual.fields.get('2') - obj_class = required.widget.attrs.get('class') - self.assertTrue(obj_class == '') + required = actual.fields.get("2") + obj_class = required.widget.attrs.get("class") + self.assertTrue(obj_class == "") def test_will_return_thesaurus_with_the_expected_defined_order(self): actual = self.sut(data={"1": 1}) fields = list(actual.fields.items()) # will check if the first element of the tuple is the thesaurus_id = 2 - self.assertEqual(fields[0][0], '2') + self.assertEqual(fields[0][0], "2") # will check if the second element of the tuple is the thesaurus_id = 1 - self.assertEqual(fields[1][0], '1') + self.assertEqual(fields[1][0], "1") def test_will_return_thesaurus_with_the_defaul_order_as_0(self): # Update thesaurus order to 0 in order to check if the default order by id is observed - t = Thesaurus.objects.get(identifier='inspire-theme') + t = Thesaurus.objects.get(identifier="inspire-theme") t.order = 0 t.save() actual = ThesaurusAvailableForm(data={"1": 1}) fields = list(actual.fields.items()) # will check if the first element of the tuple is the thesaurus_id = 2 - self.assertEqual(fields[0][0], '1') + self.assertEqual(fields[0][0], "1") # will check if the second element of the tuple is the thesaurus_id = 1 - self.assertEqual(fields[1][0], '2') + self.assertEqual(fields[1][0], "2") def test_get_thesuro_key_label_with_cmd_language_code(self): # in python test language code look like 'en' this test checks if key label result function @@ -943,79 +855,82 @@ def test_get_thesuro_key_label_with_browser_language_code(self): class TestFacets(TestCase): - def setUp(self): - self.user = get_user_model().objects.create(username='test', email='test@test.com') + self.user = get_user_model().objects.create(username="test", email="test@test.com") Dataset.objects.update_or_create( - name='test_boxes_vector', + name="test_boxes_vector", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_boxes', - abstract='nothing', - subtype='vector', - is_approved=True)) + title="test_boxes", + abstract="nothing", + subtype="vector", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_1_vector', + name="test_1_vector", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_1', - abstract='contains boxes', - subtype='vector', - is_approved=True)) + title="test_1", + abstract="contains boxes", + subtype="vector", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_2_vector', + name="test_2_vector", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_2', - purpose='contains boxes', - subtype='vector', - is_approved=True)) + title="test_2", + purpose="contains boxes", + subtype="vector", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_3_vector', - defaults=dict( - uuid=str(uuid4()), - owner=self.user, - title='test_3', - subtype='vector', - is_approved=True)) + name="test_3_vector", + defaults=dict(uuid=str(uuid4()), owner=self.user, title="test_3", subtype="vector", is_approved=True), + ) Dataset.objects.update_or_create( - name='test_boxes_vector', + name="test_boxes_vector", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_boxes', - abstract='nothing', - subtype='vector', - is_approved=True)) + title="test_boxes", + abstract="nothing", + subtype="vector", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_1_raster', + name="test_1_raster", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_1', - abstract='contains boxes', - subtype='raster', - is_approved=True)) + title="test_1", + abstract="contains boxes", + subtype="raster", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_2_raster', + name="test_2_raster", defaults=dict( uuid=str(uuid4()), owner=self.user, - title='test_2', - purpose='contains boxes', - subtype='raster', - is_approved=True)) + title="test_2", + purpose="contains boxes", + subtype="raster", + is_approved=True, + ), + ) Dataset.objects.update_or_create( - name='test_boxes_raster', - defaults=dict( - uuid=str(uuid4()), - owner=self.user, - title='test_boxes', - subtype='raster', - is_approved=True)) + name="test_boxes_raster", + defaults=dict(uuid=str(uuid4()), owner=self.user, title="test_boxes", subtype="raster", is_approved=True), + ) self.request_mock = Mock(spec=requests.Request, GET=Mock()) @@ -1025,31 +940,29 @@ def test_facets_filter_datasets_returns_correctly(self): _l.clear_dirty_state() _l.set_processing_state(enumerations.STATE_PROCESSED) self.request_mock.GET.get.side_effect = lambda key, self: { - 'title__icontains': 'boxes', - 'abstract__icontains': 'boxes', - 'purpose__icontains': 'boxes', - 'date__gte': None, - 'date__range': None, - 'date__lte': None, - 'extent': None + "title__icontains": "boxes", + "abstract__icontains": "boxes", + "purpose__icontains": "boxes", + "date__gte": None, + "date__range": None, + "date__lte": None, + "extent": None, }.get(key) self.request_mock.GET.getlist.return_value = None self.request_mock.user = self.user - results = facets({'request': self.request_mock}) - self.assertEqual(results['vector'], 3) - self.assertEqual(results['raster'], 3) + results = facets({"request": self.request_mock}) + self.assertEqual(results["vector"], 3) + self.assertEqual(results["raster"], 3) class TestGenerateThesaurusReference(TestCase): - fixtures = [ - "test_thesaurus.json" - ] + fixtures = ["test_thesaurus.json"] def setUp(self): self.site_url = settings.SITEURL if hasattr(settings, "SITEURL") else "http://localhost" - ''' + """ If the keyword.about does not exists, the url created will have a prefix and a specifier: as prefix: - use the Keyword's thesaurus.about URI if it exists, @@ -1058,27 +971,27 @@ def setUp(self): - we may use the ThesaurusKeyword.alt_label if it exists, otherwise its id So the final about field value will be composed as f'{prefix}#{specifier}' - ''' + """ def test_should_return_keyword_url(self): expected = "http://inspire.ec.europa.eu/theme/ad" keyword = ThesaurusKeyword.objects.get(id=1) actual = generate_thesaurus_reference(keyword) keyword.refresh_from_db() - ''' + """ Check if the expected about has been created and that the instance is correctly updated - ''' + """ self.assertEqual(expected, actual) self.assertEqual(expected, keyword.about) def test_should_return_as_url_thesaurus_about_and_keyword_alt_label(self): expected = "http://inspire.ec.europa.eu/theme#foo_keyword" - keyword = ThesaurusKeyword.objects.get(alt_label='foo_keyword') + keyword = ThesaurusKeyword.objects.get(alt_label="foo_keyword") actual = generate_thesaurus_reference(keyword) keyword.refresh_from_db() - ''' + """ Check if the expected about has been created and that the instance is correctly updated - ''' + """ self.assertEqual(expected, actual) self.assertEqual(expected, keyword.about) @@ -1087,9 +1000,9 @@ def test_should_return_as_url_thesaurus_about_and_keyword_id(self): keyword = ThesaurusKeyword.objects.get(id=37) actual = generate_thesaurus_reference(keyword) keyword.refresh_from_db() - ''' + """ Check if the expected about has been created and that the instance is correctly updated - ''' + """ self.assertEqual(expected, actual) self.assertEqual(expected, keyword.about) @@ -1098,9 +1011,9 @@ def test_should_return_as_url_site_url_and_keyword_label(self): keyword = ThesaurusKeyword.objects.get(id=39) actual = generate_thesaurus_reference(keyword) keyword.refresh_from_db() - ''' + """ Check if the expected about has been created and that the instance is correctly updated - ''' + """ self.assertEqual(expected, actual) self.assertEqual(expected, keyword.about) @@ -1109,17 +1022,15 @@ def test_should_return_as_url_site_url_and_keyword_id(self): keyword = ThesaurusKeyword.objects.get(id=38) actual = generate_thesaurus_reference(keyword) keyword.refresh_from_db() - ''' + """ Check if the expected about has been created and that the instance is correctly updated - ''' + """ self.assertEqual(expected, actual) self.assertEqual(expected, keyword.about) class TestHandleMetadataKeyword(TestCase): - fixtures = [ - "test_thesaurus.json" - ] + fixtures = ["test_thesaurus.json"] def setUp(self): self.keyword = [ @@ -1148,14 +1059,11 @@ def setUp(self): }, {"keywords": ["Global"], "thesaurus": {"date": None, "datetype": None, "title": None}, "type": "place"}, ] - self.dataset = create_single_dataset('keyword-handler') - self.sut = KeywordHandler( - instance=self.dataset, - keywords=self.keyword - ) + self.dataset = create_single_dataset("keyword-handler") + self.sut = KeywordHandler(instance=self.dataset, keywords=self.keyword) def test_return_empty_if_keywords_is_an_empty_list(self): - setattr(self.sut, 'keywords', []) + setattr(self.sut, "keywords", []) keyword, thesaurus_keyword = self.sut.handle_metadata_keywords() self.assertListEqual([], keyword) self.assertListEqual([], thesaurus_keyword) diff --git a/geonode/base/translation.py b/geonode/base/translation.py index 9c914afe8c7..ca171cc9fb0 100755 --- a/geonode/base/translation.py +++ b/geonode/base/translation.py @@ -18,28 +18,47 @@ ######################################################################### from modeltranslation.translator import translator, TranslationOptions -from geonode.base.models import (TopicCategory, SpatialRepresentationType, Region, - RestrictionCodeType, License, ResourceBase) +from geonode.base.models import ( + TopicCategory, + SpatialRepresentationType, + Region, + RestrictionCodeType, + License, + ResourceBase, +) class TopicCategoryTranslationOptions(TranslationOptions): - fields = ('description', 'gn_description',) + fields = ( + "description", + "gn_description", + ) class SpatialRepresentationTypeTranslationOptions(TranslationOptions): - fields = ('description', 'gn_description',) + fields = ( + "description", + "gn_description", + ) class RegionTranslationOptions(TranslationOptions): - fields = ('name',) + fields = ("name",) class RestrictionCodeTypeTranslationOptions(TranslationOptions): - fields = ('description', 'gn_description',) + fields = ( + "description", + "gn_description", + ) class LicenseTranslationOptions(TranslationOptions): - fields = ('name', 'description', 'license_text',) + fields = ( + "name", + "description", + "license_text", + ) translator.register(TopicCategory, TopicCategoryTranslationOptions) diff --git a/geonode/base/urls.py b/geonode/base/urls.py index 8eba1fb652b..76a22d6138a 100644 --- a/geonode/base/urls.py +++ b/geonode/base/urls.py @@ -26,52 +26,50 @@ OwnerRightsRequestView, ResourceBaseAutocomplete, HierarchicalKeywordAutocomplete, - ThesaurusKeywordLabelAutocomplete) + ThesaurusKeywordLabelAutocomplete, +) urlpatterns = [ url( - r'^autocomplete_response/$', + r"^autocomplete_response/$", ResourceBaseAutocomplete.as_view(), - name='autocomplete_base', + name="autocomplete_base", ), - url( - r'^autocomplete_region/$', + r"^autocomplete_region/$", RegionAutocomplete.as_view(), - name='autocomplete_region', + name="autocomplete_region", ), - url( - r'^autocomplete_hierachical_keyword/$', + r"^autocomplete_hierachical_keyword/$", HierarchicalKeywordAutocomplete.as_view(), - name='autocomplete_hierachical_keyword', + name="autocomplete_hierachical_keyword", ), - url( - r'^thesaurus_available', + r"^thesaurus_available", ThesaurusAvailable.as_view(), - name='thesaurus_available', + name="thesaurus_available", ), url( - r'^thesaurus_autocomplete/$', + r"^thesaurus_autocomplete/$", ThesaurusKeywordLabelAutocomplete.as_view(), - name='thesaurus_autocomplete', + name="thesaurus_autocomplete", ), url( - r'^datasets_autocomplete/$', + r"^datasets_autocomplete/$", DatasetsAutocomplete.as_view(), - name='datasets_autocomplete', + name="datasets_autocomplete", ), url( - r'^resource_rights/(?P\d+)$', + r"^resource_rights/(?P\d+)$", OwnerRightsRequestView.as_view(), - name='owner_rights_request', + name="owner_rights_request", ), url( - r'^resource_clone/?$', + r"^resource_clone/?$", resource_clone, - name='resource_clone', + name="resource_clone", ), - url(r'^', include('geonode.base.api.urls')), + url(r"^", include("geonode.base.api.urls")), ] diff --git a/geonode/base/utils.py b/geonode/base/utils.py index f899a731c7f..0afee6192e1 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -37,20 +37,40 @@ from geonode.layers.models import Dataset from geonode.base.models import ResourceBase, Link, Configuration from geonode.security.utils import AdvancedSecurityWorkflowManager -from geonode.thumbs.utils import ( - get_thumbs, - remove_thumb) +from geonode.thumbs.utils import get_thumbs, remove_thumb from geonode.utils import get_legend_url -logger = logging.getLogger('geonode.base.utils') - -_names = ['Zipped Shapefile', 'Zipped', 'Shapefile', 'GML 2.0', 'GML 3.1.1', 'CSV', - 'GeoJSON', 'Excel', 'Legend', 'GeoTIFF', 'GZIP', 'Original Dataset', - 'ESRI Shapefile', 'View in Google Earth', 'KML', 'KMZ', 'Atom', 'DIF', - 'Dublin Core', 'ebRIM', 'FGDC', 'ISO', 'ISO with XSL'] +logger = logging.getLogger("geonode.base.utils") + +_names = [ + "Zipped Shapefile", + "Zipped", + "Shapefile", + "GML 2.0", + "GML 3.1.1", + "CSV", + "GeoJSON", + "Excel", + "Legend", + "GeoTIFF", + "GZIP", + "Original Dataset", + "ESRI Shapefile", + "View in Google Earth", + "KML", + "KMZ", + "Atom", + "DIF", + "Dublin Core", + "ebRIM", + "FGDC", + "ISO", + "ISO with XSL", +] thumb_filename_regex = re.compile( - r"^(document|map|layer|dataset|geoapp)-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})-thumb-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})\.png$") + r"^(document|map|layer|dataset|geoapp)-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})-thumb-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})\.png$" +) def get_thumb_uuid(filename): @@ -99,35 +119,32 @@ def remove_duplicate_links(resource): if layer.default_style and not layer.get_legend_url(style_name=layer.default_style.name): Link.objects.update_or_create( resource=layer.resourcebase_ptr, - name='Legend', - extension='png', + name="Legend", + extension="png", url=get_legend_url(layer, layer.default_style.name), - mime='image/png', - link_type='image') + mime="image/png", + link_type="image", + ) def configuration_session_cache(session): CONFIG_CACHE_TIMEOUT_SEC = 60 - _config = session.get('config') + _config = session.get("config") _now = datetime.utcnow() - _dt = isoparse(_config.get('expiration')) if _config else _now + _dt = isoparse(_config.get("expiration")) if _config else _now if _config is None or _dt < _now: config = Configuration.load() _dt = _now + timedelta(seconds=CONFIG_CACHE_TIMEOUT_SEC) - cached_config = { - 'configuration': {}, - 'expiration': _dt.isoformat() - } + cached_config = {"configuration": {}, "expiration": _dt.isoformat()} - for field_name in ['read_only', 'maintenance']: - cached_config['configuration'][field_name] = getattr(config, field_name) + for field_name in ["read_only", "maintenance"]: + cached_config["configuration"][field_name] = getattr(config, field_name) - session['config'] = cached_config + session["config"] = cached_config class OwnerRightsRequestViewUtils: - @staticmethod def get_message_recipients(owner): User = get_user_model() @@ -136,6 +153,7 @@ def get_message_recipients(owner): allowed_users |= User.objects.filter(is_superuser=True).exclude(pk=owner.pk) try: from geonode.groups.models import GroupProfile + groups = owner.groups.all() obj_group_managers = [] for group in groups: @@ -164,16 +182,10 @@ def validate_extra_metadata(data, instance): # starting validation of extra metadata passed via JSON # if schema for metadata validation is not defined, an error is raised - resource_type = ( - instance.polymorphic_ctype.model - if instance.polymorphic_ctype - else instance.class_name.lower() - ) + resource_type = instance.polymorphic_ctype.model if instance.polymorphic_ctype else instance.class_name.lower() extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) if not extra_metadata_validation_schema: - raise ValidationError( - f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}" - ) + raise ValidationError(f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}") # starting json structure validation. The Field can contain multiple metadata try: if isinstance(data, str): @@ -192,7 +204,7 @@ def validate_extra_metadata(data, instance): def remove_country_from_languagecode(language: str): - """ Remove country code (us) from language name (en-us) + """Remove country code (us) from language name (en-us) >>> remove_country_from_lanugecode("en-us") 'en' """ diff --git a/geonode/base/views.py b/geonode/base/views.py index d3739496bfd..e31b788f744 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -52,30 +52,21 @@ from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_languagecode from geonode.base.forms import UserAndGroupPermissionsForm -from geonode.base.forms import ( - BatchEditForm, - OwnerRightsRequestForm -) -from geonode.base.models import ( - Region, - ResourceBase, - HierarchicalKeyword, - ThesaurusKeyword, - ThesaurusKeywordLabel -) +from geonode.base.forms import BatchEditForm, OwnerRightsRequestForm +from geonode.base.models import Region, ResourceBase, HierarchicalKeyword, ThesaurusKeyword, ThesaurusKeywordLabel logger = logging.getLogger(__name__) def get_url_for_app_model(model, model_class): - return reverse(f'admin:{model_class._meta.app_label}_{model}_changelist') + return reverse(f"admin:{model_class._meta.app_label}_{model}_changelist") # was: return f'/admin/{model_class._meta.app_label}/{model}/' def get_url_for_model(model): - url = f'admin:{model.lower()}s_{model.lower()}_changelist' - if model.lower() == 'dataset': - url = f'admin:layers_{model.lower()}_changelist' + url = f"admin:{model.lower()}s_{model.lower()}_changelist" + if model.lower() == "dataset": + url = f"admin:layers_{model.lower()}_changelist" return reverse(url) # was: f'/admin/{model.lower()}s/{model.lower()}/' @@ -84,108 +75,103 @@ def user_and_group_permission(request, model): if not request.user.is_superuser: raise PermissionDenied - model_mapper = { - "profile": get_user_model(), - "groupprofile": GroupProfile - } + model_mapper = {"profile": get_user_model(), "groupprofile": GroupProfile} model_class = model_mapper[model] ids = request.POST.get("ids") if "cancel" in request.POST or not ids: - return HttpResponseRedirect( - get_url_for_app_model(model, model_class)) + return HttpResponseRedirect(get_url_for_app_model(model, model_class)) users_usernames = None groups_names = None - if request.method == 'POST': + if request.method == "POST": form = UserAndGroupPermissionsForm(request.POST) ids = ids.split(",") - _message = '' + _message = "" _errors = False if form.is_valid(): - resources_names = form.cleaned_data.get('layers') - users_usernames = [user.username for user in model_class.objects.filter( - id__in=ids)] if model == 'profile' else None - groups_names = [group_profile.group.name for group_profile in model_class.objects.filter( - id__in=ids)] if model in ('group', 'groupprofile') else None - - if users_usernames and 'AnonymousUser' in users_usernames and \ - (not groups_names or 'anonymous' not in groups_names): + resources_names = form.cleaned_data.get("layers") + users_usernames = ( + [user.username for user in model_class.objects.filter(id__in=ids)] if model == "profile" else None + ) + groups_names = ( + [group_profile.group.name for group_profile in model_class.objects.filter(id__in=ids)] + if model in ("group", "groupprofile") + else None + ) + + if ( + users_usernames + and "AnonymousUser" in users_usernames + and (not groups_names or "anonymous" not in groups_names) + ): if not groups_names: groups_names = [] - groups_names.append('anonymous') - if groups_names and 'anonymous' in groups_names and \ - (not users_usernames or 'AnonymousUser' not in users_usernames): + groups_names.append("anonymous") + if ( + groups_names + and "anonymous" in groups_names + and (not users_usernames or "AnonymousUser" not in users_usernames) + ): if not users_usernames: users_usernames = [] - users_usernames.append('AnonymousUser') + users_usernames.append("AnonymousUser") - delete_flag = form.cleaned_data.get('mode') == 'unset' - permissions_names = form.cleaned_data.get('permission_type') + delete_flag = form.cleaned_data.get("mode") == "unset" + permissions_names = form.cleaned_data.get("permission_type") if permissions_names: - if 'edit' in permissions_names and 'AnonymousUser' in users_usernames: + if "edit" in permissions_names and "AnonymousUser" in users_usernames: if not _errors: _message = '"EDIT" permissions not allowed for the "AnonymousUser".' _errors = True else: set_permissions.apply_async( - ([permissions_names], resources_names, users_usernames, groups_names, delete_flag)) + ([permissions_names], resources_names, users_usernames, groups_names, delete_flag) + ) if not _errors: _message = f'The asyncronous permissions {form.cleaned_data.get("mode")} request for {", ".join(users_usernames or groups_names)} has been sent' else: if not _errors: - _message = 'No permissions have been set.' + _message = "No permissions have been set." _errors = True else: if not _errors: - _message = f'Some error has occured {form.errors}' + _message = f"Some error has occured {form.errors}" _errors = True - messages.add_message( - request, - (messages.INFO if not _errors else messages.ERROR), - _message - ) - return HttpResponseRedirect( - get_url_for_app_model(model, model_class)) + messages.add_message(request, (messages.INFO if not _errors else messages.ERROR), _message) + return HttpResponseRedirect(get_url_for_app_model(model, model_class)) - form = UserAndGroupPermissionsForm({ - 'permission_type': 'view', - 'mode': 'set', - }) - return render( - request, - "base/user_and_group_permissions.html", - context={ - "form": form, - "model": model + form = UserAndGroupPermissionsForm( + { + "permission_type": "view", + "mode": "set", } ) + return render(request, "base/user_and_group_permissions.html", context={"form": form, "model": model}) def batch_modify(request, model): if not request.user.is_superuser: raise PermissionDenied - if model == 'Document': + if model == "Document": Resource = Document - if model == 'Dataset': + if model == "Dataset": Resource = Dataset - if model == 'Map': + if model == "Map": Resource = Map - template = 'base/batch_edit.html' + template = "base/batch_edit.html" ids = request.POST.get("ids") if "cancel" in request.POST or not ids: - return HttpResponseRedirect( - get_url_for_model(model)) + return HttpResponseRedirect(get_url_for_model(model)) - if request.method == 'POST': + if request.method == "POST": form = BatchEditForm(request.POST) if form.is_valid(): - keywords = [keyword.strip() for keyword in - form.cleaned_data.pop("keywords").split(',') if keyword] + keywords = [keyword.strip() for keyword in form.cleaned_data.pop("keywords").split(",") if keyword] regions = form.cleaned_data.pop("regions") ids = form.cleaned_data.pop("ids") if not form.cleaned_data.get("date"): @@ -195,7 +181,7 @@ def batch_modify(request, model): for _key, _value in form.cleaned_data.items(): if _value: to_update[_key] = _value - resources = Resource.objects.filter(id__in=ids.split(',')) + resources = Resource.objects.filter(id__in=ids.split(",")) resources.update(**to_update) if regions: regions_through = Resource.regions.through @@ -211,25 +197,26 @@ def get_or_create(keyword): return HierarchicalKeyword.objects.get(name=keyword) except HierarchicalKeyword.DoesNotExist: return HierarchicalKeyword.add_root(name=keyword) + hierarchical_keyword = [get_or_create(keyword) for keyword in keywords] new_keywords = [] for keyword in hierarchical_keyword: - new_keywords += [keywords_through( - content_object=resource, tag_id=keyword.pk) for resource in resources] + new_keywords += [ + keywords_through(content_object=resource, tag_id=keyword.pk) for resource in resources + ] keywords_through.objects.bulk_create(new_keywords, ignore_conflicts=True) - return HttpResponseRedirect( - get_url_for_model(model)) + return HttpResponseRedirect(get_url_for_model(model)) return render( request, template, context={ - 'form': form, - 'ids': ids, - 'model': model, - } + "form": form, + "ids": ids, + "model": model, + }, ) form = BatchEditForm() @@ -237,27 +224,27 @@ def get_or_create(keyword): request, template, context={ - 'form': form, - 'ids': ids, - 'model': model, - } + "form": form, + "ids": ids, + "model": model, + }, ) class SimpleSelect2View(autocomplete.Select2QuerySetView): - """ Generic select2 view for autocompletes - Params: - model: model to perform the autocomplete query on - filter_arg: property to filter with ie. name__icontains + """Generic select2 view for autocompletes + Params: + model: model to perform the autocomplete query on + filter_arg: property to filter with ie. name__icontains """ def __init__(self, *args, **kwargs): super(views.BaseQuerySetView, self).__init__(*args, **kwargs) - if not hasattr(self, 'filter_arg'): + if not hasattr(self, "filter_arg"): raise AttributeError("SimpleSelect2View missing required 'filter_arg' argument") def get_queryset(self): - qs = super(views.BaseQuerySetView, self).get_queryset().order_by('pk') + qs = super(views.BaseQuerySetView, self).get_queryset().order_by("pk") if self.q: qs = qs.filter(**{self.filter_arg: self.q}) @@ -265,49 +252,46 @@ def get_queryset(self): class ResourceBaseAutocomplete(autocomplete.Select2QuerySetView): - """ Base resource autocomplete - searches all the resources by title - returns any visible resources in this queryset for autocomplete + """Base resource autocomplete - searches all the resources by title + returns any visible resources in this queryset for autocomplete """ def get_queryset(self): request = self.request - permitted = get_objects_for_user(request.user, 'base.view_resourcebase') + permitted = get_objects_for_user(request.user, "base.view_resourcebase") qs = ResourceBase.objects.all().filter(id__in=permitted) if self.q: - qs = qs.filter(title__icontains=self.q).order_by('title') + qs = qs.filter(title__icontains=self.q).order_by("title") return get_visible_resources( qs, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES)[:100] + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + )[:100] class RegionAutocomplete(SimpleSelect2View): model = Region - filter_arg = 'name__icontains' + filter_arg = "name__icontains" class HierarchicalKeywordAutocomplete(SimpleSelect2View): model = HierarchicalKeyword - filter_arg = 'slug__icontains' + filter_arg = "slug__icontains" class ThesaurusKeywordLabelAutocomplete(autocomplete.Select2QuerySetView): - def get_queryset(self): thesaurus = settings.THESAURUS - tname = thesaurus['name'] - lang = 'en' + tname = thesaurus["name"] + lang = "en" # Filters thesaurus results based on thesaurus name and language - qs = ThesaurusKeywordLabel.objects.all().filter( - keyword__thesaurus__identifier=tname, - lang=lang - ) + qs = ThesaurusKeywordLabel.objects.all().filter(keyword__thesaurus__identifier=tname, lang=lang) if self.q: qs = qs.filter(label__icontains=self.q) @@ -318,24 +302,26 @@ def get_queryset(self): def get_results(self, context): return [ { - 'id': self.get_result_value(result.keyword), - 'text': self.get_result_label(result), - 'selected_text': self.get_selected_result_label(result), - } for result in context['object_list'] + "id": self.get_result_value(result.keyword), + "text": self.get_result_label(result), + "selected_text": self.get_selected_result_label(result), + } + for result in context["object_list"] ] class DatasetsAutocomplete(SimpleSelect2View): model = Dataset - filter_arg = 'title__icontains' + filter_arg = "title__icontains" def get_results(self, context): return [ { - 'id': self.get_result_value(result), - 'text': self.get_result_label(result.title), - 'selected_text': self.get_selected_result_label(result.title), - } for result in context['object_list'] + "id": self.get_result_value(result), + "text": self.get_result_label(result.title), + "selected_text": self.get_selected_result_label(result.title), + } + for result in context["object_list"] ] @@ -347,12 +333,21 @@ def get_queryset(self): keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid) # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again - qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") if len(qs_keyword_ids) == 0: lang = remove_country_from_languagecode(lang) - qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") - - not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + + not_qs_ids = ( + ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids) + .order_by("keyword_id") + .distinct("keyword_id") + .values("keyword_id") + ) qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus) if self.q: qs = qs.filter(label__istartswith=self.q) @@ -364,58 +359,62 @@ def get_queryset(self): def get_results(self, context): return [ { - 'id': str(result.keyword.pk) if isinstance(result, ThesaurusKeywordLabel) else str(result.pk), - 'text': self.get_result_label(result), - 'selected_text': self.get_selected_result_label(result), - } for result in context['object_list'] + "id": str(result.keyword.pk) if isinstance(result, ThesaurusKeywordLabel) else str(result.pk), + "text": self.get_result_label(result), + "selected_text": self.get_selected_result_label(result), + } + for result in context["object_list"] ] class OwnerRightsRequestView(LoginRequiredMixin, FormView): - template_name = 'owner_rights_request.html' + template_name = "owner_rights_request.html" form_class = OwnerRightsRequestForm resource = None - redirect_field_name = 'next' + redirect_field_name = "next" def get_success_url(self): return self.resource.get_absolute_url() def get(self, request, *args, **kwargs): - r_base = ResourceBase.objects.get(pk=kwargs.get('pk')) + r_base = ResourceBase.objects.get(pk=kwargs.get("pk")) self.resource = OwnerRightsRequestViewUtils.get_resource(r_base) - initial = { - 'resource': r_base - } + initial = {"resource": r_base} form = self.form_class(initial=initial) - return render(request, self.template_name, {'form': form, 'resource': self.resource}) + return render(request, self.template_name, {"form": form, "resource": self.resource}) def post(self, request, *args, **kwargs): - r_base = ResourceBase.objects.get(pk=kwargs.get('pk')) + r_base = ResourceBase.objects.get(pk=kwargs.get("pk")) self.resource = OwnerRightsRequestViewUtils.get_resource(r_base) form = self.form_class(request.POST) if form.is_valid(): - reason = form.cleaned_data['reason'] - notice_type_label = 'request_resource_edit' + reason = form.cleaned_data["reason"] + notice_type_label = "request_resource_edit" recipients = OwnerRightsRequestViewUtils.get_message_recipients(self.resource.owner) Message.objects.new_message( from_user=request.user, to_users=recipients, - subject=_('System message: A request to modify resource'), - content=_('The resource owner has requested to modify the resource') + '.\n' - ' ' + - _('Resource title') + ': ' + self.resource.title + '.\n' - ' ' + - _('Reason for the request') + ': "' + reason + '".\n' + - ' ' + - _('To allow the change, set the resource to not "Approved" under the metadata settings' + - 'and write message to the owner to notify him') + '.' + subject=_("System message: A request to modify resource"), + content=_("The resource owner has requested to modify the resource") + ".\n" + " " + _("Resource title") + ": " + self.resource.title + ".\n" + " " + + _("Reason for the request") + + ': "' + + reason + + '".\n' + + " " + + _( + 'To allow the change, set the resource to not "Approved" under the metadata settings' + + "and write message to the owner to notify him" + ) + + ".", + ) + send_notification( + recipients, + notice_type_label, + {"resource": self.resource, "site_url": settings.SITEURL[:-1], "reason": reason}, ) - send_notification(recipients, notice_type_label, { - 'resource': self.resource, - 'site_url': settings.SITEURL[:-1], - 'reason': reason - }) return self.form_valid(form) else: return self.form_invalid(form) @@ -424,10 +423,8 @@ def post(self, request, *args, **kwargs): @login_required def resource_clone(request): try: - uuid = request.POST['uuid'] - resource = resolve_object( - request, ResourceBase, { - 'uuid': uuid}, 'base.change_resourcebase') + uuid = request.POST["uuid"] + resource = resolve_object(request, ResourceBase, {"uuid": uuid}, "base.change_resourcebase") except PermissionDenied: return HttpResponse("Not allowed", status=403) except Exception: @@ -437,26 +434,19 @@ def resource_clone(request): out = {} try: - getattr(resource_manager, "copy")( - resource.get_real_instance(), - uuid=None, - defaults={ - 'user': request.user}) - out['success'] = True - out['message'] = _("Resource Cloned Successfully!") + getattr(resource_manager, "copy")(resource.get_real_instance(), uuid=None, defaults={"user": request.user}) + out["success"] = True + out["message"] = _("Resource Cloned Successfully!") except Exception as e: logger.exception(e) - out['success'] = False - out['message'] = _(f"Error Occurred while Cloning the Resource: {e}") - out['errors'] = str(e) + out["success"] = False + out["message"] = _(f"Error Occurred while Cloning the Resource: {e}") + out["errors"] = str(e) - if out['success']: + if out["success"]: status_code = 200 - register_event(request, 'change', resource) + register_event(request, "change", resource) else: status_code = 400 - return HttpResponse( - json.dumps(out), - content_type='application/json', - status=status_code) + return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) diff --git a/geonode/base/widgets.py b/geonode/base/widgets.py index 08e41930e23..a10395ab485 100644 --- a/geonode/base/widgets.py +++ b/geonode/base/widgets.py @@ -3,7 +3,7 @@ class TaggitSelect2Custom(TaggitSelect2): """Overriding Select2 tag widget for taggit's TagField. - Fixes error in tests where 'value' is None. + Fixes error in tests where 'value' is None. """ def value_from_datadict(self, data, files, name): @@ -15,8 +15,8 @@ def value_from_datadict(self, data, files, name): try: value = super().value_from_datadict(data, files, name) - if value and ',' not in value: - value = f'{value},' + if value and "," not in value: + value = f"{value}," return value except TypeError: return "" diff --git a/geonode/br/__init__.py b/geonode/br/__init__.py index 5094cf7ca78..df3f634b661 100644 --- a/geonode/br/__init__.py +++ b/geonode/br/__init__.py @@ -21,8 +21,8 @@ class BackupRestoreAppConfig(AppConfig): - name = 'geonode.br' - verbose_name = 'Backup/Restore' + name = "geonode.br" + verbose_name = "Backup/Restore" -default_app_config = 'geonode.br.BackupRestoreAppConfig' +default_app_config = "geonode.br.BackupRestoreAppConfig" diff --git a/geonode/br/admin.py b/geonode/br/admin.py index 97c222d35dc..056364f303a 100644 --- a/geonode/br/admin.py +++ b/geonode/br/admin.py @@ -23,9 +23,9 @@ @admin.register(RestoredBackup) class RestoredBackupAdmin(admin.ModelAdmin): - readonly_fields = ('name', 'restoration_date', 'archive_md5', 'creation_date') + readonly_fields = ("name", "restoration_date", "archive_md5", "creation_date") actions = None - list_display = ('name', 'restoration_date', 'archive_md5', 'creation_date') + list_display = ("name", "restoration_date", "archive_md5", "creation_date") def has_add_permission(self, request): return False diff --git a/geonode/br/tasks.py b/geonode/br/tasks.py index 0a3f0a9a92e..1d496e99595 100644 --- a/geonode/br/tasks.py +++ b/geonode/br/tasks.py @@ -27,28 +27,33 @@ @app.task( bind=True, - name='geonode.br.tasks.restore_notification', - queue='email', + name="geonode.br.tasks.restore_notification", + queue="email", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def restore_notification(recipients: List, backup_file: str, backup_md5: str, exception: str = None): """ Function sending a CC email report of the restore procedure to a provided emails. """ if exception: - subject = 'Geonode restore procedure FAILED.' - message = f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' \ - f'GeoNode instance: {settings.SITEURL} FAILED with an exception: {exception}' + subject = "Geonode restore procedure FAILED." + message = ( + f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' + f"GeoNode instance: {settings.SITEURL} FAILED with an exception: {exception}" + ) else: - subject = 'Geonode restore procedure finished with SUCCESS.' - message = f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' \ - f'GeoNode instance: {settings.SITEURL} was finished SUCCESSFULLY.' + subject = "Geonode restore procedure finished with SUCCESS." + message = ( + f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' + f"GeoNode instance: {settings.SITEURL} was finished SUCCESSFULLY." + ) msg = EmailMessage(subject=subject, body=message, to=recipients) msg.send() diff --git a/geonode/br/tests/__init__.py b/geonode/br/tests/__init__.py index 7fb28c2fa8c..36ea6cf5db6 100644 --- a/geonode/br/tests/__init__.py +++ b/geonode/br/tests/__init__.py @@ -17,6 +17,6 @@ # ######################################################################### -from geonode.br.tests.test_restore import * # noqa aside -from geonode.br.tests.test_restore_helpers import * # noqa aside -from geonode.br.tests.test_backup import * # noqa aside +from geonode.br.tests.test_restore import * # noqa aside +from geonode.br.tests.test_restore_helpers import * # noqa aside +from geonode.br.tests.test_backup import * # noqa aside diff --git a/geonode/br/tests/factories.py b/geonode/br/tests/factories.py index 87f13cc7af5..cabf772ab76 100644 --- a/geonode/br/tests/factories.py +++ b/geonode/br/tests/factories.py @@ -33,9 +33,7 @@ def random_md5_hash() -> str: :return: hex representation of md5 hash of a random string """ return hashlib.md5( - ''.join( - random.choices(string.ascii_uppercase + string.digits, k=15) - ).encode('utf-8') + "".join(random.choices(string.ascii_uppercase + string.digits, k=15)).encode("utf-8") ).hexdigest() @@ -43,7 +41,7 @@ class RestoredBackupFactory(DjangoModelFactory): class Meta: model = RestoredBackup - name = factory.Faker('word') + name = factory.Faker("word") archive_md5 = factory.LazyFunction(random_md5_hash) restoration_date = factory.LazyFunction(datetime.now) - creation_date = factory.Faker('date_time') + creation_date = factory.Faker("date_time") diff --git a/geonode/br/tests/test_backup.py b/geonode/br/tests/test_backup.py index 3867ebf4750..e175c7d563d 100644 --- a/geonode/br/tests/test_backup.py +++ b/geonode/br/tests/test_backup.py @@ -29,86 +29,77 @@ class BackupCommandTests(GeoNodeBaseTestSupport): - def setUp(self): super().setUp() # make sure Configuration exists in the database for Read Only mode tests Configuration.load() # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_with_read_only_mode(self, mock_configuration_save, fake_confirm): with tempfile.TemporaryDirectory() as tmp_dir: args = [] kwargs = { - 'backup_dir': tmp_dir, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_dir": tmp_dir, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } - call_command('backup', *args, **kwargs) + call_command("backup", *args, **kwargs) # make sure Configuration was saved twice (Read-Only set, and revert) self.assertEqual(mock_configuration_save.call_count, 2) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_without_read_only_mode(self, mock_configuration_save, fake_confirm): with tempfile.TemporaryDirectory() as tmp_dir: - args = ['--skip-read-only'] + args = ["--skip-read-only"] kwargs = { - 'backup_dir': tmp_dir, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_dir": tmp_dir, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } - call_command('backup', *args, **kwargs) + call_command("backup", *args, **kwargs) # make sure Configuration wasn't called at all self.assertEqual(mock_configuration_save.call_count, 0) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_config_file_not_provided(self, mock_configuration_save, fake_confirm): with tempfile.TemporaryDirectory() as tmp_dir: - args = ['--skip-read-only'] - kwargs = {'backup_dir': tmp_dir} + args = ["--skip-read-only"] + kwargs = {"backup_dir": tmp_dir} with self.assertRaises(CommandError) as exc: - call_command('backup', *args, **kwargs) + call_command("backup", *args, **kwargs) self.assertIn( - 'Mandatory option (-c / --config)', + "Mandatory option (-c / --config)", exc.exception.args[0], - '"Mandatory option (-c / --config)" exception expected.' + '"Mandatory option (-c / --config)" exception expected.', ) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_config_file_does_not_exist(self, mock_configuration_save, fake_confirm): with tempfile.TemporaryDirectory() as tmp_dir: - args = ['--skip-read-only'] - kwargs = {'backup_dir': tmp_dir, 'config': '/some/random/path'} + args = ["--skip-read-only"] + kwargs = {"backup_dir": tmp_dir, "config": "/some/random/path"} with self.assertRaises(CommandError) as exc: - call_command('backup', *args, **kwargs) + call_command("backup", *args, **kwargs) - self.assertIn( - "file does not exist", - exc.exception.args[0], - '"file does not exist" exception expected.' - ) + self.assertIn("file does not exist", exc.exception.args[0], '"file does not exist" exception expected.') diff --git a/geonode/br/tests/test_restore.py b/geonode/br/tests/test_restore.py index d31fadc3dfa..5d750ed7d10 100644 --- a/geonode/br/tests/test_restore.py +++ b/geonode/br/tests/test_restore.py @@ -32,174 +32,152 @@ class RestoreCommandTests(GeoNodeBaseTestSupport): - def setUp(self): super().setUp() # make sure Configuration exists in the database for Read Only mode tests Configuration.load() # force restore interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) def test_with_logs_success(self, fake_confirm): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") # create an entry in restoration history with the same dump's hash - RestoredBackupFactory(archive_md5='91162629d258a876ee994e9233b2ad87') + RestoredBackupFactory(archive_md5="91162629d258a876ee994e9233b2ad87") - args = ['-l'] + args = ["-l"] kwargs = { - 'backup_file': tmp_file.name, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) # force restore interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) def test_with_logs_failure(self, fake_confirm): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") # create an entry in restoration history with the same dump's hash RestoredBackupFactory(archive_md5=md5_file_hash(tmp_file.name)) - args = ['-l'] + args = ["-l"] kwargs = { - 'backup_file': tmp_file.name, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } with self.assertRaises(RuntimeError) as exc: - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) self.assertIn( - 'Backup archive has already been restored', + "Backup archive has already been restored", exc.exception.args[0], - '"Backup archive has already been restored" exception expected.' + '"Backup archive has already been restored" exception expected.', ) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_with_read_only_mode(self, mock_configuration_save, fake_confirm): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") args = [] kwargs = { - 'backup_file': tmp_file.name, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) # make sure Configuration was saved twice (Read-Only set, and revert) self.assertEqual(mock_configuration_save.call_count, 2) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_without_read_only_mode(self, mock_configuration_save, fake_confirm): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") - args = ['--skip-read-only'] + args = ["--skip-read-only"] kwargs = { - 'backup_file': tmp_file.name, - 'config': os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..', - 'management/commands/settings_sample.ini' - ) + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), } - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) # make sure Configuration wasn't called at all self.assertEqual(mock_configuration_save.call_count, 0) # force backup interruption before starting the procedure itself - @mock.patch('geonode.br.management.commands.utils.utils.confirm', return_value=False) + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) # mock geonode.base.models.Configuration save() method - @mock.patch('geonode.base.models.Configuration.save', return_value=None) + @mock.patch("geonode.base.models.Configuration.save", return_value=None) def test_config_files(self, mock_configuration_save, fake_confirm): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") - args = ['--skip-read-only'] - kwargs = { - 'backup_file': tmp_file.name - } + args = ["--skip-read-only"] + kwargs = {"backup_file": tmp_file.name} with self.assertRaises(CommandError) as exc: - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) self.assertIn( - 'Mandatory option (-c / --config)', + "Mandatory option (-c / --config)", exc.exception.args[0], - '"Mandatory option (-c / --config)" exception expected.' + '"Mandatory option (-c / --config)" exception expected.', ) # create the backup file with ini file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") tmp_ini_file = f"{tmp_file.name.rsplit('.', 1)[0]}.ini" from configparser import ConfigParser + config = ConfigParser() - config['database'] = { - 'pgdump': 'pg_dump', - 'pgrestore': 'pg_restore' - } + config["database"] = {"pgdump": "pg_dump", "pgrestore": "pg_restore"} - config['geoserver'] = { - 'datadir': 'geoserver/data', - 'dumpvectordata': 'yes', - 'dumprasterdata': 'yes' - } + config["geoserver"] = {"datadir": "geoserver/data", "dumpvectordata": "yes", "dumprasterdata": "yes"} - config['fixtures'] = { - 'apps': 'fake', - 'dumps': 'fake' - } + config["fixtures"] = {"apps": "fake", "dumps": "fake"} - with open(tmp_ini_file, 'w') as configfile: + with open(tmp_ini_file, "w") as configfile: config.write(configfile) - args = ['--skip-read-only'] - kwargs = { - 'backup_file': tmp_file.name - } + args = ["--skip-read-only"] + kwargs = {"backup_file": tmp_file.name} - call_command('restore', *args, **kwargs) + call_command("restore", *args, **kwargs) diff --git a/geonode/br/tests/test_restore_helpers.py b/geonode/br/tests/test_restore_helpers.py index c5fbb09d061..e29a32a9fed 100644 --- a/geonode/br/tests/test_restore_helpers.py +++ b/geonode/br/tests/test_restore_helpers.py @@ -42,37 +42,26 @@ def test_mandatory_option_failure(self): with self.assertRaises(CommandError) as exc: RestoreCommand().validate_backup_file_options(**options) - self.assertIn( - 'Mandatory option', - exc.exception.args[0], - '"Mandatory option" failure exception expected.' - ) + self.assertIn("Mandatory option", exc.exception.args[0], '"Mandatory option" failure exception expected.') # validate_backup_file_options() method test def test_exclusive_option_failure(self): - options = { - 'backup_file': '/some/random/path/1.zip', - 'backup_files_dir': '/some/random/path' - } + options = {"backup_file": "/some/random/path/1.zip", "backup_files_dir": "/some/random/path"} with self.assertRaises(CommandError) as exc: RestoreCommand().validate_backup_file_options(**options) - self.assertIn( - 'Exclusive option', - exc.exception.args[0], - '"Exclusive option" failure exception expected.' - ) + self.assertIn("Exclusive option", exc.exception.args[0], '"Exclusive option" failure exception expected.') # validate_backup_file_options() method test def test_mandatory_option_backup_file_success(self): with tempfile.NamedTemporaryFile() as tmp: - with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") - options = {'backup_file': tmp.name} + options = {"backup_file": tmp.name} RestoreCommand().validate_backup_file_options(**options) # validate_backup_file_options() method test @@ -80,22 +69,18 @@ def test_mandatory_option_backup_file_failure(self): with tempfile.NamedTemporaryFile() as tmp: - options = {'backup_file': tmp.name} + options = {"backup_file": tmp.name} with self.assertRaises(CommandError) as exc: RestoreCommand().validate_backup_file_options(**options) - self.assertIn( - 'not a .zip file', - exc.exception.args[0], - '"not a .zip file" failure exception expected.' - ) + self.assertIn("not a .zip file", exc.exception.args[0], '"not a .zip file" failure exception expected.') # validate_backup_file_options() method test def test_mandatory_option_backup_files_dir_success(self): with tempfile.TemporaryDirectory() as tmp: - options = {'backup_files_dir': tmp} + options = {"backup_files_dir": tmp} RestoreCommand().validate_backup_file_options(**options) # validate_backup_file_options() method test @@ -103,16 +88,12 @@ def test_mandatory_option_backup_files_dir_failure(self): with tempfile.NamedTemporaryFile() as tmp: - options = {'backup_files_dir': tmp.name} + options = {"backup_files_dir": tmp.name} with self.assertRaises(CommandError) as exc: RestoreCommand().validate_backup_file_options(**options) - self.assertIn( - 'not a directory', - exc.exception.args[0], - '"not a directory" failure exception expected.' - ) + self.assertIn("not a directory", exc.exception.args[0], '"not a directory" failure exception expected.') # parse_backup_files_dir() method test def test_backup_files_dir_no_archive(self): @@ -120,7 +101,7 @@ def test_backup_files_dir_no_archive(self): with tempfile.TemporaryDirectory() as tmp: backup_file = RestoreCommand().parse_backup_files_dir(tmp) - self.assertIsNone(backup_file, 'Expecting backup file to be None') + self.assertIsNone(backup_file, "Expecting backup file to be None") # parse_backup_files_dir() method test def test_backup_files_dir_multiple_archives(self): @@ -130,20 +111,20 @@ def test_backup_files_dir_multiple_archives(self): # create the 1st backup file with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file_1: - with zipfile.ZipFile(tmp_file_1, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something1.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file_1, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something1.txt", "Some Content Here") # make sure timestamps of files modification differ time.sleep(1) # create the 2nd backup file with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file_2: - with zipfile.ZipFile(tmp_file_2, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something2.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file_2, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something2.txt", "Some Content Here") backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) - self.assertIn(tmp_file_2.name, backup_file, 'Expected the younger file to be chosen.') + self.assertIn(tmp_file_2.name, backup_file, "Expected the younger file to be chosen.") # parse_backup_files_dir() method test def test_backup_files_dir_with_newer_restored_backup(self): @@ -156,12 +137,12 @@ def test_backup_files_dir_with_newer_restored_backup(self): # create the backup file with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) - self.assertIsNone(backup_file, 'Expected the backup not to be chosen.') + self.assertIsNone(backup_file, "Expected the backup not to be chosen.") # parse_backup_files_dir() method test def test_backup_files_dir_with_older_restored_backup(self): @@ -174,46 +155,46 @@ def test_backup_files_dir_with_older_restored_backup(self): # create the backup file with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) - self.assertIn(tmp_file.name, backup_file, 'Expected the backup to be chosen.') + self.assertIn(tmp_file.name, backup_file, "Expected the backup to be chosen.") # validate_backup_file_hash() method test def test_backup_hash_no_md5_file(self): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") file_hash = RestoreCommand().validate_backup_file_hash(tmp_file.name) - self.assertIsNotNone(file_hash, 'Expected the backup file MD5 hash to be returned.') + self.assertIsNotNone(file_hash, "Expected the backup file MD5 hash to be returned.") # validate_backup_file_hash() method test def test_backup_hash_failure(self): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") # create a md5 hash file for the backup temporary file tmp_hash_file = f"{tmp_file.name}.md5" - with open(tmp_hash_file, 'w') as hash_file: - hash_file.write('91162629d258a876ee994e9233b2ad87') + with open(tmp_hash_file, "w") as hash_file: + hash_file.write("91162629d258a876ee994e9233b2ad87") try: with self.assertRaises(RuntimeError) as exc: RestoreCommand().validate_backup_file_hash(tmp_file.name) self.assertIn( - 'Backup archive integrity failure', + "Backup archive integrity failure", exc.exception.args[0], - '"Backup archive integrity failure" exception expected.' + '"Backup archive integrity failure" exception expected.', ) finally: @@ -225,18 +206,18 @@ def test_backup_hash_success(self): # create the backup file with tempfile.NamedTemporaryFile() as tmp_file: - with zipfile.ZipFile(tmp_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.writestr('something.txt', 'Some Content Here') + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") # create a md5 hash file for the backup temporary file tmp_hash_file = f"{tmp_file.name}.md5" - with open(tmp_hash_file, 'w') as hash_file: + with open(tmp_hash_file, "w") as hash_file: hash_file.write(md5_file_hash(tmp_file.name)) try: file_hash = RestoreCommand().validate_backup_file_hash(tmp_file.name) - self.assertIsNotNone(file_hash, 'Expected the backup file MD5 hash to be returned.') + self.assertIsNotNone(file_hash, "Expected the backup file MD5 hash to be returned.") finally: # remove temporary hash file diff --git a/geonode/catalogue/__init__.py b/geonode/catalogue/__init__.py index 1e015e43390..3b577289766 100644 --- a/geonode/catalogue/__init__.py +++ b/geonode/catalogue/__init__.py @@ -25,11 +25,11 @@ from django.core.exceptions import ImproperlyConfigured from importlib import import_module -DEFAULT_CATALOGUE_ALIAS = 'default' +DEFAULT_CATALOGUE_ALIAS = "default" # GeoNode uses this if the CATALOGUE setting is empty (None). -if not hasattr(settings, 'CATALOGUE'): - settings.CATALOGUE = {DEFAULT_CATALOGUE_ALIAS: 'geonode.backends.dummy'} +if not hasattr(settings, "CATALOGUE"): + settings.CATALOGUE = {DEFAULT_CATALOGUE_ALIAS: "geonode.backends.dummy"} # If settings.CATALOGUE is defined, we expect it to be properly named if DEFAULT_CATALOGUE_ALIAS not in settings.CATALOGUE: @@ -44,18 +44,25 @@ def load_backend(backend_name): except ImportError as e_user: # The CSW backend wasn't found. Display a helpful error message # listing all possible (built-in) CSW backends. - backend_dir = os.path.join(os.path.dirname(__file__), 'backends') + backend_dir = os.path.join(os.path.dirname(__file__), "backends") try: - available_backends = sorted([f for f in os.listdir(backend_dir) if os.path.isdir(os.path.join(backend_dir, f)) and - not f.startswith('.')]) + available_backends = sorted( + [ + f + for f in os.listdir(backend_dir) + if os.path.isdir(os.path.join(backend_dir, f)) and not f.startswith(".") + ] + ) except OSError: available_backends = [] if backend_name not in available_backends: backend_reprs = list(map(repr, available_backends)) - error_msg = (f"{backend_name!r} isn't an available catalogue backend.\n" - "Try using geonode.catalogue.backends.BACKEND, where BACKEND is one of:\n" - f" {', '.join(backend_reprs)}\nError was: {e_user}") + error_msg = ( + f"{backend_name!r} isn't an available catalogue backend.\n" + "Try using geonode.catalogue.backends.BACKEND, where BACKEND is one of:\n" + f" {', '.join(backend_reprs)}\nError was: {e_user}" + ) raise ImproperlyConfigured(error_msg) else: # If there's some other error, this must be an error in GeoNode @@ -63,21 +70,18 @@ def load_backend(backend_name): def default_catalogue_backend(): - """Get the default backend - """ + """Get the default backend""" msg = f"There is no '{DEFAULT_CATALOGUE_ALIAS}' backend in CATALOGUE" assert DEFAULT_CATALOGUE_ALIAS in settings.CATALOGUE, msg return settings.CATALOGUE[DEFAULT_CATALOGUE_ALIAS] def get_catalogue(backend=None, skip_caps=True): - """Returns a catalogue object. - """ + """Returns a catalogue object.""" default_backend_config = backend or default_catalogue_backend() - backend_name = default_backend_config['ENGINE'] + backend_name = default_backend_config["ENGINE"] catalog_module = load_backend(backend_name) - assert hasattr(catalog_module, 'CatalogueBackend'), \ - '%s must define a CatalogueBackend class' + assert hasattr(catalog_module, "CatalogueBackend"), "%s must define a CatalogueBackend class" catalog_class = catalog_module.CatalogueBackend cat = catalog_class(skip_caps=skip_caps, **default_backend_config) return cat diff --git a/geonode/catalogue/backends/generic.py b/geonode/catalogue/backends/generic.py index 58f5b600591..e7bceb6353e 100644 --- a/geonode/catalogue/backends/generic.py +++ b/geonode/catalogue/backends/generic.py @@ -33,48 +33,37 @@ TIMEOUT = 10 METADATA_FORMATS = { - 'Atom': ( - 'atom:entry', - 'http://www.w3.org/2005/Atom'), - 'DIF': ( - 'dif:DIF', - 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/'), - 'Dublin Core': ( - 'csw:Record', - 'http://www.opengis.net/cat/csw/2.0.2'), - 'ebRIM': ( - 'rim:RegistryObject', - 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0'), - 'FGDC': ( - 'fgdc:metadata', - 'http://www.opengis.net/cat/csw/csdgm'), - 'ISO': ( - 'gmd:MD_Metadata', - 'http://www.isotc211.org/2005/gmd')} + "Atom": ("atom:entry", "http://www.w3.org/2005/Atom"), + "DIF": ("dif:DIF", "http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/"), + "Dublin Core": ("csw:Record", "http://www.opengis.net/cat/csw/2.0.2"), + "ebRIM": ("rim:RegistryObject", "urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0"), + "FGDC": ("fgdc:metadata", "http://www.opengis.net/cat/csw/csdgm"), + "ISO": ("gmd:MD_Metadata", "http://www.isotc211.org/2005/gmd"), +} class Catalogue(CatalogueServiceWeb): def __init__(self, *args, **kwargs): - self.url = kwargs['URL'] + self.url = kwargs["URL"] self.user = None self.password = None - self.type = kwargs['ENGINE'].split('.')[-1] + self.type = kwargs["ENGINE"].split(".")[-1] self.local = False self._group_ids = {} self._operation_ids = {} self.connected = False - skip_caps = kwargs.get('skip_caps', True) + skip_caps = kwargs.get("skip_caps", True) CatalogueServiceWeb.__init__(self, url=self.url, skip_caps=skip_caps) upurl = urlparse(self.url) - self.base = f'{upurl.scheme}://{upurl.netloc}/' + self.base = f"{upurl.scheme}://{upurl.netloc}/" # User and Password are optional - if 'USER' in kwargs: - self.user = kwargs['USER'] - if 'PASSWORD' in kwargs: - self.password = kwargs['PASSWORD'] + if "USER" in kwargs: + self.user = kwargs["USER"] + if "PASSWORD" in kwargs: + self.password = kwargs["PASSWORD"] def __enter__(self, *args, **kwargs): self.login() @@ -95,31 +84,29 @@ def get_by_uuid(self, uuid): except Exception: return None - if hasattr(self, 'records'): + if hasattr(self, "records"): if len(self.records) < 1: return None record = list(self.records.values())[0] record.keywords = [] - if hasattr( - record, - 'identification') and hasattr( - record.identification, - 'keywords'): + if hasattr(record, "identification") and hasattr(record.identification, "keywords"): for kw in record.identification.keywords: - record.keywords.extend(kw['keywords']) + record.keywords.extend(kw["keywords"]) return record else: return None def url_for_uuid(self, uuid, outputschema): - _query_string = urlencode({ - "request": "GetRecordById", - "service": "CSW", - "version": "2.0.2", - "id": uuid, - "outputschema": outputschema, - "elementsetname": "full" - }) + _query_string = urlencode( + { + "request": "GetRecordById", + "service": "CSW", + "version": "2.0.2", + "id": uuid, + "outputschema": outputschema, + "elementsetname": "full", + } + ) return f"{self.url}?{_query_string}" def urls_for_uuid(self, uuid): @@ -127,34 +114,28 @@ def urls_for_uuid(self, uuid): urls = [] for mformat in self.formats: - urls.append( - ('text/xml', - mformat, - self.url_for_uuid( - uuid, - METADATA_FORMATS[mformat][1]))) + urls.append(("text/xml", mformat, self.url_for_uuid(uuid, METADATA_FORMATS[mformat][1]))) return urls def csw_gen_xml(self, layer, template): - id_pname = 'dc:identifier' - if self.type == 'deegree': - id_pname = 'apiso:Identifier' - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + id_pname = "dc:identifier" + if self.type == "deegree": + id_pname = "apiso:Identifier" + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL tpl = get_template(template) - ctx = {'layer': layer, - 'SITEURL': site_url, - 'id_pname': id_pname, - 'LICENSES_METADATA': getattr(settings, - 'LICENSES', - dict()).get('METADATA', - 'never')} + ctx = { + "layer": layer, + "SITEURL": site_url, + "id_pname": id_pname, + "LICENSES_METADATA": getattr(settings, "LICENSES", dict()).get("METADATA", "never"), + } md_doc = tpl.render(context=ctx) return md_doc def csw_gen_anytext(self, xml): - """ get all element data from an XML document """ + """get all element data from an XML document""" xml = dlxml.fromstring(xml) - return ' '.join([value.strip() for value in xml.xpath('//text()')]) + return " ".join([value.strip() for value in xml.xpath("//text()")]) def csw_request(self, layer, template): md_doc = self.csw_gen_xml(layer, template) @@ -164,14 +145,14 @@ def csw_request(self, layer, template): def create_from_dataset(self, layer): response = self.csw_request(layer, "catalogue/transaction_insert.xml") # noqa # TODO: Parse response, check for error report - return self.url_for_uuid(layer.uuid, namespaces['gmd']) + return self.url_for_uuid(layer.uuid, namespaces["gmd"]) def delete_dataset(self, layer): response = self.csw_request(layer, "catalogue/transaction_delete.xml") # noqa # TODO: Parse response, check for error report def update_dataset(self, layer): - tmpl = 'catalogue/transaction_update.xml' + tmpl = "catalogue/transaction_update.xml" response = self.csw_request(layer, tmpl) # noqa # TODO: Parse response, check for error report @@ -184,17 +165,18 @@ def search(self, keywords, startposition, maxrecords, bbox): dataset_query_like = [] if keywords: for _kw in keywords: - dataset_query_like.append(PropertyIsLike('csw:AnyText', _kw)) + dataset_query_like.append(PropertyIsLike("csw:AnyText", _kw)) bbox_query = [] if bbox: bbox_query = BBox(bbox) return self.getrecords2( - typenames=' '.join(formats), + typenames=" ".join(formats), constraints=dataset_query_like + bbox_query, startposition=startposition, maxrecords=maxrecords, - outputschema='http://www.isotc211.org/2005/gmd', - esn='full') + outputschema="http://www.isotc211.org/2005/gmd", + esn="full", + ) def normalize_bbox(self, bbox): return [bbox[1], bbox[0], bbox[3], bbox[2]] @@ -210,39 +192,36 @@ def metadatarecord2dict(self, rec): return None # Let owslib do some parsing for us... result = {} - result['uuid'] = rec.identifier - result['title'] = rec.identification.title - result['abstract'] = rec.identification.abstract + result["uuid"] = rec.identifier + result["title"] = rec.identification.title + result["abstract"] = rec.identification.abstract keywords = [] for kw in rec.identification.keywords: - keywords.extend(kw['keywords']) + keywords.extend(kw["keywords"]) - result['keywords'] = keywords + result["keywords"] = keywords # XXX needs indexing ? how - result['attribution'] = {'title': '', 'href': ''} + result["attribution"] = {"title": "", "href": ""} - result['name'] = result['uuid'] + result["name"] = result["uuid"] - result['bbox'] = { - 'minx': rec.identification.bbox.minx, - 'maxx': rec.identification.bbox.maxx, - 'miny': rec.identification.bbox.miny, - 'maxy': rec.identification.bbox.maxy + result["bbox"] = { + "minx": rec.identification.bbox.minx, + "maxx": rec.identification.bbox.maxx, + "miny": rec.identification.bbox.miny, + "maxy": rec.identification.bbox.maxy, } # locate all distribution links - result['download_links'] = self.extract_links(rec) + result["download_links"] = self.extract_links(rec) # construct the link to the Catalogue metadata record (not # self-indexed) - result['metadata_links'] = [ - ("text/xml", - "ISO", - self.url_for_uuid( - rec.identifier, - 'http://www.isotc211.org/2005/gmd'))] + result["metadata_links"] = [ + ("text/xml", "ISO", self.url_for_uuid(rec.identifier, "http://www.isotc211.org/2005/gmd")) + ] return result @@ -253,15 +232,15 @@ def extract_links(self, rec): # extract subset of description value for user-friendly display format_re = re.compile(r".*\((.*)(\s*Format*\s*)\).*?") - if not hasattr(rec, 'distribution'): + if not hasattr(rec, "distribution"): return None - if not hasattr(rec.distribution, 'online'): + if not hasattr(rec.distribution, "online"): return None for link_el in rec.distribution.online: - if 'WWW:DOWNLOAD' in link_el.protocol: + if "WWW:DOWNLOAD" in link_el.protocol: try: - extension = link_el.name.split('.')[-1] + extension = link_el.name.split(".")[-1] format = format_re.match(link_el.description).groups()[0] href = link_el.url links.append((extension, format, href)) @@ -279,8 +258,8 @@ def get_record(self, uuid): rec = self.catalogue.get_by_uuid(uuid) if rec is not None: rec.links = dict() - rec.links['metadata'] = self.catalogue.urls_for_uuid(uuid) - rec.links['download'] = self.catalogue.extract_links(rec) + rec.links["metadata"] = self.catalogue.urls_for_uuid(uuid) + rec.links["download"] = self.catalogue.extract_links(rec) return rec def search_records(self, keywords, start, limit, bbox): @@ -289,13 +268,13 @@ def search_records(self, keywords, start, limit, bbox): self.catalogue.search(keywords, start + 1, limit, bbox) # build results into JSON for API - results = [ - self.catalogue.metadatarecord2dict(doc) for v, - doc in self.catalogue.records.items()] + results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.items()] - result = {'rows': results, - 'total': self.catalogue.results['matches'], - 'next_page': self.catalogue.results.get('nextrecord', 0)} + result = { + "rows": results, + "total": self.catalogue.results["matches"], + "next_page": self.catalogue.results.get("nextrecord", 0), + } return result @@ -310,8 +289,7 @@ def remove_record(self, uuid): # too. self.catalogue.delete_dataset({"uuid": uuid}) except Exception: - logger.exception( - 'Couldn\'t delete Catalogue record during cleanup()') + logger.exception("Couldn't delete Catalogue record during cleanup()") def create_record(self, item): with self.catalogue: diff --git a/geonode/catalogue/backends/pycsw_http.py b/geonode/catalogue/backends/pycsw_http.py index 493f02ff05f..0058c41359c 100644 --- a/geonode/catalogue/backends/pycsw_http.py +++ b/geonode/catalogue/backends/pycsw_http.py @@ -17,8 +17,7 @@ # ######################################################################### -from geonode.catalogue.backends.generic import CatalogueBackend \ - as GenericCatalogueBackend +from geonode.catalogue.backends.generic import CatalogueBackend as GenericCatalogueBackend class CatalogueBackend(GenericCatalogueBackend): @@ -27,5 +26,4 @@ class CatalogueBackend(GenericCatalogueBackend): def __init__(self, *args, **kwargs): """initialize pycsw HTTP CSW backend""" GenericCatalogueBackend.__init__(CatalogueBackend, self, *args, **kwargs) - self.catalogue.formats = \ - ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'ISO'] + self.catalogue.formats = ["Atom", "DIF", "Dublin Core", "ebRIM", "FGDC", "ISO"] diff --git a/geonode/catalogue/backends/pycsw_local.py b/geonode/catalogue/backends/pycsw_local.py index 499b859fd9b..d5cf16d5546 100644 --- a/geonode/catalogue/backends/pycsw_local.py +++ b/geonode/catalogue/backends/pycsw_local.py @@ -26,41 +26,46 @@ from geonode.catalogue.backends.generic import METADATA_FORMATS from shapely.errors import WKBReadingError, WKTReadingError -true_value = 'true' -false_value = 'false' -if settings.DATABASES['default']['ENGINE'].endswith(('sqlite', 'sqlite3', 'spatialite',)): - true_value = '1' - false_value = '0' +true_value = "true" +false_value = "false" +if settings.DATABASES["default"]["ENGINE"].endswith( + ( + "sqlite", + "sqlite3", + "spatialite", + ) +): + true_value = "1" + false_value = "0" # pycsw settings that the user shouldn't have to worry about CONFIGURATION = { - 'server': { - 'home': '.', - 'url': settings.CATALOGUE['default']['URL'], - 'encoding': 'UTF-8', - 'language': settings.LANGUAGE_CODE, - 'maxrecords': '10', + "server": { + "home": ".", + "url": settings.CATALOGUE["default"]["URL"], + "encoding": "UTF-8", + "language": settings.LANGUAGE_CODE, + "maxrecords": "10", # 'loglevel': 'DEBUG', # 'logfile': '/tmp/pycsw.log', # 'federatedcatalogues': 'http://geo.data.gov/geoportal/csw/discovery', - 'pretty_print': 'true', - 'domainquerytype': 'range', - 'domaincounts': 'true', - 'profiles': 'apiso,ebrim', + "pretty_print": "true", + "domainquerytype": "range", + "domaincounts": "true", + "profiles": "apiso,ebrim", + }, + "repository": { + "source": "geonode.catalogue.backends.pycsw_plugin.GeoNodeRepository", + "filter": "uuid IS NOT NULL", + "mappings": os.path.join(os.path.dirname(__file__), "pycsw_local_mappings.py"), }, - 'repository': { - 'source': 'geonode.catalogue.backends.pycsw_plugin.GeoNodeRepository', - 'filter': 'uuid IS NOT NULL', - 'mappings': os.path.join(os.path.dirname(__file__), 'pycsw_local_mappings.py') - } } class CatalogueBackend(GenericCatalogueBackend): - def __init__(self, *args, **kwargs): GenericCatalogueBackend.__init__(CatalogueBackend, self, *args, **kwargs) - self.catalogue.formats = ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'ISO'] + self.catalogue.formats = ["Atom", "DIF", "Dublin Core", "ebRIM", "FGDC", "ISO"] self.catalogue.local = True def remove_record(self, uuid): @@ -74,20 +79,20 @@ def get_record(self, uuid): if len(results) < 1: return None - result = dlxml.fromstring(results).find('{http://www.isotc211.org/2005/gmd}MD_Metadata') + result = dlxml.fromstring(results).find("{http://www.isotc211.org/2005/gmd}MD_Metadata") if result is None: return None record = MD_Metadata(result) record.keywords = [] - if hasattr(record, 'identification') and hasattr(record.identification, 'keywords'): + if hasattr(record, "identification") and hasattr(record.identification, "keywords"): for kw in record.identification.keywords: - record.keywords.extend(kw['keywords']) + record.keywords.extend(kw["keywords"]) record.links = {} - record.links['metadata'] = self.catalogue.urls_for_uuid(uuid) - record.links['download'] = self.catalogue.extract_links(record) + record.links["metadata"] = self.catalogue.urls_for_uuid(uuid) + record.links["download"] = self.catalogue.extract_links(record) return record def search_records(self, keywords, start, limit, bbox): @@ -96,18 +101,20 @@ def search_records(self, keywords, start, limit, bbox): # serialize XML e = dlxml.fromstring(lresults) - self.catalogue.records = \ - [MD_Metadata(x) for x in e.findall('//{http://www.isotc211.org/2005/gmd}MD_Metadata')] + self.catalogue.records = [ + MD_Metadata(x) for x in e.findall("//{http://www.isotc211.org/2005/gmd}MD_Metadata") + ] # build results into JSON for API results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.items()] - result = {'rows': results, - 'total': e.find('{http://www.opengis.net/cat/csw/2.0.2}SearchResults').attrib.get( - 'numberOfRecordsMatched'), - 'next_page': e.find('{http://www.opengis.net/cat/csw/2.0.2}SearchResults').attrib.get( - 'nextRecord') - } + result = { + "rows": results, + "total": e.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib.get( + "numberOfRecordsMatched" + ), + "next_page": e.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib.get("nextRecord"), + } return result @@ -116,19 +123,19 @@ def _csw_local_dispatch(self, keywords=None, start=0, limit=10, bbox=None, ident HTTP-less CSW """ - mdict = dict(settings.PYCSW['CONFIGURATION'], **CONFIGURATION) - if 'server' in settings.PYCSW['CONFIGURATION']: + mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) + if "server" in settings.PYCSW["CONFIGURATION"]: # override server system defaults with user specified directives - mdict['server'].update(settings.PYCSW['CONFIGURATION']['server']) + mdict["server"].update(settings.PYCSW["CONFIGURATION"]["server"]) # fake HTTP environment variable - os.environ['QUERY_STRING'] = '' + os.environ["QUERY_STRING"] = "" # init pycsw - csw = server.Csw(mdict, version='2.0.2') + csw = server.Csw(mdict, version="2.0.2") # fake HTTP method - csw.requesttype = 'GET' + csw.requesttype = "GET" # fake HTTP request parameters if identifier is None: # it's a GetRecords request @@ -137,25 +144,25 @@ def _csw_local_dispatch(self, keywords=None, start=0, limit=10, bbox=None, ident formats.append(METADATA_FORMATS[f][0]) csw.kvp = { - 'service': 'CSW', - 'version': '2.0.2', - 'elementsetname': 'full', - 'typenames': formats, - 'resulttype': 'results', - 'constraintlanguage': 'CQL_TEXT', - 'outputschema': 'http://www.isotc211.org/2005/gmd', - 'constraint': None, - 'startposition': start, - 'maxrecords': limit + "service": "CSW", + "version": "2.0.2", + "elementsetname": "full", + "typenames": formats, + "resulttype": "results", + "constraintlanguage": "CQL_TEXT", + "outputschema": "http://www.isotc211.org/2005/gmd", + "constraint": None, + "startposition": start, + "maxrecords": limit, } response = csw.getrecords2() else: # it's a GetRecordById request csw.kvp = { - 'service': 'CSW', - 'version': '2.0.2', - 'request': 'GetRecordById', - 'id': identifier, - 'outputschema': 'http://www.isotc211.org/2005/gmd', + "service": "CSW", + "version": "2.0.2", + "request": "GetRecordById", + "id": identifier, + "outputschema": "http://www.isotc211.org/2005/gmd", } # FIXME(Ariel): Remove this try/except block when pycsw deals with # empty geometry fields better. diff --git a/geonode/catalogue/backends/pycsw_local_mappings.py b/geonode/catalogue/backends/pycsw_local_mappings.py index ead12f37b9a..4ba4f7902ff 100644 --- a/geonode/catalogue/backends/pycsw_local_mappings.py +++ b/geonode/catalogue/backends/pycsw_local_mappings.py @@ -18,65 +18,65 @@ ######################################################################### MD_CORE_MODEL = { - 'typename': 'pycsw:CoreMetadata', - 'outputschema': 'http://pycsw.org/metadata', - 'mappings': { - 'pycsw:Identifier': 'uuid', - 'pycsw:Typename': 'csw_typename', - 'pycsw:Schema': 'csw_schema', - 'pycsw:MdSource': 'csw_mdsource', - 'pycsw:InsertDate': 'csw_insert_date', - 'pycsw:XML': 'metadata_xml', - 'pycsw:AnyText': 'csw_anytext', - 'pycsw:Language': 'language', - 'pycsw:Title': 'title', - 'pycsw:Abstract': 'raw_abstract', - 'pycsw:Keywords': 'keyword_csv', - 'pycsw:KeywordType': 'keywordstype', - 'pycsw:Format': 'spatial_representation_type_string', - 'pycsw:Source': 'source', - 'pycsw:Date': 'date', - 'pycsw:Modified': 'date', - 'pycsw:Type': 'csw_type', - 'pycsw:BoundingBox': 'csw_wkt_geometry', - 'pycsw:CRS': 'csw_crs', - 'pycsw:AlternateTitle': 'alternate', - 'pycsw:RevisionDate': 'date', - 'pycsw:CreationDate': 'date', - 'pycsw:PublicationDate': 'date', - 'pycsw:Organization': 'organizationname', - 'pycsw:OrganizationName': 'organizationname', - 'pycsw:SecurityConstraints': 'securityconstraints', - 'pycsw:ParentIdentifier': 'parentidentifier', - 'pycsw:TopicCategory': 'topiccategory', - 'pycsw:ResourceLanguage': 'language', - 'pycsw:GeographicDescriptionCode': 'geodescode', - 'pycsw:Denominator': 'denominator', - 'pycsw:DistanceValue': 'distancevalue', - 'pycsw:DistanceUOM': 'distanceuom', - 'pycsw:TempExtent_begin': 'temporal_extent_start', - 'pycsw:TempExtent_end': 'temporal_extent_end', - 'pycsw:ServiceType': 'servicetype', - 'pycsw:ServiceTypeVersion': 'servicetypeversion', - 'pycsw:Operation': 'operation', - 'pycsw:CouplingType': 'couplingtype', - 'pycsw:OperatesOn': 'operateson', - 'pycsw:OperatesOnIdentifier': 'operatesonidentifier', - 'pycsw:OperatesOnName': 'operatesoname', - 'pycsw:Degree': 'degree', - 'pycsw:AccessConstraints': 'restriction_code', - 'pycsw:OtherConstraints': 'raw_constraints_other', - 'pycsw:Classification': 'classification', - 'pycsw:ConditionApplyingToAccessAndUse': 'conditionapplyingtoaccessanduse', - 'pycsw:Lineage': 'lineage', - 'pycsw:ResponsiblePartyRole': 'responsiblepartyrole', - 'pycsw:SpecificationTitle': 'specificationtitle', - 'pycsw:SpecificationDate': 'specificationdate', - 'pycsw:SpecificationDateType': 'specificationdatetype', - 'pycsw:Creator': 'creator', - 'pycsw:Publisher': 'publisher', - 'pycsw:Contributor': 'contributor', - 'pycsw:Relation': 'relation', - 'pycsw:Links': 'download_links', - } + "typename": "pycsw:CoreMetadata", + "outputschema": "http://pycsw.org/metadata", + "mappings": { + "pycsw:Identifier": "uuid", + "pycsw:Typename": "csw_typename", + "pycsw:Schema": "csw_schema", + "pycsw:MdSource": "csw_mdsource", + "pycsw:InsertDate": "csw_insert_date", + "pycsw:XML": "metadata_xml", + "pycsw:AnyText": "csw_anytext", + "pycsw:Language": "language", + "pycsw:Title": "title", + "pycsw:Abstract": "raw_abstract", + "pycsw:Keywords": "keyword_csv", + "pycsw:KeywordType": "keywordstype", + "pycsw:Format": "spatial_representation_type_string", + "pycsw:Source": "source", + "pycsw:Date": "date", + "pycsw:Modified": "date", + "pycsw:Type": "csw_type", + "pycsw:BoundingBox": "csw_wkt_geometry", + "pycsw:CRS": "csw_crs", + "pycsw:AlternateTitle": "alternate", + "pycsw:RevisionDate": "date", + "pycsw:CreationDate": "date", + "pycsw:PublicationDate": "date", + "pycsw:Organization": "organizationname", + "pycsw:OrganizationName": "organizationname", + "pycsw:SecurityConstraints": "securityconstraints", + "pycsw:ParentIdentifier": "parentidentifier", + "pycsw:TopicCategory": "topiccategory", + "pycsw:ResourceLanguage": "language", + "pycsw:GeographicDescriptionCode": "geodescode", + "pycsw:Denominator": "denominator", + "pycsw:DistanceValue": "distancevalue", + "pycsw:DistanceUOM": "distanceuom", + "pycsw:TempExtent_begin": "temporal_extent_start", + "pycsw:TempExtent_end": "temporal_extent_end", + "pycsw:ServiceType": "servicetype", + "pycsw:ServiceTypeVersion": "servicetypeversion", + "pycsw:Operation": "operation", + "pycsw:CouplingType": "couplingtype", + "pycsw:OperatesOn": "operateson", + "pycsw:OperatesOnIdentifier": "operatesonidentifier", + "pycsw:OperatesOnName": "operatesoname", + "pycsw:Degree": "degree", + "pycsw:AccessConstraints": "restriction_code", + "pycsw:OtherConstraints": "raw_constraints_other", + "pycsw:Classification": "classification", + "pycsw:ConditionApplyingToAccessAndUse": "conditionapplyingtoaccessanduse", + "pycsw:Lineage": "lineage", + "pycsw:ResponsiblePartyRole": "responsiblepartyrole", + "pycsw:SpecificationTitle": "specificationtitle", + "pycsw:SpecificationDate": "specificationdate", + "pycsw:SpecificationDateType": "specificationdatetype", + "pycsw:Creator": "creator", + "pycsw:Publisher": "publisher", + "pycsw:Contributor": "contributor", + "pycsw:Relation": "relation", + "pycsw:Links": "download_links", + }, } diff --git a/geonode/catalogue/backends/pycsw_plugin.py b/geonode/catalogue/backends/pycsw_plugin.py index 855c5da59c7..7c951b235a9 100644 --- a/geonode/catalogue/backends/pycsw_plugin.py +++ b/geonode/catalogue/backends/pycsw_plugin.py @@ -33,12 +33,12 @@ GEONODE_SERVICE_TYPES = { # 'GeoNode enum': 'CSW enum' - 'http://www.opengis.net/cat/csw/2.0.2': 'OGC:CSW', - 'http://www.opengis.net/wms': 'OGC:WMS', - 'http://www.opengis.net/wmts/1.0': 'OGC:WMTS', - 'https://wiki.osgeo.org/wiki/TMS': 'OSGeo:TMS', - 'urn:x-esri:serviceType:ArcGIS:MapServer': 'ESRI:ArcGIS:MapServer', - 'urn:x-esri:serviceType:ArcGIS:ImageServer': 'ESRI:ArcGIS:ImageServer' + "http://www.opengis.net/cat/csw/2.0.2": "OGC:CSW", + "http://www.opengis.net/wms": "OGC:WMS", + "http://www.opengis.net/wmts/1.0": "OGC:WMTS", + "https://wiki.osgeo.org/wiki/TMS": "OSGeo:TMS", + "urn:x-esri:serviceType:ArcGIS:MapServer": "ESRI:ArcGIS:MapServer", + "urn:x-esri:serviceType:ArcGIS:ImageServer": "ESRI:ArcGIS:ImageServer", } @@ -55,90 +55,89 @@ def __init__(self, context, repo_filter=None): self.context = context self.filter = repo_filter self.fts = False - self.label = 'GeoNode' + self.label = "GeoNode" self.local_ingest = True - self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1] + self.dbtype = settings.DATABASES["default"]["ENGINE"].split(".")[-1] # GeoNode PostgreSQL installs are PostGIS enabled - if self.dbtype == 'postgis': - self.dbtype = 'postgresql+postgis+wkt' + if self.dbtype == "postgis": + self.dbtype = "postgresql+postgis+wkt" - if self.dbtype in {'sqlite', 'sqlite3'}: # load SQLite query bindings - connection.connection.create_function( - 'query_spatial', 4, query_spatial) - connection.connection.create_function( - 'get_anytext', 1, util.get_anytext) - connection.connection.create_function( - 'get_geometry_area', 1, get_geometry_area) + if self.dbtype in {"sqlite", "sqlite3"}: # load SQLite query bindings + connection.connection.create_function("query_spatial", 4, query_spatial) + connection.connection.create_function("get_anytext", 1, util.get_anytext) + connection.connection.create_function("get_geometry_area", 1, get_geometry_area) # generate core queryables db and obj bindings self.queryables = {} - for tname in self.context.model['typenames']: - for qname in self.context.model['typenames'][tname]['queryables']: + for tname in self.context.model["typenames"]: + for qname in self.context.model["typenames"][tname]["queryables"]: self.queryables[qname] = {} - items = list(self.context.model['typenames'][tname]['queryables'][qname].items()) + items = list(self.context.model["typenames"][tname]["queryables"][qname].items()) for qkey, qvalue in items: self.queryables[qname][qkey] = qvalue # flatten all queryables # TODO smarter way of doing this - self.queryables['_all'] = {} + self.queryables["_all"] = {} for qbl in self.queryables: - self.queryables['_all'].update(self.queryables[qbl]) - self.queryables['_all'].update(self.context.md_core_model['mappings']) + self.queryables["_all"].update(self.queryables[qbl]) + self.queryables["_all"].update(self.context.md_core_model["mappings"]) - if 'Harvest' in self.context.model['operations'] and 'Transaction' in self.context.model['operations']: - self.context.model['operations']['Harvest']['parameters']['ResourceType']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa - self.context.model['operations']['Transaction']['parameters']['TransactionSchemas']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa + if "Harvest" in self.context.model["operations"] and "Transaction" in self.context.model["operations"]: + self.context.model["operations"]["Harvest"]["parameters"]["ResourceType"]["values"] = list( + GEONODE_SERVICE_TYPES.keys() + ) # noqa + self.context.model["operations"]["Transaction"]["parameters"]["TransactionSchemas"]["values"] = list( + GEONODE_SERVICE_TYPES.keys() + ) # noqa def dataset(self): """ Stub to mock a pycsw dataset object for Transactions """ - return type('ResourceBase', (object,), {}) + return type("ResourceBase", (object,), {}) def query_ids(self, ids): """ Query by list of identifiers """ - results = self\ - ._get_repo_filter(ResourceBase.objects)\ - .filter(uuid__in=ids)\ - .all() + results = self._get_repo_filter(ResourceBase.objects).filter(uuid__in=ids).all() return results - def query_domain(self, domain, typenames, - domainquerytype='list', count=False): + def query_domain(self, domain, typenames, domainquerytype="list", count=False): """ Query by property domain values """ objects = self._get_repo_filter(ResourceBase.objects) - if domainquerytype == 'range': - return [tuple(objects.aggregate( - Min(domain), Max(domain)).values())] + if domainquerytype == "range": + return [tuple(objects.aggregate(Min(domain), Max(domain)).values())] else: if count: - return [(d[domain], d[f'{domain}__count']) - for d in objects.values(domain).annotate(Count(domain))] + return [(d[domain], d[f"{domain}__count"]) for d in objects.values(domain).annotate(Count(domain))] else: return objects.values_list(domain).distinct() - def query_insert(self, direction='max'): + def query_insert(self, direction="max"): """ Query to get latest (default) or earliest update to repository """ - if direction == 'min': - return ResourceBase.objects.aggregate( - Min('last_updated'))['last_updated__min'].strftime('%Y-%m-%dT%H:%M:%SZ') - return self._get_repo_filter(ResourceBase.objects).aggregate( - Max('last_updated'))['last_updated__max'].strftime('%Y-%m-%dT%H:%M:%SZ') + if direction == "min": + return ResourceBase.objects.aggregate(Min("last_updated"))["last_updated__min"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + return ( + self._get_repo_filter(ResourceBase.objects) + .aggregate(Max("last_updated"))["last_updated__max"] + .strftime("%Y-%m-%dT%H:%M:%SZ") + ) def query_source(self, source): """ @@ -146,8 +145,7 @@ def query_source(self, source): """ return self._get_repo_filter(ResourceBase.objects).filter(url=source) - def query(self, constraint, sortby=None, typenames=None, - maxrecords=10, startposition=0): + def query(self, constraint, sortby=None, typenames=None, maxrecords=10, startposition=0): """ Query records from underlying repository """ @@ -155,51 +153,51 @@ def query(self, constraint, sortby=None, typenames=None, # run the raw query and get total # we want to exclude layers which are not valid, as it is done in the # search engine - pycsw_filters = settings.PYCSW.get('FILTER', {'resource_type__in': ['dataset']}) - - if 'where' in constraint: # GetRecords with constraint - query = self._get_repo_filter( - ResourceBase.objects.filter(**pycsw_filters)).extra( - where=[ - constraint['where']], - params=constraint['values']) + pycsw_filters = settings.PYCSW.get("FILTER", {"resource_type__in": ["dataset"]}) + + if "where" in constraint: # GetRecords with constraint + query = self._get_repo_filter(ResourceBase.objects.filter(**pycsw_filters)).extra( + where=[constraint["where"]], params=constraint["values"] + ) else: # GetRecords sans constraint - query = self._get_repo_filter( - ResourceBase.objects.filter(**pycsw_filters)) + query = self._get_repo_filter(ResourceBase.objects.filter(**pycsw_filters)) total = query.count() # apply sorting, limit and offset if sortby is not None: - if 'spatial' in sortby and sortby['spatial']: # spatial sort + if "spatial" in sortby and sortby["spatial"]: # spatial sort desc = False - if sortby['order'] == 'DESC': + if sortby["order"] == "DESC": desc = True query = query.all() - return [str(total), - sorted(query, - key=lambda x: float(util.get_geometry_area( - getattr(x, sortby['propertyname']))), - reverse=desc, - )[startposition:startposition + int(maxrecords)]] + return [ + str(total), + sorted( + query, + key=lambda x: float(util.get_geometry_area(getattr(x, sortby["propertyname"]))), + reverse=desc, + )[startposition : startposition + int(maxrecords)], + ] else: - if sortby['order'] == 'DESC': + if sortby["order"] == "DESC": pname = f"-{sortby['propertyname']}" else: - pname = sortby['propertyname'] - return [str(total), - query.order_by(pname)[startposition:startposition + int(maxrecords)]] + pname = sortby["propertyname"] + return [str(total), query.order_by(pname)[startposition : startposition + int(maxrecords)]] else: # no sort - return [str(total), query.all()[ - startposition:startposition + int(maxrecords)]] + return [str(total), query.all()[startposition : startposition + int(maxrecords)]] def delete(self, constraint): """ Delete a record from the repository """ - results = self._get_repo_filter(ResourceBase.objects).extra(where=[constraint['where']], - params=constraint['values']).all() + results = ( + self._get_repo_filter(ResourceBase.objects) + .extra(where=[constraint["where"]], params=constraint["values"]) + .all() + ) deleted = len(results) results.delete() return deleted diff --git a/geonode/catalogue/backends/tests.py b/geonode/catalogue/backends/tests.py index 4125ff8ceaa..f579b68872b 100644 --- a/geonode/catalogue/backends/tests.py +++ b/geonode/catalogue/backends/tests.py @@ -10,8 +10,8 @@ pycsw_settings = settings.PYCSW.copy() pycsw_settings_all = settings.PYCSW.copy() -pycsw_settings['FILTER'] = {'resource_type__in': ['dataset', 'map']} -pycsw_settings_all['FILTER'] = {'resource_type__in': ['dataset', 'map', 'document']} +pycsw_settings["FILTER"] = {"resource_type__in": ["dataset", "map"]} +pycsw_settings_all["FILTER"] = {"resource_type__in": ["dataset", "map", "document"]} class TestGeoNodeRepository(TestCase): @@ -26,24 +26,24 @@ def setUp(self): def test_if_pycsw_filter_is_not_set_should_return_only_the_dataset_by_default(self): response = csw_global_dispatch(self.request) root = etree.fromstring(response.content) - child = [x.attrib for x in root if 'numberOfRecordsMatched' in x.attrib] - returned_results = ast.literal_eval(child[0].get('numberOfRecordsMatched', '0')) if child else 0 + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 self.assertEqual(1, returned_results) @override_settings(PYCSW=pycsw_settings) def test_if_pycsw_filter_is_set_should_return_only_datasets_and_map(self): response = csw_global_dispatch(self.request) root = etree.fromstring(response.content) - child = [x.attrib for x in root if 'numberOfRecordsMatched' in x.attrib] - returned_results = ast.literal_eval(child[0].get('numberOfRecordsMatched', '0')) if child else 0 + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 self.assertEqual(2, returned_results) @override_settings(PYCSW=pycsw_settings_all) def test_if_pycsw_filter_is_set_should_return_all_datasets_map_doc(self): response = csw_global_dispatch(self.request) root = etree.fromstring(response.content) - child = [x.attrib for x in root if 'numberOfRecordsMatched' in x.attrib] - returned_results = ast.literal_eval(child[0].get('numberOfRecordsMatched', '0')) if child else 0 + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 self.assertEqual(3, returned_results) @staticmethod diff --git a/geonode/catalogue/metadataxsl/models.py b/geonode/catalogue/metadataxsl/models.py index 566d910147b..56b06bac00f 100644 --- a/geonode/catalogue/metadataxsl/models.py +++ b/geonode/catalogue/metadataxsl/models.py @@ -28,37 +28,33 @@ from geonode.documents.models import Document -ISO_XSL_NAME = 'ISO with XSL' +ISO_XSL_NAME = "ISO with XSL" settings.DOWNLOAD_FORMATS_METADATA.append(ISO_XSL_NAME) def xsl_post_save(instance, sender, **kwargs): - """Add a link to the enriched ISO metadata - """ + """Add a link to the enriched ISO metadata""" add_xsl_link(instance.resourcebase_ptr) def add_xsl_link(resourcebase): - """Add a link to the enriched ISO metadata - """ + """Add a link to the enriched ISO metadata""" try: - urlpath = reverse('prefix_xsl_line', args=[resourcebase.id]) - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + urlpath = reverse("prefix_xsl_line", args=[resourcebase.id]) + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL url = urljoin(site_url, urlpath) link, created = Link.objects.get_or_create( resource=resourcebase, url=url, - defaults=dict(name=ISO_XSL_NAME, - extension='xml', - mime='text/xml', - link_type='metadata')) + defaults=dict(name=ISO_XSL_NAME, extension="xml", mime="text/xml", link_type="metadata"), + ) return created except Exception: return False -if 'geonode.catalogue' in settings.INSTALLED_APPS: +if "geonode.catalogue" in settings.INSTALLED_APPS: signals.post_save.connect(xsl_post_save, sender=Dataset) signals.post_save.connect(xsl_post_save, sender=Document) # TODO: maps as well? diff --git a/geonode/catalogue/metadataxsl/tests.py b/geonode/catalogue/metadataxsl/tests.py index 64a568031ab..e09c1291a5f 100644 --- a/geonode/catalogue/metadataxsl/tests.py +++ b/geonode/catalogue/metadataxsl/tests.py @@ -21,10 +21,7 @@ from geonode.base.models import ResourceBase from geonode.tests.base import GeoNodeBaseTestSupport -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models class MetadataXSLTest(GeoNodeBaseTestSupport): @@ -32,14 +29,11 @@ class MetadataXSLTest(GeoNodeBaseTestSupport): """ Tests geonode.catalogue.metadataxsl app/module """ - type = 'dataset' + + type = "dataset" # loading test initial data - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] @classmethod def setUpClass(cls): @@ -63,7 +57,7 @@ def test_showmetadata_access_perms(self): _dataset.set_permissions({"users": {}, "groups": {}}) _dataset.save() self.client.login(username=self.adm_un, password=self.adm_pw) - url = reverse('prefix_xsl_line', kwargs={'id': _dataset.id}) + url = reverse("prefix_xsl_line", kwargs={"id": _dataset.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.client.logout() diff --git a/geonode/catalogue/metadataxsl/urls.py b/geonode/catalogue/metadataxsl/urls.py index 8e6b4348df7..10467b86ff8 100644 --- a/geonode/catalogue/metadataxsl/urls.py +++ b/geonode/catalogue/metadataxsl/urls.py @@ -20,6 +20,4 @@ from django.conf.urls import url from . import views -urlpatterns = [ - url(r'^xsl/(?P\d+)$', views.prefix_xsl_line, name='prefix_xsl_line') -] +urlpatterns = [url(r"^xsl/(?P\d+)$", views.prefix_xsl_line, name="prefix_xsl_line")] diff --git a/geonode/catalogue/metadataxsl/views.py b/geonode/catalogue/metadataxsl/views.py index a232ac80709..e6cb71feb2f 100644 --- a/geonode/catalogue/metadataxsl/views.py +++ b/geonode/catalogue/metadataxsl/views.py @@ -44,15 +44,14 @@ def prefix_xsl_line(req, id): resource = None try: resource = get_object_or_404(ResourceBase, pk=id) - query = { - 'id': resource.get_real_instance().id - } + query = {"id": resource.get_real_instance().id} resource = resolve_object( req, resource.get_real_instance_class(), query, - permission='base.view_resourcebase', - permission_msg=_("You are not permitted to view this resource")) + permission="base.view_resourcebase", + permission_msg=_("You are not permitted to view this resource"), + ) catalogue = get_catalogue() record = catalogue.get_record(resource.uuid) if record: @@ -73,15 +72,10 @@ def prefix_xsl_line(req, id): xml = md_doc except Exception: logger.debug(traceback.format_exc()) - return HttpResponse( - "Resource Metadata not available!" - ) - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL - xsl_static = getattr(settings, 'CATALOG_METADATA_XSL', '/static/metadataxsl/metadata.xsl') - xsl_path = f'{site_url}{xsl_static}' + return HttpResponse("Resource Metadata not available!") + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + xsl_static = getattr(settings, "CATALOG_METADATA_XSL", "/static/metadataxsl/metadata.xsl") + xsl_path = f"{site_url}{xsl_static}" xsl_line = f'' - return HttpResponse( - xsl_line + xml, - content_type="text/xml" - ) + return HttpResponse(xsl_line + xml, content_type="text/xml") diff --git a/geonode/catalogue/models.py b/geonode/catalogue/models.py index e132be97348..caebd2fe42e 100644 --- a/geonode/catalogue/models.py +++ b/geonode/catalogue/models.py @@ -40,7 +40,7 @@ def catalogue_pre_delete(instance, sender, **kwargs): def catalogue_post_save(instance, sender, **kwargs): """Get information from catalogue""" - _id = instance.resourcebase_ptr.id if hasattr(instance, 'resourcebase_ptr') else instance.id + _id = instance.resourcebase_ptr.id if hasattr(instance, "resourcebase_ptr") else instance.id resources = ResourceBase.objects.filter(id=_id) # Update the Catalog @@ -57,38 +57,28 @@ def catalogue_post_save(instance, sender, **kwargs): raise err if not record: - msg = f'Metadata record for {instance.title} does not exist, check the catalogue signals.' + msg = f"Metadata record for {instance.title} does not exist, check the catalogue signals." LOGGER.warning(msg) return - if not hasattr(record, 'links'): - msg = f'Metadata record for {instance.title} should contain links.' + if not hasattr(record, "links"): + msg = f"Metadata record for {instance.title} should contain links." raise Exception(msg) # Create the different metadata links with the available formats if resources.exists(): - for mime, name, metadata_url in record.links['metadata']: + for mime, name, metadata_url in record.links["metadata"]: try: Link.objects.get_or_create( resource=resources.get(), url=metadata_url, - defaults=dict( - name=name, - extension='xml', - mime=mime, - link_type='metadata' - ) + defaults=dict(name=name, extension="xml", mime=mime, link_type="metadata"), ) except Exception: - _d = dict(name=name, - extension='xml', - mime=mime, - link_type='metadata') + _d = dict(name=name, extension="xml", mime=mime, link_type="metadata") Link.objects.filter( - resource=resources.get(), - url=metadata_url, - extension='xml', - link_type='metadata').update(**_d) + resource=resources.get(), url=metadata_url, extension="xml", link_type="metadata" + ).update(**_d) # generate an XML document (GeoNode's default is ISO) if instance.metadata_uploaded and instance.metadata_uploaded_preserve: @@ -99,15 +89,12 @@ def catalogue_post_save(instance, sender, **kwargs): csw_anytext = catalogue.catalogue.csw_gen_anytext(md_doc) except Exception as e: LOGGER.exception(e) - csw_anytext = '' + csw_anytext = "" - resources.update( - metadata_xml=md_doc, - csw_wkt_geometry=instance.geographic_bounding_box, - csw_anytext=csw_anytext) + resources.update(metadata_xml=md_doc, csw_wkt_geometry=instance.geographic_bounding_box, csw_anytext=csw_anytext) -if 'geonode.catalogue' in settings.INSTALLED_APPS: +if "geonode.catalogue" in settings.INSTALLED_APPS: signals.post_save.connect(catalogue_post_save, sender=Dataset) signals.pre_delete.connect(catalogue_pre_delete, sender=Dataset) signals.post_save.connect(catalogue_post_save, sender=Document) diff --git a/geonode/catalogue/tests.py b/geonode/catalogue/tests.py index e98290a15b6..e67d56849c3 100644 --- a/geonode/catalogue/tests.py +++ b/geonode/catalogue/tests.py @@ -21,6 +21,8 @@ from django.db.models import Q from django.test import RequestFactory +from django.http.response import Http404 +from django.core.exceptions import PermissionDenied from geonode.layers.models import Dataset from geonode.catalogue import get_catalogue from django.contrib.auth import get_user_model @@ -28,19 +30,15 @@ from geonode.tests.base import GeoNodeBaseTestSupport from geonode.catalogue.models import catalogue_post_save -from geonode.catalogue.views import csw_global_dispatch +from geonode.catalogue.views import csw_global_dispatch, resolve_uuid from geonode.layers.populate_datasets_data import create_dataset_data -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset logger = logging.getLogger(__name__) class CatalogueTest(GeoNodeBaseTestSupport): - @classmethod def setUpClass(cls): super().setUpClass() @@ -144,3 +142,39 @@ def __request_factory_multiple(): request = factory.get(url) request.user = get_user_model().objects.first() return request + + +class UUIDResolverTest(GeoNodeBaseTestSupport): + def setUp(self): + self.dataset = create_single_dataset(name="test_uuid_resolver_dataset") + + def tearDown(self): + Dataset.objects.filter(name="test_uuid_resolver_dataset").delete() + + def test_uuid_resolver_existing_dataset(self): + user = get_user_model().objects.first() + self.dataset.set_default_permissions(owner=user) + request = RequestFactory().get(f"http://localhost:8000/catalogue/uuid/{self.dataset.uuid}") + request.user = user + response = resolve_uuid(request, self.dataset.uuid) + self.assertEqual(302, response.status_code) + self.assertEqual(f"/catalogue/#/dataset/{self.dataset.pk}", response.headers["Location"]) + + def test_uuid_resolver_non_existing_dataset(self): + user = get_user_model().objects.first() + self.dataset.set_default_permissions(owner=user) + request = RequestFactory().get("http://localhost:8000/catalogue/uuid/asdfasdf") + request.user = user + with self.assertRaises(Http404) as context: + resolve_uuid(request, "asdfasdf") + self.assertTrue("No ResourceBase matches the given query." in str(context.exception)) + + def test_uuid_resolver_missing_permissions(self): + self.dataset.set_permissions( + {"groups": {"registered-members": ["base.view_resourcebase", "base.download_resourcebase"]}} + ) + request = RequestFactory().get(f"http://localhost:8000/catalogue/uuid/{self.dataset.uuid}") + request.user = AnonymousUser() + with self.assertRaises(PermissionDenied) as context: + resolve_uuid(request, self.dataset.uuid) + self.assertTrue("Permission Denied" in str(context.exception)) diff --git a/geonode/catalogue/urls.py b/geonode/catalogue/urls.py index 7ea79e56a27..fed2c84eed1 100644 --- a/geonode/catalogue/urls.py +++ b/geonode/catalogue/urls.py @@ -18,13 +18,21 @@ ######################################################################### from django.conf.urls import url +from django.urls import path from . import views urlpatterns = [ - url(r'^csw$', views.csw_global_dispatch, name='csw_global_dispatch'), - url(r'^opensearch$', views.opensearch_dispatch, name='opensearch_dispatch'), - url(r'^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).txt$', - views.csw_render_extra_format_txt, name="csw_render_extra_format_txt"), - url(r'^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).html$', - views.csw_render_extra_format_html, name="csw_render_extra_format_html") + url(r"^csw$", views.csw_global_dispatch, name="csw_global_dispatch"), + url(r"^opensearch$", views.opensearch_dispatch, name="opensearch_dispatch"), + url( + r"^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).txt$", + views.csw_render_extra_format_txt, + name="csw_render_extra_format_txt", + ), + url( + r"^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).html$", + views.csw_render_extra_format_html, + name="csw_render_extra_format_html", + ), + path(r"uuid/", views.resolve_uuid, name="resolve_uuid"), ] diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py index 021063e0bc3..eb79f6f90af 100644 --- a/geonode/catalogue/views.py +++ b/geonode/catalogue/views.py @@ -20,7 +20,7 @@ import logging from django.conf import settings from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import render +from django.shortcuts import render, redirect from django.contrib.auth import get_user_model from django.views.decorators.csrf import csrf_exempt from pycsw import server @@ -31,6 +31,7 @@ from geonode.base.auth import get_or_create_token from geonode.base.models import ContactRole, SpatialRepresentationType from geonode.groups.models import GroupProfile +from geonode.utils import resolve_object from django.db import connection from django.core.exceptions import ObjectDoesNotExist @@ -41,10 +42,10 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): # this view should only operate if pycsw_local is the backend # else, redirect to the URL of the non-pycsw_local backend - if settings.CATALOGUE['default']['ENGINE'] != 'geonode.catalogue.backends.pycsw_local': - return HttpResponseRedirect(settings.CATALOGUE['default']['URL']) + if settings.CATALOGUE["default"]["ENGINE"] != "geonode.catalogue.backends.pycsw_local": + return HttpResponseRedirect(settings.CATALOGUE["default"]["URL"]) - mdict = dict(settings.PYCSW['CONFIGURATION'], **CONFIGURATION) + mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) mdict = config_updater(mdict) if config_updater else mdict access_token = None @@ -53,22 +54,20 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): if access_token and access_token.is_expired(): access_token = None - absolute_uri = f'{request.build_absolute_uri()}' + absolute_uri = f"{request.build_absolute_uri()}" query_string = f"{request.META['QUERY_STRING']}" env = request.META.copy() if access_token and not access_token.is_expired(): - env.update({'access_token': access_token.token}) - if 'access_token' not in query_string: - absolute_uri = f'{absolute_uri}&access_token={access_token.token}' - query_string = f'{query_string}&access_token={access_token.token}' + env.update({"access_token": access_token.token}) + if "access_token" not in query_string: + absolute_uri = f"{absolute_uri}&access_token={access_token.token}" + query_string = f"{query_string}&access_token={access_token.token}" - env.update({'local.app_root': os.path.dirname(__file__), - 'REQUEST_URI': absolute_uri, - 'QUERY_STRING': query_string}) + env.update({"local.app_root": os.path.dirname(__file__), "REQUEST_URI": absolute_uri, "QUERY_STRING": query_string}) # Save original filter before doing anything - mdict_filter = mdict['repository']['filter'] + mdict_filter = mdict["repository"]["filter"] try: # Filter out Layers not accessible to the User @@ -78,13 +77,9 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): else: profiles = get_user_model().objects.filter(username="AnonymousUser") if profiles: - authorized = list( - get_objects_for_user( - profiles[0], - 'base.view_resourcebase').values('id')) + authorized = list(get_objects_for_user(profiles[0], "base.view_resourcebase").values("id")) - layers = ResourceBase.objects.filter( - id__in=[d['id'] for d in authorized]) + layers = ResourceBase.objects.filter(id__in=[d["id"] for d in authorized]) if dataset_filter and layers: layers = dataset_filter(layers) @@ -95,16 +90,16 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): if len(authorized_ids) > 0: authorized_datasets = f"({', '.join(str(e) for e in authorized_ids)})" authorized_datasets_filter = f"id IN {authorized_datasets}" - mdict['repository']['filter'] += f" AND {authorized_datasets_filter}" + mdict["repository"]["filter"] += f" AND {authorized_datasets_filter}" if request.user and request.user.is_authenticated: - mdict['repository']['filter'] = f"({mdict['repository']['filter']}) OR ({authorized_datasets_filter})" + mdict["repository"]["filter"] = f"({mdict['repository']['filter']}) OR ({authorized_datasets_filter})" else: authorized_datasets_filter = "id = -9999" - mdict['repository']['filter'] += f" AND {authorized_datasets_filter}" + mdict["repository"]["filter"] += f" AND {authorized_datasets_filter}" # Filter out Documents and Maps - if 'ALTERNATES_ONLY' in settings.CATALOGUE['default'] and settings.CATALOGUE['default']['ALTERNATES_ONLY']: - mdict['repository']['filter'] += " AND alternate IS NOT NULL" + if "ALTERNATES_ONLY" in settings.CATALOGUE["default"] and settings.CATALOGUE["default"]["ALTERNATES_ONLY"]: + mdict["repository"]["filter"] += " AND alternate IS NOT NULL" # Filter out Layers belonging to specific Groups is_admin = False @@ -118,34 +113,33 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): groups_ids.append(group.id) group_list_all = [] try: - group_list_all = request.user.group_list_all().values('group') + group_list_all = request.user.group_list_all().values("group") except Exception: pass for group in group_list_all: if isinstance(group, dict): - if 'group' in group: - groups_ids.append(group['group']) + if "group" in group: + groups_ids.append(group["group"]) else: groups_ids.append(group.id) - public_groups = GroupProfile.objects.exclude( - access="private").values('group') + public_groups = GroupProfile.objects.exclude(access="private").values("group") for group in public_groups: if isinstance(group, dict): - if 'group' in group: - groups_ids.append(group['group']) + if "group" in group: + groups_ids.append(group["group"]) else: groups_ids.append(group.id) if len(groups_ids) > 0: groups = f"({', '.join(str(e) for e in groups_ids)})" groups_filter = f"(group_id IS NULL OR group_id IN {groups})" - mdict['repository']['filter'] += f" AND {groups_filter}" + mdict["repository"]["filter"] += f" AND {groups_filter}" else: groups_filter = "group_id IS NULL" - mdict['repository']['filter'] += f" AND {groups_filter}" + mdict["repository"]["filter"] += f" AND {groups_filter}" - csw = server.Csw(mdict, env, version='2.0.2') + csw = server.Csw(mdict, env, version="2.0.2") content = csw.dispatch_wsgi() @@ -159,7 +153,7 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): finally: # Restore original filter before doing anything - mdict['repository']['filter'] = mdict_filter + mdict["repository"]["filter"] = mdict_filter return HttpResponse(content, content_type=csw.contenttype) @@ -169,17 +163,21 @@ def opensearch_dispatch(request): """OpenSearch wrapper""" ctx = { - 'shortname': settings.PYCSW['CONFIGURATION']['metadata:main']['identification_title'], - 'description': settings.PYCSW['CONFIGURATION']['metadata:main']['identification_abstract'], - 'developer': settings.PYCSW['CONFIGURATION']['metadata:main']['contact_name'], - 'contact': settings.PYCSW['CONFIGURATION']['metadata:main']['contact_email'], - 'attribution': settings.PYCSW['CONFIGURATION']['metadata:main']['provider_name'], - 'tags': settings.PYCSW['CONFIGURATION']['metadata:main']['identification_keywords'].replace(',', ' '), - 'url': settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + "shortname": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_title"], + "description": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_abstract"], + "developer": settings.PYCSW["CONFIGURATION"]["metadata:main"]["contact_name"], + "contact": settings.PYCSW["CONFIGURATION"]["metadata:main"]["contact_email"], + "attribution": settings.PYCSW["CONFIGURATION"]["metadata:main"]["provider_name"], + "tags": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_keywords"].replace(",", " "), + "url": settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL, } - return render(request, 'catalogue/opensearch_description.xml', context=ctx, - content_type='application/opensearchdescription+xml') + return render( + request, + "catalogue/opensearch_description.xml", + context=ctx, + content_type="application/opensearchdescription+xml", + ) # transforms a row sql query into a two dimension array @@ -191,14 +189,14 @@ def dictfetchall(cursor): # choose separators def get_CSV_spec_char(): - return {"separator": ';', "carriage_return": '\r\n'} + return {"separator": ";", "carriage_return": "\r\n"} # format value to unicode str without ';' char def fst(value): chrs = get_CSV_spec_char() result = str(value) - result = result.replace(chrs["separator"], ',').replace('\\n', ' ').replace('\r\n', ' ') + result = result.replace(chrs["separator"], ",").replace("\\n", " ").replace("\r\n", " ") return result @@ -206,18 +204,19 @@ def fst(value): # the aim is to handle the output format (csv, html or pdf) the same structure def build_md_dict(resource): md_dict = { - 'r_uuid': {'label': 'uuid', 'value': resource.uuid}, - 'r_title': {'label': 'titre', 'value': resource.title} + "r_uuid": {"label": "uuid", "value": resource.uuid}, + "r_title": {"label": "titre", "value": resource.title}, } return md_dict def get_keywords(resource): - content = ' ' + content = " " cursor = connection.cursor() cursor.execute( - "SELECT a.*,b.* FROM taggit_taggeditem as a,taggit_tag" - " as b WHERE a.object_id = %s AND a.tag_id=b.id", [resource.id]) + "SELECT a.*,b.* FROM taggit_taggeditem as a,taggit_tag" " as b WHERE a.object_id = %s AND a.tag_id=b.id", + [resource.id], + ) for x in dictfetchall(cursor): content += f"{fst(x['name'])}, " return content[:-2] @@ -230,8 +229,8 @@ def csw_render_extra_format_txt(request, layeruuid, resname): """pycsw wrapper""" resource = ResourceBase.objects.get(uuid=layeruuid) chrs = get_CSV_spec_char() - s = chrs['separator'] - c = chrs['carriage_return'] + s = chrs["separator"] + c = chrs["carriage_return"] sc = s + c content = f"Resource metadata{sc}" content += f"uuid{s}{fst(resource.uuid)}{sc}" @@ -245,8 +244,7 @@ def csw_render_extra_format_txt(request, layeruuid, resname): content += f"maintenance frequency{s}{fst(resource.maintenance_frequency)}{sc}" try: - sprt = SpatialRepresentationType.objects.get( - id=resource.spatial_representation_type_id) + sprt = SpatialRepresentationType.objects.get(id=resource.spatial_representation_type_id) content += f"identifier{s}{fst(sprt.identifier)}{sc}" except ObjectDoesNotExist: content += f"ObjectDoesNotExist{sc}" @@ -279,17 +277,16 @@ def csw_render_extra_format_txt(request, layeruuid, resname): content = content[:-1] content += sc - if resource.detail_url.find('/datasets/') > -1: + if resource.detail_url.find("/datasets/") > -1: layer = Dataset.objects.get(resourcebase_ptr_id=resource.id) content += f"attribute data{sc}" - content += 'attribute name;label;description\n' + content += "attribute name;label;description\n" for attr in layer.attribute_set.all(): content += fst(attr.attribute) + s content += fst(attr.attribute_label) + s content += fst(attr.description) + sc - pocr = ContactRole.objects.get( - resource_id=resource.id, role='pointOfContact') + pocr = ContactRole.objects.get(resource_id=resource.id, role="pointOfContact") pocp = get_user_model().objects.get(id=pocr.contact_id) content += f"Point of Contact{sc}" content += f"name{s}{fst(pocp.last_name)}{sc}" @@ -305,28 +302,30 @@ def csw_render_extra_format_html(request, layeruuid, resname): resource = ResourceBase.objects.get(uuid=layeruuid) extra_res_md = {} try: - sprt = SpatialRepresentationType.objects.get( - id=resource.spatial_representation_type_id) - extra_res_md['sprt_identifier'] = sprt.identifier + sprt = SpatialRepresentationType.objects.get(id=resource.spatial_representation_type_id) + extra_res_md["sprt_identifier"] = sprt.identifier except ObjectDoesNotExist: - extra_res_md['sprt_identifier'] = 'not filled' + extra_res_md["sprt_identifier"] = "not filled" kw = get_keywords(resource) if len(kw) == 0: - extra_res_md['keywords'] = "no keywords" + extra_res_md["keywords"] = "no keywords" else: - extra_res_md['keywords'] = get_keywords(resource) + extra_res_md["keywords"] = get_keywords(resource) - if resource.detail_url.find('/datasets/') > -1: + if resource.detail_url.find("/datasets/") > -1: layer = Dataset.objects.get(resourcebase_ptr_id=resource.id) - extra_res_md['atrributes'] = '' + extra_res_md["atrributes"] = "" for attr in layer.attribute_set.all(): s = f"{attr.attribute}{attr.attribute_label}{attr.description}" - extra_res_md['atrributes'] += s + extra_res_md["atrributes"] += s - pocr = ContactRole.objects.get( - resource_id=resource.id, role='pointOfContact') + pocr = ContactRole.objects.get(resource_id=resource.id, role="pointOfContact") pocp = get_user_model().objects.get(id=pocr.contact_id) - extra_res_md['poc_last_name'] = pocp.last_name - extra_res_md['poc_email'] = pocp.email - return render(request, "geonode_metadata_full.html", context={"resource": resource, - "extra_res_md": extra_res_md}) + extra_res_md["poc_last_name"] = pocp.last_name + extra_res_md["poc_email"] = pocp.email + return render(request, "geonode_metadata_full.html", context={"resource": resource, "extra_res_md": extra_res_md}) + + +def resolve_uuid(request, uuid): + resource = resolve_object(request, ResourceBase, {"uuid": uuid}) + return redirect(resource) diff --git a/geonode/celery_app.py b/geonode/celery_app.py index a299fb00012..0e3c411c2ad 100644 --- a/geonode/celery_app.py +++ b/geonode/celery_app.py @@ -20,10 +20,11 @@ import logging from celery import Celery + # from celery.schedules import crontab -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geonode.settings') -app = Celery('geonode') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.settings") +app = Celery("geonode") logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ def _log(msg, *args): # Using a string here means the worker will not have to # pickle the object when using Windows. -app.config_from_object('django.conf:settings', namespace="CELERY") +app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() app.autodiscover_tasks(packages=["geonode.harvesting.harvesters"]) diff --git a/geonode/client/conf.py b/geonode/client/conf.py index 1da7749f2a3..e17645110b3 100644 --- a/geonode/client/conf.py +++ b/geonode/client/conf.py @@ -26,7 +26,7 @@ def load_path_attr(path): i = path.rfind(".") - module, attr = path[:i], path[i + 1:] + module, attr = path[:i], path[i + 1 :] try: mod = importlib.import_module(module) except ImportError as e: @@ -48,7 +48,7 @@ def is_installed(package): class GeoNodeClientAppConf(AppConf): - LAYER_PREVIEW_LIBRARY = 'geonode' + LAYER_PREVIEW_LIBRARY = "geonode" HOOKSET = "geonode.client.hooksets.BaseHookSet" def configure_hookset(self, value): diff --git a/geonode/client/hooks.py b/geonode/client/hooks.py index d651a416d3d..f595a1ffc9c 100644 --- a/geonode/client/hooks.py +++ b/geonode/client/hooks.py @@ -20,12 +20,12 @@ class HookProxy: - def __getattr__(self, attr): if not isinstance(settings.GEONODE_CLIENT_HOOKSET, str): return getattr(settings.GEONODE_CLIENT_HOOKSET, attr) else: import importlib + cls = settings.GEONODE_CLIENT_HOOKSET.split(".") module_name, class_name = (".".join(cls[:-1]), cls[-1]) i = importlib.import_module(module_name) diff --git a/geonode/client/hooksets.py b/geonode/client/hooksets.py index 63134a92d5b..963fe47a4f2 100644 --- a/geonode/client/hooksets.py +++ b/geonode/client/hooksets.py @@ -138,5 +138,5 @@ def update_from_viewer(self, conf, context=None): def metadata_update_redirect(self, url, request=None): if "metadata_uri" in url: - return url.replace('/metadata_uri', '') - return url.replace('/metadata', '') + return url.replace("/metadata_uri", "") + return url.replace("/metadata", "") diff --git a/geonode/client/templatetags/client_lib_tags.py b/geonode/client/templatetags/client_lib_tags.py index 9bebb529c2a..3ce659a32bf 100644 --- a/geonode/client/templatetags/client_lib_tags.py +++ b/geonode/client/templatetags/client_lib_tags.py @@ -132,111 +132,65 @@ def render(self, context): t = None # LAYERS - if self.tag_name == 'get_dataset_list': - t = context.template.engine.get_template( - hookset.dataset_list_template( - context=context)) - elif self.tag_name == 'get_dataset_detail': - t = context.template.engine.get_template('geonode-mapstore-client/legacy/dataset_detail.html') - elif self.tag_name == 'get_dataset_new': - t = context.template.engine.get_template( - hookset.dataset_new_template( - context=context)) - elif self.tag_name == 'get_dataset_view': - t = context.template.engine.get_template( - hookset.dataset_view_template( - context=context)) - elif self.tag_name == 'get_dataset_edit': - t = context.template.engine.get_template( - hookset.dataset_edit_template( - context=context)) - elif self.tag_name == 'get_dataset_update': - t = context.template.engine.get_template( - hookset.dataset_update_template( - context=context)) - elif self.tag_name == 'get_dataset_embed': - t = context.template.engine.get_template( - hookset.dataset_embed_template( - context=context)) - elif self.tag_name == 'get_dataset_download': - t = context.template.engine.get_template( - hookset.dataset_download_template( - context=context)) - elif self.tag_name == 'get_dataset_style_edit': - t = context.template.engine.get_template( - hookset.dataset_style_edit_template( - context=context)) + if self.tag_name == "get_dataset_list": + t = context.template.engine.get_template(hookset.dataset_list_template(context=context)) + elif self.tag_name == "get_dataset_detail": + t = context.template.engine.get_template("geonode-mapstore-client/legacy/dataset_detail.html") + elif self.tag_name == "get_dataset_new": + t = context.template.engine.get_template(hookset.dataset_new_template(context=context)) + elif self.tag_name == "get_dataset_view": + t = context.template.engine.get_template(hookset.dataset_view_template(context=context)) + elif self.tag_name == "get_dataset_edit": + t = context.template.engine.get_template(hookset.dataset_edit_template(context=context)) + elif self.tag_name == "get_dataset_update": + t = context.template.engine.get_template(hookset.dataset_update_template(context=context)) + elif self.tag_name == "get_dataset_embed": + t = context.template.engine.get_template(hookset.dataset_embed_template(context=context)) + elif self.tag_name == "get_dataset_download": + t = context.template.engine.get_template(hookset.dataset_download_template(context=context)) + elif self.tag_name == "get_dataset_style_edit": + t = context.template.engine.get_template(hookset.dataset_style_edit_template(context=context)) # MAPS - if self.tag_name == 'get_map_list': - t = context.template.engine.get_template( - hookset.map_list_template( - context=context)) - elif self.tag_name == 'get_map_detail': - t = context.template.engine.get_template( - hookset.map_detail_template( - context=context)) - elif self.tag_name == 'get_map_new': - t = context.template.engine.get_template( - hookset.map_new_template( - context=context)) - elif self.tag_name == 'get_map_view': - t = context.template.engine.get_template( - hookset.map_view_template( - context=context)) - elif self.tag_name == 'get_map_edit': - t = context.template.engine.get_template( - hookset.map_edit_template( - context=context)) - elif self.tag_name == 'get_map_update': - t = context.template.engine.get_template( - hookset.map_update_template( - context=context)) - elif self.tag_name == 'get_map_embed': - t = context.template.engine.get_template('geonode-mapstore-client/map_embed.html') - elif self.tag_name == 'get_map_download': - t = context.template.engine.get_template( - hookset.map_download_template( - context=context)) + if self.tag_name == "get_map_list": + t = context.template.engine.get_template(hookset.map_list_template(context=context)) + elif self.tag_name == "get_map_detail": + t = context.template.engine.get_template(hookset.map_detail_template(context=context)) + elif self.tag_name == "get_map_new": + t = context.template.engine.get_template(hookset.map_new_template(context=context)) + elif self.tag_name == "get_map_view": + t = context.template.engine.get_template(hookset.map_view_template(context=context)) + elif self.tag_name == "get_map_edit": + t = context.template.engine.get_template(hookset.map_edit_template(context=context)) + elif self.tag_name == "get_map_update": + t = context.template.engine.get_template(hookset.map_update_template(context=context)) + elif self.tag_name == "get_map_embed": + t = context.template.engine.get_template("geonode-mapstore-client/map_embed.html") + elif self.tag_name == "get_map_download": + t = context.template.engine.get_template(hookset.map_download_template(context=context)) # GEONODE_APPS - if self.tag_name == 'get_geoapp_list': - t = context.template.engine.get_template( - hookset.geoapp_list_template( - context=context)) - elif self.tag_name == 'get_geoapp_detail': - t = context.template.engine.get_template( - hookset.geoapp_detail_template( - context=context)) - elif self.tag_name == 'get_geoapp_new': - t = context.template.engine.get_template( - hookset.geoapp_new_template( - context=context)) - elif self.tag_name == 'get_geoapp_view': - t = context.template.engine.get_template( - hookset.geoapp_view_template( - context=context)) - elif self.tag_name == 'get_geoapp_edit': - t = context.template.engine.get_template( - hookset.geoapp_edit_template( - context=context)) - elif self.tag_name == 'get_geoapp_update': - t = context.template.engine.get_template( - hookset.geoapp_update_template( - context=context)) - elif self.tag_name == 'get_geoapp_embed': - t = context.template.engine.get_template( - hookset.geoapp_embed_template( - context=context)) - elif self.tag_name == 'get_geoapp_download': - t = context.template.engine.get_template( - hookset.geoapp_download_template( - context=context)) + if self.tag_name == "get_geoapp_list": + t = context.template.engine.get_template(hookset.geoapp_list_template(context=context)) + elif self.tag_name == "get_geoapp_detail": + t = context.template.engine.get_template(hookset.geoapp_detail_template(context=context)) + elif self.tag_name == "get_geoapp_new": + t = context.template.engine.get_template(hookset.geoapp_new_template(context=context)) + elif self.tag_name == "get_geoapp_view": + t = context.template.engine.get_template(hookset.geoapp_view_template(context=context)) + elif self.tag_name == "get_geoapp_edit": + t = context.template.engine.get_template(hookset.geoapp_edit_template(context=context)) + elif self.tag_name == "get_geoapp_update": + t = context.template.engine.get_template(hookset.geoapp_update_template(context=context)) + elif self.tag_name == "get_geoapp_embed": + t = context.template.engine.get_template(hookset.geoapp_embed_template(context=context)) + elif self.tag_name == "get_geoapp_download": + t = context.template.engine.get_template(hookset.geoapp_download_template(context=context)) if t: return t.render(context) else: - return '' + return "" def do_get_client_library_template(parser, token): @@ -244,30 +198,30 @@ def do_get_client_library_template(parser, token): return GeoNodeClientLibraryTag(tag_name, args, kwargs) -register.tag('get_dataset_list', do_get_client_library_template) -register.tag('get_dataset_detail', do_get_client_library_template) -register.tag('get_dataset_new', do_get_client_library_template) -register.tag('get_dataset_view', do_get_client_library_template) -register.tag('get_dataset_edit', do_get_client_library_template) -register.tag('get_dataset_update', do_get_client_library_template) -register.tag('get_dataset_embed', do_get_client_library_template) -register.tag('get_dataset_download', do_get_client_library_template) -register.tag('get_dataset_style_edit', do_get_client_library_template) - -register.tag('get_map_list', do_get_client_library_template) -register.tag('get_map_detail', do_get_client_library_template) -register.tag('get_map_new', do_get_client_library_template) -register.tag('get_map_view', do_get_client_library_template) -register.tag('get_map_edit', do_get_client_library_template) -register.tag('get_map_update', do_get_client_library_template) -register.tag('get_map_embed', do_get_client_library_template) -register.tag('get_map_download', do_get_client_library_template) - -register.tag('get_geoapp_list', do_get_client_library_template) -register.tag('get_geoapp_detail', do_get_client_library_template) -register.tag('get_geoapp_new', do_get_client_library_template) -register.tag('get_geoapp_view', do_get_client_library_template) -register.tag('get_geoapp_edit', do_get_client_library_template) -register.tag('get_geoapp_update', do_get_client_library_template) -register.tag('get_geoapp_embed', do_get_client_library_template) -register.tag('get_geoapp_download', do_get_client_library_template) +register.tag("get_dataset_list", do_get_client_library_template) +register.tag("get_dataset_detail", do_get_client_library_template) +register.tag("get_dataset_new", do_get_client_library_template) +register.tag("get_dataset_view", do_get_client_library_template) +register.tag("get_dataset_edit", do_get_client_library_template) +register.tag("get_dataset_update", do_get_client_library_template) +register.tag("get_dataset_embed", do_get_client_library_template) +register.tag("get_dataset_download", do_get_client_library_template) +register.tag("get_dataset_style_edit", do_get_client_library_template) + +register.tag("get_map_list", do_get_client_library_template) +register.tag("get_map_detail", do_get_client_library_template) +register.tag("get_map_new", do_get_client_library_template) +register.tag("get_map_view", do_get_client_library_template) +register.tag("get_map_edit", do_get_client_library_template) +register.tag("get_map_update", do_get_client_library_template) +register.tag("get_map_embed", do_get_client_library_template) +register.tag("get_map_download", do_get_client_library_template) + +register.tag("get_geoapp_list", do_get_client_library_template) +register.tag("get_geoapp_detail", do_get_client_library_template) +register.tag("get_geoapp_new", do_get_client_library_template) +register.tag("get_geoapp_view", do_get_client_library_template) +register.tag("get_geoapp_edit", do_get_client_library_template) +register.tag("get_geoapp_update", do_get_client_library_template) +register.tag("get_geoapp_embed", do_get_client_library_template) +register.tag("get_geoapp_download", do_get_client_library_template) diff --git a/geonode/compat.py b/geonode/compat.py index 82edb00ff4e..411c2ccb76c 100644 --- a/geonode/compat.py +++ b/geonode/compat.py @@ -25,6 +25,7 @@ def ensure_string(payload_bytes): import re + _payload = payload_bytes try: _payload = payload_bytes.decode("utf-8") @@ -35,6 +36,6 @@ def ensure_string(payload_bytes): # when payload is a byte-like object (e.g bytearray) # primarily used in when _payload is an image return _payload - if re.match(r'b\'(.*)\'', _payload): - _payload = re.match(r'b\'(.*)\'', _payload).groups()[0] + if re.match(r"b\'(.*)\'", _payload): + _payload = re.match(r"b\'(.*)\'", _payload).groups()[0] return _payload diff --git a/geonode/context_processors.py b/geonode/context_processors.py index ec0e997eeac..e4ce5757521 100644 --- a/geonode/context_processors.py +++ b/geonode/context_processors.py @@ -34,15 +34,17 @@ def resource_urls(request): """Global values to pass to templates""" site = Site.objects.get_current() - thesaurus = Thesaurus.objects.filter(facet=True).all().order_by('order', 'id') - if hasattr(settings, 'THESAURUS'): + thesaurus = Thesaurus.objects.filter(facet=True).all().order_by("order", "id") + if hasattr(settings, "THESAURUS"): warnings.warn( - 'Thesaurus settings is going to be' - 'deprecated in the future versions, please move the settings to ' - 'the new configuration ', FutureWarning) + "Thesaurus settings is going to be" + "deprecated in the future versions, please move the settings to " + "the new configuration ", + FutureWarning, + ) defaults = dict( STATIC_URL=settings.STATIC_URL, - CATALOGUE_BASE_URL=default_catalogue_backend()['URL'], + CATALOGUE_BASE_URL=default_catalogue_backend()["URL"], ACCOUNT_OPEN_SIGNUP=settings.ACCOUNT_OPEN_SIGNUP, ACCOUNT_APPROVAL_REQUIRED=settings.ACCOUNT_APPROVAL_REQUIRED, VERSION=get_version(), @@ -52,145 +54,56 @@ def resource_urls(request): INSTALLED_APPS=settings.INSTALLED_APPS, THEME_ACCOUNT_CONTACT_EMAIL=settings.THEME_ACCOUNT_CONTACT_EMAIL, TINYMCE_DEFAULT_CONFIG=settings.TINYMCE_DEFAULT_CONFIG, - DEBUG_STATIC=getattr( - settings, - "DEBUG_STATIC", - False), - PROXY_URL=getattr( - settings, - 'PROXY_URL', - '/proxy/?url='), - DISPLAY_SOCIAL=getattr( - settings, - 'DISPLAY_SOCIAL', - False), - DISPLAY_RATINGS=getattr( - settings, - 'DISPLAY_RATINGS', - False), - DISPLAY_WMS_LINKS=getattr( - settings, - 'DISPLAY_WMS_LINKS', - True), - CREATE_LAYER=getattr( - settings, - 'CREATE_LAYER', - True), - TWITTER_CARD=getattr( - settings, - 'TWITTER_CARD', - False), - TWITTER_SITE=getattr( - settings, - 'TWITTER_SITE', - '@GeoNode'), - TWITTER_HASHTAGS=getattr( - settings, - 'TWITTER_HASHTAGS', - []), - OPENGRAPH_ENABLED=getattr( - settings, - 'OPENGRAPH_ENABLED', - False), - ADMIN_MODERATE_UPLOADS=getattr( - settings, - 'ADMIN_MODERATE_UPLOADS', - False), - TOPICCATEGORY_MANDATORY=getattr( - settings, - 'TOPICCATEGORY_MANDATORY', - False), - GROUP_MANDATORY_RESOURCES=getattr( - settings, - 'GROUP_MANDATORY_RESOURCES', - False), - GROUP_PRIVATE_RESOURCES=getattr( - settings, - 'GROUP_PRIVATE_RESOURCES', - False), - RESOURCE_PUBLISHING=getattr( - settings, - 'RESOURCE_PUBLISHING', - False), - HAYSTACK_SEARCH=getattr( - settings, - 'HAYSTACK_SEARCH', - False), - SKIP_PERMS_FILTER=getattr( - settings, - 'SKIP_PERMS_FILTER', - False), - HAYSTACK_FACET_COUNTS=getattr( - settings, - 'HAYSTACK_FACET_COUNTS', - False), - CLIENT_RESULTS_LIMIT=getattr( - settings, - 'CLIENT_RESULTS_LIMIT', - 10), - API_LIMIT_PER_PAGE=getattr( - settings, - 'API_LIMIT_PER_PAGE', - 20), - SRID_DETAIL=getattr( - settings, - 'SRID', - dict()).get( - 'DETAIL', - 'never'), - LICENSES_ENABLED=getattr( - settings, - 'LICENSES', - dict()).get( - 'ENABLED', - False), - LICENSES_DETAIL=getattr( - settings, - 'LICENSES', - dict()).get( - 'DETAIL', - 'never'), - LICENSES_METADATA=getattr( - settings, - 'LICENSES', - dict()).get( - 'METADATA', - 'never'), - USE_GEOSERVER=getattr(settings, 'USE_GEOSERVER', False), + DEBUG_STATIC=getattr(settings, "DEBUG_STATIC", False), + PROXY_URL=getattr(settings, "PROXY_URL", "/proxy/?url="), + DISPLAY_SOCIAL=getattr(settings, "DISPLAY_SOCIAL", False), + DISPLAY_RATINGS=getattr(settings, "DISPLAY_RATINGS", False), + DISPLAY_WMS_LINKS=getattr(settings, "DISPLAY_WMS_LINKS", True), + CREATE_LAYER=getattr(settings, "CREATE_LAYER", True), + TWITTER_CARD=getattr(settings, "TWITTER_CARD", False), + TWITTER_SITE=getattr(settings, "TWITTER_SITE", "@GeoNode"), + TWITTER_HASHTAGS=getattr(settings, "TWITTER_HASHTAGS", []), + OPENGRAPH_ENABLED=getattr(settings, "OPENGRAPH_ENABLED", False), + ADMIN_MODERATE_UPLOADS=getattr(settings, "ADMIN_MODERATE_UPLOADS", False), + TOPICCATEGORY_MANDATORY=getattr(settings, "TOPICCATEGORY_MANDATORY", False), + GROUP_MANDATORY_RESOURCES=getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + GROUP_PRIVATE_RESOURCES=getattr(settings, "GROUP_PRIVATE_RESOURCES", False), + RESOURCE_PUBLISHING=getattr(settings, "RESOURCE_PUBLISHING", False), + HAYSTACK_SEARCH=getattr(settings, "HAYSTACK_SEARCH", False), + SKIP_PERMS_FILTER=getattr(settings, "SKIP_PERMS_FILTER", False), + HAYSTACK_FACET_COUNTS=getattr(settings, "HAYSTACK_FACET_COUNTS", False), + CLIENT_RESULTS_LIMIT=getattr(settings, "CLIENT_RESULTS_LIMIT", 10), + API_LIMIT_PER_PAGE=getattr(settings, "API_LIMIT_PER_PAGE", 20), + SRID_DETAIL=getattr(settings, "SRID", dict()).get("DETAIL", "never"), + LICENSES_ENABLED=getattr(settings, "LICENSES", dict()).get("ENABLED", False), + LICENSES_DETAIL=getattr(settings, "LICENSES", dict()).get("DETAIL", "never"), + LICENSES_METADATA=getattr(settings, "LICENSES", dict()).get("METADATA", "never"), + USE_GEOSERVER=getattr(settings, "USE_GEOSERVER", False), USE_NOTIFICATIONS=has_notifications, USE_MONITORING=settings.MONITORING_ENABLED, - DEFAULT_ANONYMOUS_VIEW_PERMISSION=getattr(settings, 'DEFAULT_ANONYMOUS_VIEW_PERMISSION', False), - DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=getattr(settings, 'DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION', False), - EXIF_ENABLED=getattr( - settings, - "EXIF_ENABLED", - False), - FAVORITE_ENABLED=getattr( - settings, - "FAVORITE_ENABLED", - False), - SEARCH_FILTERS=getattr( - settings, - 'SEARCH_FILTERS', - False - ), - THESAURI_FILTERS=[t['name'] for t in [settings.THESAURUS, ] if - t.get('filter')] if hasattr(settings, 'THESAURUS') else [t.identifier for t in thesaurus], - MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS=getattr( - settings, 'MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS', False - ), - SHOW_PROFILE_EMAIL=getattr( - settings, - "SHOW_PROFILE_EMAIL", - False - ), - OGC_SERVER=getattr(settings, 'OGC_SERVER', None), - DELAYED_SECURITY_SIGNALS=getattr(settings, 'DELAYED_SECURITY_SIGNALS', False), - READ_ONLY_MODE=getattr(Configuration.load(), 'read_only', False), + DEFAULT_ANONYMOUS_VIEW_PERMISSION=getattr(settings, "DEFAULT_ANONYMOUS_VIEW_PERMISSION", False), + DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=getattr(settings, "DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION", False), + EXIF_ENABLED=getattr(settings, "EXIF_ENABLED", False), + FAVORITE_ENABLED=getattr(settings, "FAVORITE_ENABLED", False), + SEARCH_FILTERS=getattr(settings, "SEARCH_FILTERS", False), + THESAURI_FILTERS=[ + t["name"] + for t in [ + settings.THESAURUS, + ] + if t.get("filter") + ] + if hasattr(settings, "THESAURUS") + else [t.identifier for t in thesaurus], + MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS=getattr(settings, "MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS", False), + SHOW_PROFILE_EMAIL=getattr(settings, "SHOW_PROFILE_EMAIL", False), + OGC_SERVER=getattr(settings, "OGC_SERVER", None), + DELAYED_SECURITY_SIGNALS=getattr(settings, "DELAYED_SECURITY_SIGNALS", False), + READ_ONLY_MODE=getattr(Configuration.load(), "read_only", False), # GeoNode Apps - GEONODE_APPS_ENABLE=getattr(settings, 'GEONODE_APPS_ENABLE', False), - GEONODE_APPS_NAME=getattr(settings, 'GEONODE_APPS_NAME', 'Apps'), - GEONODE_APPS_NAV_MENU_ENABLE=getattr(settings, 'GEONODE_APPS_NAV_MENU_ENABLE', False), + GEONODE_APPS_ENABLE=getattr(settings, "GEONODE_APPS_ENABLE", False), + GEONODE_APPS_NAME=getattr(settings, "GEONODE_APPS_NAME", "Apps"), + GEONODE_APPS_NAV_MENU_ENABLE=getattr(settings, "GEONODE_APPS_NAV_MENU_ENABLE", False), CATALOG_METADATA_TEMPLATE=getattr(settings, "CATALOG_METADATA_TEMPLATE", "catalogue/full_metadata.xml"), UI_REQUIRED_FIELDS=getattr(settings, "UI_REQUIRED_FIELDS", []), REQ_THESAURI=[ diff --git a/geonode/custom_translations.py b/geonode/custom_translations.py index 9ab50b0ab89..28ca9be0d09 100644 --- a/geonode/custom_translations.py +++ b/geonode/custom_translations.py @@ -23,67 +23,101 @@ from django.utils.translation import ugettext as _ texts = [ - _("Flora and/or fauna in natural environment. Examples: wildlife, vegetation, biological sciences, ecology, " - "wilderness, sealife, wetlands, habitat."), + _( + "Flora and/or fauna in natural environment. Examples: wildlife, vegetation, biological sciences, ecology, " + "wilderness, sealife, wetlands, habitat." + ), _("Biota"), _("Legal land descriptions. Examples: political and administrative boundaries."), _("Boundaries"), - _("Processes and phenomena of the atmosphere. Examples: cloud cover, weather, climate, atmospheric conditions, " - "climate change, precipitation."), + _( + "Processes and phenomena of the atmosphere. Examples: cloud cover, weather, climate, atmospheric conditions, " + "climate change, precipitation." + ), _("Climatology Meteorology Atmosphere"), - _("Economic activities, conditions and employment. Examples: production, labour, revenue, commerce, industry, " - "tourism and ecotourism, forestry, fisheries, commercial or subsistence hunting, exploration and exploitation " - "of resources such as minerals, oil and gas."), + _( + "Economic activities, conditions and employment. Examples: production, labour, revenue, commerce, industry, " + "tourism and ecotourism, forestry, fisheries, commercial or subsistence hunting, exploration and exploitation " + "of resources such as minerals, oil and gas." + ), _("Economy"), - _("Height above or below sea level. Examples: altitude, bathymetry, digital elevation models, slope, " - "derived products."), + _( + "Height above or below sea level. Examples: altitude, bathymetry, digital elevation models, slope, " + "derived products." + ), _("Elevation"), - _("Environmental resources, protection and conservation. Examples: environmental pollution, waste storage and " - "treatment, environmental impact assessment, monitoring environmental risk, nature reserves, landscape."), + _( + "Environmental resources, protection and conservation. Examples: environmental pollution, waste storage and " + "treatment, environmental impact assessment, monitoring environmental risk, nature reserves, landscape." + ), _("Environment"), - _("Rearing of animals and/or cultivation of plants. Examples: agriculture, irrigation, aquaculture, " - "plantations, herding, pests and diseases affecting crops and livestock."), + _( + "Rearing of animals and/or cultivation of plants. Examples: agriculture, irrigation, aquaculture, " + "plantations, herding, pests and diseases affecting crops and livestock." + ), _("Farming"), - _("Information pertaining to earth sciences. Examples: geophysical features and processes, geology, minerals, " - "sciences dealing with the composition, structure and origin of the earth s rocks, risks of earthquakes, " - "volcanic activity, landslides, gravity information, soils, permafrost, hydrogeology, erosion."), + _( + "Information pertaining to earth sciences. Examples: geophysical features and processes, geology, minerals, " + "sciences dealing with the composition, structure and origin of the earth s rocks, risks of earthquakes, " + "volcanic activity, landslides, gravity information, soils, permafrost, hydrogeology, erosion." + ), _("Geoscientific Information"), - _("Health, health services, human ecology, and safety. Examples: disease and illness, factors affecting health, " - "hygiene, substance abuse, mental and physical health, health services."), + _( + "Health, health services, human ecology, and safety. Examples: disease and illness, factors affecting health, " + "hygiene, substance abuse, mental and physical health, health services." + ), _("Health"), _("Base maps. Examples: land cover, topographic maps, imagery, unclassified images, annotations."), _("Imagery Base Maps Earth Cover"), - _("Inland water features, drainage systems and their characteristics. Examples: rivers and glaciers, salt lakes, " - "water utilization plans, dams, currents, floods, water quality, hydrographic charts."), + _( + "Inland water features, drainage systems and their characteristics. Examples: rivers and glaciers, salt lakes, " + "water utilization plans, dams, currents, floods, water quality, hydrographic charts." + ), _("Inland Waters"), - _("Military bases, structures, activities. Examples: barracks, training grounds, military transportation, " - "information collection."), + _( + "Military bases, structures, activities. Examples: barracks, training grounds, military transportation, " + "information collection." + ), _("Intelligence Military"), - _("Positional information and services. Examples: addresses, geodetic networks, control points, postal zones " - "and services, place names."), + _( + "Positional information and services. Examples: addresses, geodetic networks, control points, postal zones " + "and services, place names." + ), _("Location"), - _("Features and characteristics of salt water bodies (excluding inland waters). Examples: tides, tidal waves, " - "coastal information, reefs."), + _( + "Features and characteristics of salt water bodies (excluding inland waters). Examples: tides, tidal waves, " + "coastal information, reefs." + ), _("Oceans"), - _("Information used for appropriate actions for future use of the land. Examples: land use maps, zoning maps, " - "cadastral surveys, land ownership."), + _( + "Information used for appropriate actions for future use of the land. Examples: land use maps, zoning maps, " + "cadastral surveys, land ownership." + ), _("Planning Cadastre"), - _("Settlements, anthropology, archaeology, education, traditional beliefs, manners and customs, demographic " - "data, recreational areas and activities, social impact assessments, crime and justice, census information. " - "Economic activities, conditions and employment."), + _( + "Settlements, anthropology, archaeology, education, traditional beliefs, manners and customs, demographic " + "data, recreational areas and activities, social impact assessments, crime and justice, census information. " + "Economic activities, conditions and employment." + ), _("Population"), - _("Characteristics of society and cultures. Examples: settlements, anthropology, archaeology, education, " - "traditional beliefs, manners and customs, demographic data, recreational areas and activities, social impact " - "assessments, crime and justice, census information."), + _( + "Characteristics of society and cultures. Examples: settlements, anthropology, archaeology, education, " + "traditional beliefs, manners and customs, demographic data, recreational areas and activities, social impact " + "assessments, crime and justice, census information." + ), _("Society"), _("Man-made construction. Examples: buildings, museums, churches, factories, housing, monuments, shops, towers."), _("Structure"), - _("Means and aids for conveying persons and/or goods. Examples: roads, airports/airstrips, shipping routes, " - "tunnels, nautical charts, vehicle or vessel location, aeronautical charts, railways."), + _( + "Means and aids for conveying persons and/or goods. Examples: roads, airports/airstrips, shipping routes, " + "tunnels, nautical charts, vehicle or vessel location, aeronautical charts, railways." + ), _("Transportation"), - _("Energy, water and waste systems and communications infrastructure and services. Examples: hydroelectricity, " - "geothermal, solar and nuclear sources of energy, water purification and distribution, sewage collection and " - "disposal, electricity and gas distribution, data communication, telecommunication, radio, communication " - "networks."), - _("Utilities Communication") + _( + "Energy, water and waste systems and communications infrastructure and services. Examples: hydroelectricity, " + "geothermal, solar and nuclear sources of energy, water purification and distribution, sewage collection and " + "disposal, electricity and gas distribution, data communication, telecommunication, radio, communication " + "networks." + ), + _("Utilities Communication"), ] diff --git a/geonode/decorators.py b/geonode/decorators.py index 345754c922a..6f356167f81 100644 --- a/geonode/decorators.py +++ b/geonode/decorators.py @@ -30,9 +30,7 @@ from django.utils.decorators import classonlymethod from django.core.exceptions import PermissionDenied -from geonode.utils import (check_ogc_backend, - get_client_ip, - get_client_host) +from geonode.utils import check_ogc_backend, get_client_ip, get_client_host logger = logging.getLogger(__name__) @@ -47,14 +45,16 @@ def on_ogc_backend(backend_package): Useful to decorate features/tests that only available for specific backend. """ - def decorator(func): + def decorator(func): @wraps(func) def wrapper(*args, **kwargs): on_backend = check_ogc_backend(backend_package) if on_backend: return func(*args, **kwargs) + return wrapper + return decorator @@ -72,13 +72,13 @@ def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): # They are not logged in. See if they provided login credentials # - if 'HTTP_AUTHORIZATION' in request.META: - basic_auth = request.META['HTTP_AUTHORIZATION'].split() + if "HTTP_AUTHORIZATION" in request.META: + basic_auth = request.META["HTTP_AUTHORIZATION"].split() if len(basic_auth) == 2: # NOTE: We are only support basic authentication for now. # if basic_auth[0].lower() == "basic": - uname, passwd = base64.b64decode(basic_auth[1]).decode('utf-8').split(':', 1) + uname, passwd = base64.b64decode(basic_auth[1]).decode("utf-8").split(":", 1) user = authenticate(username=uname, password=passwd) if user and user.is_active: login(request, user) @@ -92,7 +92,7 @@ def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): # response = HttpResponse() response.status_code = 401 - response['WWW-Authenticate'] = f'Basic realm="{realm}"' + response["WWW-Authenticate"] = f'Basic realm="{realm}"' return response @@ -102,10 +102,10 @@ def view_decorator(fdec, subclass=False): https://github.com/lqc/django/tree/cbvdecoration_ticket14512 """ + def decorator(cls): if not hasattr(cls, "as_view"): - raise TypeError( - "You should only decorate subclasses of View, not mixins.") + raise TypeError("You should only decorate subclasses of View, not mixins.") if subclass: cls = type(f"{cls.__name__}WithDecorator({fdec.__name__})", (cls,), {}) original = cls.as_view.__func__ @@ -113,8 +113,10 @@ def decorator(cls): @wraps(original) def as_view(current, **initkwargs): return fdec(original(current, **initkwargs)) + cls.as_view = classonlymethod(as_view) return cls + return decorator @@ -132,8 +134,8 @@ def view_or_apiauth(view, request, test_func, *args, **kwargs): # They are not logged in. See if they provided login credentials # - if 'HTTP_AUTHORIZATION' in request.META: - _auth = request.META['HTTP_AUTHORIZATION'].split() + if "HTTP_AUTHORIZATION" in request.META: + _auth = request.META["HTTP_AUTHORIZATION"].split() if len(_auth) == 2: # NOTE: We are only support basic authentication for now. # @@ -164,12 +166,13 @@ def your_view: ... """ + def view_decorator(func): def wrapper(request, *args, **kwargs): - return view_or_basicauth(func, request, - lambda u: u.has_perm(perm), - realm, *args, **kwargs) + return view_or_basicauth(func, request, lambda u: u.has_perm(perm), realm, *args, **kwargs) + return wrapper + return view_decorator @@ -192,41 +195,40 @@ def my_view(request): ) -------------------------------------------------------------------------- """ + def _inner(request, *args, **kwargs): if not auth.get_user(request).is_superuser and not auth.get_user(request).is_staff: raise PermissionDenied return function(request, *args, **kwargs) + return _inner def check_keyword_write_perms(function): def _inner(request, *args, **kwargs): - keyword_readonly = settings.FREETEXT_KEYWORDS_READONLY and request.method == "POST" \ - and not auth.get_user(request).is_superuser + keyword_readonly = ( + settings.FREETEXT_KEYWORDS_READONLY and request.method == "POST" and not auth.get_user(request).is_superuser + ) request.keyword_readonly = keyword_readonly - if keyword_readonly and 'resource-keywords' in request.POST: + if keyword_readonly and "resource-keywords" in request.POST: return HttpResponse( - "Unauthorized: Cannot edit/create Free-text Keywords", - status=401, - content_type="application/json" + "Unauthorized: Cannot edit/create Free-text Keywords", status=401, content_type="application/json" ) return function(request, *args, **kwargs) + return _inner def superuser_protected(function): - """Decorator that forces a view to be accessible by SUPERUSERS only. - """ + """Decorator that forces a view to be accessible by SUPERUSERS only.""" + def _inner(request, *args, **kwargs): if not auth.get_user(request).is_superuser: return HttpResponse( - json.dumps({ - 'error': 'unauthorized_request' - }), - status=403, - content_type="application/json" + json.dumps({"error": "unauthorized_request"}), status=403, content_type="application/json" ) return function(request, *args, **kwargs) + return _inner @@ -234,18 +236,17 @@ def whitelist_protected(function): """Decorator that forces a view to be accessible by WHITE_LISTED IPs only. """ + def _inner(request, *args, **kwargs): - if not settings.AUTH_IP_WHITELIST or \ - (get_client_ip(request) not in settings.AUTH_IP_WHITELIST and - get_client_host(request) not in settings.AUTH_IP_WHITELIST): + if not settings.AUTH_IP_WHITELIST or ( + get_client_ip(request) not in settings.AUTH_IP_WHITELIST + and get_client_host(request) not in settings.AUTH_IP_WHITELIST + ): return HttpResponse( - json.dumps({ - 'error': 'unauthorized_request' - }), - status=403, - content_type="application/json" + json.dumps({"error": "unauthorized_request"}), status=403, content_type="application/json" ) return function(request, *args, **kwargs) + return _inner @@ -278,34 +279,31 @@ def your_view: You can provide the name of the realm to ask for authentication within. """ + def view_decorator(func): def wrapper(request, *args, **kwargs): - return view_or_basicauth(func, request, - lambda u: u.is_authenticated, - realm, *args, **kwargs) + return view_or_basicauth(func, request, lambda u: u.is_authenticated, realm, *args, **kwargs) + return wrapper + return view_decorator def logged_in_or_apiauth(): - def view_decorator(func): def wrapper(request, *args, **kwargs): - return view_or_apiauth(func, request, - lambda u: u.is_authenticated, - *args, **kwargs) + return view_or_apiauth(func, request, lambda u: u.is_authenticated, *args, **kwargs) + return wrapper return view_decorator def superuser_or_apiauth(): - def view_decorator(func): def wrapper(request, *args, **kwargs): - return view_or_apiauth(func, request, - lambda u: u.is_superuser, - *args, **kwargs) + return view_or_apiauth(func, request, lambda u: u.is_superuser, *args, **kwargs) + return wrapper return view_decorator @@ -313,6 +311,7 @@ def wrapper(request, *args, **kwargs): def dump_func_name(func): def echo_func(*func_args, **func_kwargs): - logger.debug(f'Start func: {func.__name__}') + logger.debug(f"Start func: {func.__name__}") return func(*func_args, **func_kwargs) + return echo_func diff --git a/geonode/documents/__init__.py b/geonode/documents/__init__.py index e34e2a1acd5..527f5fb5f7d 100644 --- a/geonode/documents/__init__.py +++ b/geonode/documents/__init__.py @@ -22,14 +22,39 @@ class DocumentsAppConfig(NotificationsAppConfigBase): - name = 'geonode.documents' - NOTIFICATIONS = (("document_created", _("Document Created"), _("A Document was created"),), - ("document_updated", _("Document Updated"), _("A Document was updated"),), - ("document_approved", _("Document Approved"), _("A Document was approved by a Manager"),), - ("document_published", _("Document Published"), _("A Document was published"),), - ("document_deleted", _("Document Deleted"), _("A Document was deleted"),), - ("document_rated", _("Rating for Document"), _("A rating was given to a document"),), - ) + name = "geonode.documents" + NOTIFICATIONS = ( + ( + "document_created", + _("Document Created"), + _("A Document was created"), + ), + ( + "document_updated", + _("Document Updated"), + _("A Document was updated"), + ), + ( + "document_approved", + _("Document Approved"), + _("A Document was approved by a Manager"), + ), + ( + "document_published", + _("Document Published"), + _("A Document was published"), + ), + ( + "document_deleted", + _("Document Deleted"), + _("A Document was deleted"), + ), + ( + "document_rated", + _("Rating for Document"), + _("A rating was given to a document"), + ), + ) -default_app_config = 'geonode.documents.DocumentsAppConfig' +default_app_config = "geonode.documents.DocumentsAppConfig" diff --git a/geonode/documents/admin.py b/geonode/documents/admin.py index 282988bebbf..36511c7d331 100644 --- a/geonode/documents/admin.py +++ b/geonode/documents/admin.py @@ -29,28 +29,33 @@ class DocumentAdminForm(ResourceBaseAdminForm): class Meta(ResourceBaseAdminForm.Meta): model = Document - fields = '__all__' + fields = "__all__" # exclude = ( # 'resource', # ) class DocumentAdmin(TabbedTranslationAdmin): - list_display = ('id', - 'title', - 'date', - 'category', - 'group', - 'is_approved', - 'is_published', - 'metadata_completeness') - list_display_links = ('id',) - list_editable = ('title', 'category', 'group', 'is_approved', 'is_published') - list_filter = ('date', 'date_type', 'restriction_code_type', 'category', - 'group', 'is_approved', 'is_published',) - search_fields = ('title', 'abstract', 'purpose', - 'is_approved', 'is_published',) - date_hierarchy = 'date' + list_display = ("id", "title", "date", "category", "group", "is_approved", "is_published", "metadata_completeness") + list_display_links = ("id",) + list_editable = ("title", "category", "group", "is_approved", "is_published") + list_filter = ( + "date", + "date_type", + "restriction_code_type", + "category", + "group", + "is_approved", + "is_published", + ) + search_fields = ( + "title", + "abstract", + "purpose", + "is_approved", + "is_published", + ) + date_hierarchy = "date" form = DocumentAdminForm actions = [metadata_batch_edit] @@ -61,6 +66,7 @@ def delete_queryset(self, request, queryset): """ for obj in queryset: from geonode.resource.manager import resource_manager + resource_manager.delete(obj.uuid, instance=obj) diff --git a/geonode/documents/api/permissions.py b/geonode/documents/api/permissions.py index 4de90590b69..ef830404fa9 100644 --- a/geonode/documents/api/permissions.py +++ b/geonode/documents/api/permissions.py @@ -25,8 +25,9 @@ class DocumentPermissionsFilter(BaseFilterBackend): A filter backend that limits results to those where the requesting user has read object level permissions. """ + shortcut_kwargs = { - 'accept_global_perms': True, + "accept_global_perms": True, } def filter_queryset(self, request, queryset, view): @@ -43,17 +44,16 @@ def filter_queryset(self, request, queryset, view): # 'model_name': queryset.model._meta.model_name, # } - resources = get_objects_for_user( - user, - 'base.view_resourcebase', - **self.shortcut_kwargs - ).filter(polymorphic_ctype__model='document') + resources = get_objects_for_user(user, "base.view_resourcebase", **self.shortcut_kwargs).filter( + polymorphic_ctype__model="document" + ) obj_with_perms = get_visible_resources( resources, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) - return queryset.filter(id__in=obj_with_perms.values('id')) + return queryset.filter(id__in=obj_with_perms.values("id")) diff --git a/geonode/documents/api/serializers.py b/geonode/documents/api/serializers.py index 0ece3efbb8a..9887eced4df 100644 --- a/geonode/documents/api/serializers.py +++ b/geonode/documents/api/serializers.py @@ -26,13 +26,11 @@ class GeonodeFilePathField(DynamicComputedField): - def get_attribute(self, instance): return instance.files class DocumentFieldField(DynamicComputedField): - def get_attribute(self, instance): return instance.files @@ -49,4 +47,15 @@ class Meta: model = Document name = "document" view_name = "documents-list" - fields = ("pk", "uuid", "name", "href", "subtype", "extension", "mime_type", "executions", "file_path", "doc_file") + fields = ( + "pk", + "uuid", + "name", + "href", + "subtype", + "extension", + "mime_type", + "executions", + "file_path", + "doc_file", + ) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index aff90ad8661..2ba5e294050 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -35,18 +35,14 @@ class DocumentsApiTests(APITestCase): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] def setUp(self): - create_models(b'document') - create_models(b'map') - create_models(b'dataset') + create_models(b"document") + create_models(b"map") + create_models(b"dataset") self.admin = get_user_model().objects.get(username="admin") - self.url = reverse('documents-list') + self.url = reverse("documents-list") self.invalid_file_path = f"{settings.PROJECT_ROOT}/tests/data/thesaurus.rdf" self.valid_file_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" @@ -54,25 +50,25 @@ def test_documents(self): """ Ensure we can access the Documents list. """ - url = reverse('documents-list') + url = reverse("documents-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 9) + self.assertEqual(response.data["total"], 9) # Pagination - self.assertEqual(len(response.data['documents']), 9) + self.assertEqual(len(response.data["documents"]), 9) logger.debug(response.data) - for _l in response.data['documents']: - self.assertTrue(_l['resource_type'], 'document') + for _l in response.data["documents"]: + self.assertTrue(_l["resource_type"], "document") # Get Linked Resources List resource = Document.objects.first() - url = urljoin(f"{reverse('documents-detail', kwargs={'pk': resource.pk})}/", 'linked_resources/') + url = urljoin(f"{reverse('documents-detail', kwargs={'pk': resource.pk})}/", "linked_resources/") assign_perm("base.view_resourcebase", get_anonymous_user(), resource.get_self_resource()) - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) layers_data = response.data self.assertIsNotNone(layers_data) @@ -81,71 +77,61 @@ def test_documents(self): # logger.error(f"{json.dumps(layers_data)}") def test_creation_return_error_if_file_is_not_passed(self): - ''' + """ If file_path is not available, should raise error - ''' + """ self.client.force_login(self.admin) - payload = { - "document": { - "title": "New document", - "metadata_only": True - } + payload = {"document": {"title": "New document", "metadata_only": True}} + expected = { + "success": False, + "errors": ["A file path or a file must be speficied"], + "code": "document_exception", } - expected = {'success': False, 'errors': ['A file path or a file must be speficied'], 'code': 'document_exception'} actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(400, actual.status_code) self.assertDictEqual(expected, actual.json()) def test_creation_return_error_if_file_is_none(self): - ''' + """ If file_path is not available, should raise error - ''' + """ self.client.force_login(self.admin) - payload = { - "document": { - "title": "New document", - "metadata_only": True, - "file_path": None, - "doc_file": None - } + payload = {"document": {"title": "New document", "metadata_only": True, "file_path": None, "doc_file": None}} + expected = { + "success": False, + "errors": ["A file path or a file must be speficied"], + "code": "document_exception", } - expected = {'success': False, 'errors': ['A file path or a file must be speficied'], 'code': 'document_exception'} actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(400, actual.status_code) self.assertDictEqual(expected, actual.json()) def test_creation_should_rase_exec_for_unsupported_files(self): self.client.force_login(self.admin) - payload = { - "document": { - "title": "New document", - "metadata_only": True, - "file_path": self.invalid_file_path - } + payload = {"document": {"title": "New document", "metadata_only": True, "file_path": self.invalid_file_path}} + expected = { + "success": False, + "errors": ["The file provided is not in the supported extension file list"], + "code": "document_exception", } - expected = {'success': False, 'errors': ['The file provided is not in the supported extension file list'], 'code': 'document_exception'} actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(400, actual.status_code) self.assertDictEqual(expected, actual.json()) def test_creation_should_create_the_doc(self): - ''' + """ If file_path is not available, should raise error - ''' + """ self.client.force_login(self.admin) payload = { - "document": { - "title": "New document for testing", - "metadata_only": True, - "file_path": self.valid_file_path - } + "document": {"title": "New document for testing", "metadata_only": True, "file_path": self.valid_file_path} } actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(201, actual.status_code) cloned_path = actual.json().get("document", {}).get("file_path", "")[0] extension = actual.json().get("document", {}).get("extension", "") self.assertTrue(os.path.exists(cloned_path)) - self.assertEqual('xml', extension) + self.assertEqual("xml", extension) self.assertTrue(Document.objects.filter(title="New document for testing").exists()) if cloned_path: diff --git a/geonode/documents/api/urls.py b/geonode/documents/api/urls.py index 882f8dd855d..91a51d537c8 100644 --- a/geonode/documents/api/urls.py +++ b/geonode/documents/api/urls.py @@ -20,6 +20,6 @@ from . import views -router.register(r'documents', views.DocumentViewSet, 'documents') +router.register(r"documents", views.DocumentViewSet, "documents") urlpatterns = [] diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index c8ee34ec65f..ed304b426e3 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -52,19 +52,26 @@ class DocumentViewSet(DynamicModelViewSet): """ API endpoint that allows documents to be viewed or edited. """ - http_method_names = ['get', 'patch', 'put', 'post'] + + http_method_names = ["get", "patch", "put", "post"] authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})] + permission_classes = [ + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ] filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, - ExtentFilter, DocumentPermissionsFilter + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + DocumentPermissionsFilter, ] - queryset = Document.objects.all().order_by('-created') + queryset = Document.objects.all().order_by("-created") serializer_class = DocumentSerializer pagination_class = GeoNodeApiPagination def perform_create(self, serializer): - ''' + """ Function to create document via API v2. file_path: path to the file doc_file: the open file @@ -84,7 +91,7 @@ def perform_create(self, serializer): --form 'title="Super Title2"' \ --form 'doc_file=@"/C:/Users/user/Pictures/BcMc-a6T9IM.jpg"' \ --form 'metadata_only="False"' - ''' + """ manager = None serializer.is_valid(raise_exception=True) _has_file = serializer.validated_data.pop("file_path", None) or serializer.validated_data.pop("doc_file", None) @@ -110,7 +117,7 @@ def perform_create(self, serializer): "owner": self.request.user, "extension": extension, "files": [files.get("base_file")], - "resource_type": "document" + "resource_type": "document", } ) @@ -125,21 +132,25 @@ def perform_create(self, serializer): manager.delete_retrieved_paths() raise e - @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, - description="API endpoint allowing to retrieve the DocumentResourceLink(s).") - @action(detail=True, methods=['get']) + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the DocumentResourceLink(s).", + ) + @action(detail=True, methods=["get"]) def linked_resources(self, request, pk=None): document = self.get_object() - resources_id = document.links.all().values('object_id') + resources_id = document.links.all().values("object_id") resources = ResourceBase.objects.filter(id__in=resources_id) exclude = [] for resource in resources: - if not request.user.is_superuser and \ - not request.user.has_perm('view_resourcebase', resource.get_self_resource()): + if not request.user.is_superuser and not request.user.has_perm( + "view_resourcebase", resource.get_self_resource() + ): exclude.append(resource.id) resources = resources.exclude(id__in=exclude) paginator = GeoNodeApiPagination() - paginator.page_size = request.GET.get('page_size', 10) + paginator.page_size = request.GET.get("page_size", 10) result_page = paginator.paginate_queryset(resources, request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"resources": serializer.data}) diff --git a/geonode/documents/enumerations.py b/geonode/documents/enumerations.py index 8165a38fdc2..8b270b75558 100644 --- a/geonode/documents/enumerations.py +++ b/geonode/documents/enumerations.py @@ -21,124 +21,114 @@ # match values in settings.ALLOWED_DOCUMENT_TYPES DOCUMENT_TYPE_MAP = { - 'txt': 'text', - 'log': 'text', - 'doc': 'text', - 'docx': 'text', - 'ods': 'text', - 'odt': 'text', - 'sld': 'text', - 'qml': 'text', - 'xls': 'text', - 'xlsx': 'text', - 'xml': 'text', - - 'bm': 'image', - 'bmp': 'image', - 'dwg': 'archive', - 'dxf': 'archive', - 'fif': 'image', - 'gif': 'image', - 'jpg': 'image', - 'jpe': 'image', - 'jpeg': 'image', - 'png': 'image', - 'tif': 'archive', - 'tiff': 'archive', - 'pbm': 'archive', - - 'odp': 'presentation', - 'ppt': 'presentation', - 'pptx': 'presentation', - 'pdf': 'presentation', - - 'tar': 'archive', - 'tgz': 'archive', - 'rar': 'archive', - 'gz': 'archive', - '7z': 'archive', - 'zip': 'archive', - - 'aif': 'audio', - 'aifc': 'audio', - 'aiff': 'audio', - 'au': 'audio', - 'mp3': 'audio', - 'mpga': 'audio', - 'wav': 'audio', - - 'afl': 'video', - 'avi': 'video', - 'avs': 'video', - 'fli': 'video', - 'mp2': 'video', - 'mp4': 'video', - 'mpg': 'video', - 'ogg': 'video', - 'webm': 'video', - '3gp': 'video', - 'flv': 'video', - 'vdo': 'video', + "txt": "text", + "log": "text", + "doc": "text", + "docx": "text", + "ods": "text", + "odt": "text", + "sld": "text", + "qml": "text", + "xls": "text", + "xlsx": "text", + "xml": "text", + "bm": "image", + "bmp": "image", + "dwg": "archive", + "dxf": "archive", + "fif": "image", + "gif": "image", + "jpg": "image", + "jpe": "image", + "jpeg": "image", + "png": "image", + "tif": "archive", + "tiff": "archive", + "pbm": "archive", + "odp": "presentation", + "ppt": "presentation", + "pptx": "presentation", + "pdf": "presentation", + "tar": "archive", + "tgz": "archive", + "rar": "archive", + "gz": "archive", + "7z": "archive", + "zip": "archive", + "aif": "audio", + "aifc": "audio", + "aiff": "audio", + "au": "audio", + "mp3": "audio", + "mpga": "audio", + "wav": "audio", + "afl": "video", + "avi": "video", + "avs": "video", + "fli": "video", + "mp2": "video", + "mp4": "video", + "mpg": "video", + "ogg": "video", + "webm": "video", + "3gp": "video", + "flv": "video", + "vdo": "video", } DOCUMENT_MIMETYPE_MAP = { - 'txt': 'text/plain', - 'log': 'text/plain', - 'doc': 'application/msword', - 'docx': 'application/msword', - 'ods': 'text/plain', - 'odt': 'text/plain', - 'sld': 'text/plain', - 'qml': 'text/plain', - 'xls': 'application/excel', - 'xlsx': 'application/vnd.ms-excel', - 'xml': 'application/xml', - - 'bm': 'image/bmp', - 'bmp': 'image/bmp', - 'dwg': 'image/vnd.dwg', - 'dxf': 'image/vnd.dwg', - 'fif': 'image/fif', - 'gif': 'image/gif', - 'jpe': 'image/jpeg', - 'jpg': 'image/jpeg', - 'jpeg': 'image/jpeg', - 'png': 'image/png', - 'tif': 'image/tiff', - 'tiff': 'image/tiff', - 'pbm': 'image/x-portable-bitmap', - - 'odp': 'application/odp', - 'ppt': 'application/powerpoint', - 'pptx': 'application/x-mspowerpoint', - 'pdf': 'application/pdf', - - 'tar': 'application/x-tar', - 'tgz': 'application/x-compressed', - 'rar': 'application/x-rar-compressed', - 'gz': 'application/x-gzip', - '7z': 'application/zip', - 'zip': 'application/zip', - - 'aif': 'audio/aiff', - 'aifc': 'audio/aiff', - 'aiff': 'audio/aiff', - 'au': 'audio/basic', - 'mp3': 'audio/mp3', - 'mpga': 'audio/mpeg', - 'wav': 'audio/wav', - - 'afl': 'video/animaflex', - 'avi': 'video/avi', - 'avs': 'video/avs-video', - 'fli': 'video/fli', - 'mp2': 'video/mpeg', - 'mp4': 'video/mp4', - 'mpg': 'video/mpeg', - 'ogg': 'video/ogg', - 'webm': 'video/webm', - '3gp': 'video/3gp', - 'flv': 'video/flv', - 'vdo': 'video/vdo', + "txt": "text/plain", + "log": "text/plain", + "doc": "application/msword", + "docx": "application/msword", + "ods": "text/plain", + "odt": "text/plain", + "sld": "text/plain", + "qml": "text/plain", + "xls": "application/excel", + "xlsx": "application/vnd.ms-excel", + "xml": "application/xml", + "bm": "image/bmp", + "bmp": "image/bmp", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dwg", + "fif": "image/fif", + "gif": "image/gif", + "jpe": "image/jpeg", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "tif": "image/tiff", + "tiff": "image/tiff", + "pbm": "image/x-portable-bitmap", + "odp": "application/odp", + "ppt": "application/powerpoint", + "pptx": "application/x-mspowerpoint", + "pdf": "application/pdf", + "tar": "application/x-tar", + "tgz": "application/x-compressed", + "rar": "application/x-rar-compressed", + "gz": "application/x-gzip", + "7z": "application/zip", + "zip": "application/zip", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "au": "audio/basic", + "mp3": "audio/mp3", + "mpga": "audio/mpeg", + "wav": "audio/wav", + "afl": "video/animaflex", + "avi": "video/avi", + "avs": "video/avs-video", + "fli": "video/fli", + "mp2": "video/mpeg", + "mp4": "video/mp4", + "mpg": "video/mpeg", + "ogg": "video/ogg", + "webm": "video/webm", + "3gp": "video/3gp", + "flv": "video/flv", + "vdo": "video/vdo", } diff --git a/geonode/documents/exif/utils.py b/geonode/documents/exif/utils.py index 6980bd51f1f..44498f201b0 100644 --- a/geonode/documents/exif/utils.py +++ b/geonode/documents/exif/utils.py @@ -42,7 +42,7 @@ def convertExifDateToDjangoDate(date): int("".join(a[5:7])), int("".join(a[8:10])), int("".join(a[11:13])), - int("".join(a[14:16])) + int("".join(a[14:16])), ) @@ -55,7 +55,7 @@ def convertExifLocationToDecimalDegrees(location, direction): dd += float(s) / 3600.0 if direction: - if direction.lower() == 's' or direction.lower() == 'w': + if direction.lower() == "s" or direction.lower() == "w": dd = dd * -1.0 return dd else: @@ -74,11 +74,9 @@ def exif_extract_metadata_doc(doc): if ext[1:] in {"jpg", "jpeg"}: from PIL import Image, ExifTags + img = Image.open(doc.doc_file.path) - exif_data = { - ExifTags.TAGS[k]: v - for k, v in img._getexif().items() if k in ExifTags.TAGS - } + exif_data = {ExifTags.TAGS[k]: v for k, v in img._getexif().items() if k in ExifTags.TAGS} model = None date = None @@ -115,13 +113,13 @@ def exif_extract_metadata_doc(doc): decode = ExifTags.GPSTAGS.get(key, key) gpsinfo[decode] = exif_data["GPSInfo"][key] if "GPSLatitude" in gpsinfo and "GPSLongitude" in gpsinfo: - lat = convertExifLocationToDecimalDegrees(gpsinfo["GPSLatitude"], gpsinfo.get('GPSLatitudeRef', 'N')) - lon = convertExifLocationToDecimalDegrees(gpsinfo["GPSLongitude"], gpsinfo.get('GPSLongitudeRef', 'E')) + lat = convertExifLocationToDecimalDegrees(gpsinfo["GPSLatitude"], gpsinfo.get("GPSLatitudeRef", "N")) + lon = convertExifLocationToDecimalDegrees(gpsinfo["GPSLongitude"], gpsinfo.get("GPSLongitudeRef", "E")) bbox = [lon, lon, lat, lat] abstract = exif_build_abstract(model=model, date=date, lat=lat, lon=lon) - return {'date': date, 'keywords': keywords, 'bbox': bbox, 'abstract': abstract} + return {"date": date, "keywords": keywords, "bbox": bbox, "abstract": abstract} else: return None diff --git a/geonode/documents/forms.py b/geonode/documents/forms.py index 5d3414b3134..c970c46ab46 100644 --- a/geonode/documents/forms.py +++ b/geonode/documents/forms.py @@ -35,9 +35,7 @@ from geonode.layers.models import Dataset from geonode.base.forms import ResourceBaseForm, get_tree_data from geonode.resource.utils import get_related_resources -from geonode.documents.models import ( - Document, - DocumentResourceLink) +from geonode.documents.models import Document, DocumentResourceLink from geonode.upload.models import UploadSizeLimit from geonode.upload.api.exceptions import FileUploadLimitException @@ -64,9 +62,9 @@ def clean(self, *args, **kwargs): max_size = self._get_max_size() # Validate if file_size is not None and file_size > max_size: - raise FileUploadLimitException(_( - f'File size size exceeds {filesizeformat(max_size)}. Please try again with a smaller file.' - )) + raise FileUploadLimitException( + _(f"File size size exceeds {filesizeformat(max_size)}. Please try again with a smaller file.") + ) return data def _get_file_size(self, data): @@ -85,7 +83,6 @@ def _get_max_size(self): class DocumentFormMixin: - def generate_link_choices(self, resources=None): if resources is None: @@ -96,10 +93,7 @@ def generate_link_choices(self, resources=None): choices = [] for obj in resources: type_id = ContentType.objects.get_for_model(obj.__class__).id - choices.append([ - f"type:{type_id}-id:{obj.id}", - f"{obj.title} ({obj.polymorphic_ctype.model})" - ]) + choices.append([f"type:{type_id}-id:{obj.id}", f"{obj.title} ({obj.polymorphic_ctype.model})"]) return choices @@ -107,7 +101,7 @@ def generate_link_values(self, resources=None): choices = self.generate_link_choices(resources=resources) return [choice[0] for choice in choices] - def save_many2many(self, links_field='links'): + def save_many2many(self, links_field="links"): # create and fetch desired links instances = [] for link in self.cleaned_data[links_field]: @@ -122,49 +116,47 @@ def save_many2many(self, links_field='links'): instances.append(instance) # delete remaining links - DocumentResourceLink.objects\ - .filter(document_id=self.instance.id).exclude(pk__in=[i.pk for i in instances]).delete() + DocumentResourceLink.objects.filter(document_id=self.instance.id).exclude( + pk__in=[i.pk for i in instances] + ).delete() class DocumentForm(ResourceBaseForm, DocumentFormMixin): title = forms.CharField(required=False) - links = forms.MultipleChoiceField( - label=_("Link to"), - required=False) + links = forms.MultipleChoiceField(label=_("Link to"), required=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['regions'].choices = get_tree_data() - self.fields['links'].choices = self.generate_link_choices() - self.fields['links'].initial = self.generate_link_values( - resources=get_related_resources(self.instance) - ) + self.fields["regions"].choices = get_tree_data() + self.fields["links"].choices = self.generate_link_choices() + self.fields["links"].initial = self.generate_link_values(resources=get_related_resources(self.instance)) for field in self.fields: help_text = self.fields[field].help_text self.fields[field].help_text = None - if help_text != '': + if help_text != "": self.fields[field].widget.attrs.update( { - 'class': 'has-external-popover', - 'data-content': help_text, - 'placeholder': help_text, - 'data-placement': 'right', - 'data-container': 'body', - 'data-html': 'true' + "class": "has-external-popover", + "data-content": help_text, + "placeholder": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", } ) class Meta(ResourceBaseForm.Meta): model = Document exclude = ResourceBaseForm.Meta.exclude + ( - 'content_type', - 'object_id', - 'doc_file', - 'extension', - 'subtype', - 'doc_url') + "content_type", + "object_id", + "doc_file", + "extension", + "subtype", + "doc_url", + ) class DocumentDescriptionForm(forms.Form): @@ -178,41 +170,33 @@ class DocumentCreateForm(TranslationModelForm, DocumentFormMixin): """ The document upload form. """ + permissions = forms.CharField( - widget=HiddenInput( - attrs={ - 'name': 'permissions', - 'id': 'permissions'}), - required=False) - - links = forms.MultipleChoiceField( - label=_("Link to"), - required=False) - - doc_file = SizeRestrictedFileField( - label=_("File"), - required=False, - field_slug="document_upload_size" + widget=HiddenInput(attrs={"name": "permissions", "id": "permissions"}), required=False ) + links = forms.MultipleChoiceField(label=_("Link to"), required=False) + + doc_file = SizeRestrictedFileField(label=_("File"), required=False, field_slug="document_upload_size") + class Meta: model = Document - fields = ['title', 'doc_file', 'doc_url'] + fields = ["title", "doc_file", "doc_url"] widgets = { - 'name': HiddenInput(attrs={'cols': 80, 'rows': 20}), + "name": HiddenInput(attrs={"cols": 80, "rows": 20}), } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['links'].choices = self.generate_link_choices() + self.fields["links"].choices = self.generate_link_choices() def clean_permissions(self): """ Ensures the JSON field is JSON. """ - permissions = self.cleaned_data['permissions'] + permissions = self.cleaned_data["permissions"] - if not self.fields['permissions'].required and (permissions is None or permissions == ''): + if not self.fields["permissions"].required and (permissions is None or permissions == ""): return None try: @@ -225,8 +209,8 @@ def clean(self): Ensures the doc_file or the doc_url field is populated. """ cleaned_data = super().clean() - doc_file = self.cleaned_data.get('doc_file') - doc_url = self.cleaned_data.get('doc_url') + doc_file = self.cleaned_data.get("doc_file") + doc_url = self.cleaned_data.get("doc_url") if not doc_file and not doc_url and "doc_file" not in self.errors and "doc_url" not in self.errors: logger.error("Document must be a file or url.") @@ -234,8 +218,7 @@ def clean(self): if doc_file and doc_url: logger.error("A document cannot have both a file and a url.") - raise forms.ValidationError( - _("A document cannot have both a file and a url.")) + raise forms.ValidationError(_("A document cannot have both a file and a url.")) return cleaned_data @@ -243,11 +226,9 @@ def clean_doc_file(self): """ Ensures the doc_file is valid. """ - doc_file = self.cleaned_data.get('doc_file') + doc_file = self.cleaned_data.get("doc_file") - if doc_file and not os.path.splitext( - doc_file.name)[1].lower()[ - 1:] in settings.ALLOWED_DOCUMENT_TYPES: + if doc_file and not os.path.splitext(doc_file.name)[1].lower()[1:] in settings.ALLOWED_DOCUMENT_TYPES: logger.debug("This file type is not allowed") raise forms.ValidationError(_("This file type is not allowed")) @@ -258,22 +239,19 @@ class DocumentReplaceForm(forms.ModelForm): """ The form used to replace a document. """ - doc_file = SizeRestrictedFileField( - label=_("File"), - required=True, - field_slug="document_upload_size" - ) + + doc_file = SizeRestrictedFileField(label=_("File"), required=True, field_slug="document_upload_size") class Meta: model = Document - fields = ['doc_file'] + fields = ["doc_file"] def clean(self): """ Ensures the doc_file field is populated. """ cleaned_data = super().clean() - doc_file = self.cleaned_data.get('doc_file') + doc_file = self.cleaned_data.get("doc_file") if not doc_file: raise forms.ValidationError(_("Document must be a file.")) @@ -284,11 +262,9 @@ def clean_doc_file(self): """ Ensures the doc_file is valid. """ - doc_file = self.cleaned_data.get('doc_file') + doc_file = self.cleaned_data.get("doc_file") - if doc_file and not os.path.splitext( - doc_file.name)[1].lower()[ - 1:] in settings.ALLOWED_DOCUMENT_TYPES: + if doc_file and not os.path.splitext(doc_file.name)[1].lower()[1:] in settings.ALLOWED_DOCUMENT_TYPES: raise forms.ValidationError(_("This file type is not allowed")) return doc_file diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 938d736efd8..fbe3d44212d 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -34,13 +34,8 @@ from geonode.base.models import ResourceBase from geonode.maps.signals import map_changed_signal from geonode.groups.conf import settings as groups_settings -from geonode.documents.enumerations import ( - DOCUMENT_TYPE_MAP, - DOCUMENT_MIMETYPE_MAP) -from geonode.security.permissions import ( - VIEW_PERMISSIONS, - OWNER_PERMISSIONS, - DOWNLOAD_PERMISSIONS) +from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP +from geonode.security.permissions import VIEW_PERMISSIONS, OWNER_PERMISSIONS, DOWNLOAD_PERMISSIONS from geonode.utils import build_absolute_uri logger = logging.getLogger(__name__) @@ -58,8 +53,9 @@ class Document(ResourceBase): blank=True, null=True, max_length=255, - help_text=_('The URL of the document if it is external.'), - verbose_name=_('URL')) + help_text=_("The URL of the document if it is external."), + verbose_name=_("URL"), + ) def __str__(self): return str(self.title) @@ -72,7 +68,7 @@ def allowed_permissions(cls): return { "anonymous": VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS, "default": OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS, - groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS + groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS, } @classproperty @@ -83,7 +79,7 @@ def compact_permission_labels(cls): "download": _("View and Download"), "edit": _("Edit"), "manage": _("Manage"), - "owner": _("Owner") + "owner": _("Owner"), } @property @@ -98,17 +94,14 @@ def name_long(self): if not self.title: return str(self.id) else: - return f'{self.title} ({self.id})' + return f"{self.title} ({self.id})" @property def href(self): if self.doc_url: return self.doc_url elif self.files: - return urljoin( - settings.SITEURL, - reverse('document_link', args=(self.id,)) - ) + return urljoin(settings.SITEURL, reverse("document_link", args=(self.id,))) @property def is_file(self): @@ -122,17 +115,17 @@ def mime_type(self): @property def is_audio(self): - AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'audio'] + AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "audio"] return self.is_file and self.extension.lower() in AUDIOTYPES @property def is_image(self): - IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'image'] + IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "image"] return self.is_file and self.extension.lower() in IMGTYPES @property def is_video(self): - VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'video'] + VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "video"] return self.is_file and self.extension.lower() in VIDEOTYPES @property @@ -141,11 +134,11 @@ def class_name(self): @property def embed_url(self): - return reverse('document_embed', args=(self.id,)) + return reverse("document_embed", args=(self.id,)) @property def download_url(self): - return build_absolute_uri(reverse('document_download', args=(self.id,))) + return build_absolute_uri(reverse("document_download", args=(self.id,))) class Meta(ResourceBase.Meta): pass @@ -154,28 +147,18 @@ class Meta(ResourceBase.Meta): class DocumentResourceLink(models.Model): # relation to the document model - document = models.ForeignKey( - Document, - null=True, - blank=True, - related_name='links', - on_delete=models.CASCADE) + document = models.ForeignKey(Document, null=True, blank=True, related_name="links", on_delete=models.CASCADE) # relation to the resource model - content_type = models.ForeignKey( - ContentType, - null=True, - blank=True, - on_delete=models.CASCADE) + content_type = models.ForeignKey(ContentType, null=True, blank=True, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() - resource = GenericForeignKey('content_type', 'object_id') + resource = GenericForeignKey("content_type", "object_id") def get_related_documents(resource): if isinstance(resource, Dataset) or isinstance(resource, Map): content_type = ContentType.objects.get_for_model(resource) - return Document.objects.filter(links__content_type=content_type, - links__object_id=resource.pk) + return Document.objects.filter(links__content_type=content_type, links__object_id=resource.pk) else: return None diff --git a/geonode/documents/search_indexes.py b/geonode/documents/search_indexes.py index 9deeb35aedc..d5460936b88 100644 --- a/geonode/documents/search_indexes.py +++ b/geonode/documents/search_indexes.py @@ -25,7 +25,7 @@ class DocumentIndex(indexes.SearchIndex, indexes.Indexable): - id = indexes.IntegerField(model_attr='id') + id = indexes.IntegerField(model_attr="id") abstract = indexes.CharField(model_attr="abstract", boost=1.5) category__gn_description = indexes.CharField(model_attr="category__gn_description", null=True) csw_type = indexes.CharField(model_attr="csw_type") @@ -42,37 +42,16 @@ class DocumentIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.EdgeNgramField(document=True, use_template=True, stored=False) type = indexes.CharField(faceted=True) title_sortable = indexes.CharField(indexed=False, stored=False) # Necessary for sorting - category = indexes.CharField( - model_attr="category__identifier", - faceted=True, - null=True, - stored=True) + category = indexes.CharField(model_attr="category__identifier", faceted=True, null=True, stored=True) bbox_left = indexes.FloatField(model_attr="bbox_x0", null=True, stored=False) bbox_right = indexes.FloatField(model_attr="bbox_x1", null=True, stored=False) bbox_bottom = indexes.FloatField(model_attr="bbox_y0", null=True, stored=False) bbox_top = indexes.FloatField(model_attr="bbox_y1", null=True, stored=False) - temporal_extent_start = indexes.DateTimeField( - model_attr="temporal_extent_start", - null=True, - stored=False) - temporal_extent_end = indexes.DateTimeField( - model_attr="temporal_extent_end", - null=True, - stored=False) - keywords = indexes.MultiValueField( - model_attr="keyword_slug_list", - null=True, - faceted=True, - stored=True) - regions = indexes.MultiValueField( - model_attr="region_name_list", - null=True, - faceted=True, - stored=True) - popular_count = indexes.IntegerField( - model_attr="popular_count", - default=0, - boost=20) + temporal_extent_start = indexes.DateTimeField(model_attr="temporal_extent_start", null=True, stored=False) + temporal_extent_end = indexes.DateTimeField(model_attr="temporal_extent_end", null=True, stored=False) + keywords = indexes.MultiValueField(model_attr="keyword_slug_list", null=True, faceted=True, stored=True) + regions = indexes.MultiValueField(model_attr="region_name_list", null=True, faceted=True, stored=True) + popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) num_ratings = indexes.IntegerField(stored=False) @@ -86,10 +65,7 @@ def prepare_type(self, obj): def prepare_rating(self, obj): ct = ContentType.objects.get_for_model(obj) try: - rating = OverallRating.objects.filter( - object_id=obj.pk, - content_type=ct - ).aggregate(r=Avg("rating"))["r"] + rating = OverallRating.objects.filter(object_id=obj.pk, content_type=ct).aggregate(r=Avg("rating"))["r"] return float(str(rating or "0")) except OverallRating.DoesNotExist: return 0.0 @@ -97,10 +73,7 @@ def prepare_rating(self, obj): def prepare_num_ratings(self, obj): ct = ContentType.objects.get_for_model(obj) try: - return OverallRating.objects.filter( - object_id=obj.pk, - content_type=ct - ).all().count() + return OverallRating.objects.filter(object_id=obj.pk, content_type=ct).all().count() except OverallRating.DoesNotExist: return 0 diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index 76a3c2c2126..57b132d457b 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -33,12 +33,10 @@ logger = get_task_logger(__name__) -class DocumentRenderer(): - FILETYPES = ['pdf'] +class DocumentRenderer: + FILETYPES = ["pdf"] # See https://pillow.readthedocs.io/en/stable/reference/ImageOps.html#PIL.ImageOps.fit - CROP_CENTERING = { - 'pdf': (0.0, 0.0) - } + CROP_CENTERING = {"pdf": (0.0, 0.0)} def __init__(self) -> None: pass @@ -50,7 +48,7 @@ def render(self, filename): content = None if self.supports(filename): filetype = self._get_filetype(filename) - render = getattr(self, f'render_{filetype}') + render = getattr(self, f"render_{filetype}") content = render(filename) return content @@ -60,7 +58,7 @@ def render_pdf(self, filename): pix = doc[0].get_pixmap(matrix=fitz.Matrix(2.0, 2.0)) return pix.pil_tobytes(format="PNG") except Exception as e: - logger.warning(f'Cound not generate thumbnail for {filename}: {e}') + logger.warning(f"Cound not generate thumbnail for {filename}: {e}") return None def preferred_crop_centering(self, filename): @@ -75,16 +73,17 @@ def _get_filetype(self, filname): @app.task( bind=True, - name='geonode.documents.tasks.create_document_thumbnail', - queue='geonode', + name="geonode.documents.tasks.create_document_thumbnail", + queue="geonode", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def create_document_thumbnail(self, object_id): """ Create thumbnail for a document. @@ -104,12 +103,12 @@ def create_document_thumbnail(self, object_id): if document.is_image: dname = storage_manager.path(document.files[0]) if storage_manager.exists(dname): - image_file = storage_manager.open(dname, 'rb') + image_file = storage_manager.open(dname, "rb") try: image = Image.open(image_file) with io.BytesIO() as output: - image.save(output, format='PNG') + image.save(output, format="PNG") thumbnail_content = output.getvalue() output.close() except Exception as e: @@ -130,40 +129,44 @@ def create_document_thumbnail(self, object_id): logger.warning(f"Thumbnail for document #{object_id} empty.") ResourceBase.objects.filter(id=document.id).update(thumbnail_url=None) else: - filename = f'document-{document.uuid}-thumb.jpg' + filename = f"document-{document.uuid}-thumb.jpg" document.save_thumbnail(filename, thumbnail_content, centering=centering) logger.debug(f"Thumbnail for document #{object_id} created.") @app.task( bind=True, - name='geonode.documents.tasks.delete_orphaned_document_files', - queue='cleanup', + name="geonode.documents.tasks.delete_orphaned_document_files", + queue="cleanup", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def delete_orphaned_document_files(self): from geonode.documents.utils import delete_orphaned_document_files + delete_orphaned_document_files() @app.task( bind=True, - name='geonode.documents.tasks.delete_orphaned_thumbnails', - queue='cleanup', + name="geonode.documents.tasks.delete_orphaned_thumbnails", + queue="cleanup", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def delete_orphaned_thumbnails(self): from geonode.base.utils import delete_orphaned_thumbs + delete_orphaned_thumbs() diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 3ad6c36fbf7..aa25c82ed81 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -55,10 +55,7 @@ from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models from geonode.upload.api.exceptions import FileUploadLimitException from .forms import DocumentCreateForm @@ -66,21 +63,14 @@ class DocumentsTest(GeoNodeBaseTestSupport): - type = 'document' + type = "document" - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] perm_spec = { - "users": { - "admin": [ - "change_resourcebase", - "change_resourcebase_permissions", - "view_resourcebase"]}, - "groups": {}} + "users": {"admin": ["change_resourcebase", "change_resourcebase_permissions", "view_resourcebase"]}, + "groups": {}, + } @classmethod def setUpClass(cls): @@ -95,29 +85,30 @@ def tearDownClass(cls): def setUp(self): super().setUp() - create_models('map') + create_models("map") self.project_root = os.path.abspath(os.path.dirname(__file__)) self.imgfile = io.BytesIO( - b'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' - b'\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" + b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + ) self.anonymous_user = get_anonymous_user() def test_document_mimetypes_rendering(self): - ARCHIVETYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'archive'] - AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'audio'] - IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'image'] - VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == 'video'] + ARCHIVETYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "archive"] + AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "audio"] + IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "image"] + VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "video"] self.assertIsNotNone(ARCHIVETYPES) self.assertIsNotNone(AUDIOTYPES) self.assertIsNotNone(IMGTYPES) self.assertIsNotNone(VIDEOTYPES) # Make sure we won't have template rendering issues - self.assertTrue('dwg' in ARCHIVETYPES) - self.assertTrue('dxf' in ARCHIVETYPES) - self.assertTrue('tif' in ARCHIVETYPES) - self.assertTrue('tiff' in ARCHIVETYPES) - self.assertTrue('pbm' in ARCHIVETYPES) + self.assertTrue("dwg" in ARCHIVETYPES) + self.assertTrue("dxf" in ARCHIVETYPES) + self.assertTrue("tif" in ARCHIVETYPES) + self.assertTrue("tiff" in ARCHIVETYPES) + self.assertTrue("pbm" in ARCHIVETYPES) @patch("geonode.documents.tasks.create_document_thumbnail") def test_create_document_with_no_rel(self, thumb): @@ -126,12 +117,9 @@ def test_create_document_with_no_rel(self, thumb): f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - c = Document.objects.create( - files=f, - owner=superuser, - title='theimg') + c = Document.objects.create(files=f, owner=superuser, title="theimg") c.set_default_permissions() - self.assertEqual(Document.objects.get(pk=c.id).title, 'theimg') + self.assertEqual(Document.objects.get(pk=c.id).title, "theimg") @patch("geonode.documents.tasks.create_document_thumbnail") def test_create_document_with_rel(self, thumb): @@ -141,19 +129,13 @@ def test_create_document_with_rel(self, thumb): superuser = get_user_model().objects.get(pk=2) - c = Document.objects.create( - files=f, - owner=superuser, - title='theimg') + c = Document.objects.create(files=f, owner=superuser, title="theimg") m = Map.objects.first() ctype = ContentType.objects.get_for_model(m) - _d = DocumentResourceLink.objects.create( - document_id=c.id, - content_type=ctype, - object_id=m.id) + _d = DocumentResourceLink.objects.create(document_id=c.id, content_type=ctype, object_id=m.id) - self.assertEqual(Document.objects.get(pk=c.id).title, 'theimg') + self.assertEqual(Document.objects.get(pk=c.id).title, "theimg") self.assertEqual(DocumentResourceLink.objects.get(pk=_d.id).object_id, m.id) def test_create_document_url(self): @@ -167,7 +149,8 @@ def test_create_document_url(self): doc_url="http://geonode.org/map.pdf", owner=superuser, title="GeoNode Map", - )) + ), + ) doc = Document.objects.get(pk=c.id) self.assertEqual(doc.title, "GeoNode Map") self.assertEqual(doc.extension, "pdf") @@ -176,17 +159,18 @@ def test_create_document_url_view(self): """ Tests creating and updating external documents. """ - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") form_data = { - 'title': 'GeoNode Map', - 'permissions': '{"users":{"AnonymousUser": ["view_resourcebase"]},"groups":{}}', - 'doc_url': 'http://www.geonode.org/map.pdf'} + "title": "GeoNode Map", + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]},"groups":{}}', + "doc_url": "http://www.geonode.org/map.pdf", + } - response = self.client.post(reverse('document_upload'), data=form_data) + response = self.client.post(reverse("document_upload"), data=form_data) self.assertEqual(response.status_code, 302) - d = Document.objects.get(title='GeoNode Map') - self.assertEqual(d.doc_url, 'http://www.geonode.org/map.pdf') + d = Document.objects.get(title="GeoNode Map") + self.assertEqual(d.doc_url, "http://www.geonode.org/map.pdf") def test_upload_document_form(self): """ @@ -197,105 +181,92 @@ def test_upload_document_form(self): self.assertFalse(form.is_valid()) # title is required - self.assertTrue('title' in form.errors) + self.assertTrue("title" in form.errors) # since neither a doc_file nor a doc_url are included __all__ should be # in form.errors. - self.assertTrue('__all__' in form.errors) + self.assertTrue("__all__" in form.errors) form_data = { - 'title': 'GeoNode Map', - 'permissions': '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', - 'doc_url': 'http://www.geonode.org/map.pdf'} + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + "doc_url": "http://www.geonode.org/map.pdf", + } form = DocumentCreateForm(data=form_data) self.assertTrue(form.is_valid()) - self.assertTrue(isinstance(form.cleaned_data['permissions'], dict)) + self.assertTrue(isinstance(form.cleaned_data["permissions"], dict)) # if permissions are not JSON serializable, the field should be in # form.errors. - form_data['permissions'] = 'non-json string' - self.assertTrue( - 'permissions' in DocumentCreateForm( - data=form_data).errors) + form_data["permissions"] = "non-json string" + self.assertTrue("permissions" in DocumentCreateForm(data=form_data).errors) form_data = { - 'title': 'GeoNode Map', - 'permissions': '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', } - file_data = { - 'doc_file': SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif')} + file_data = {"doc_file": SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif")} form = DocumentCreateForm(form_data, file_data) self.assertTrue(form.is_valid()) # The form should raise a validation error when a url and file is # present. - form_data['doc_url'] = 'http://www.geonode.org/map.pdf' + form_data["doc_url"] = "http://www.geonode.org/map.pdf" form = DocumentCreateForm(form_data, file_data) self.assertFalse(form.is_valid()) - self.assertTrue('__all__' in form.errors) + self.assertTrue("__all__" in form.errors) def test_replace_document(self): - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') + f = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") response = self.client.post( - reverse('document_upload'), + reverse("document_upload"), data={ - 'title': 'File Doc', - 'doc_file': f, - 'permissions': '{"users":{"AnonymousUser": ["view_resourcebase"]}}'}, - follow=True) + "title": "File Doc", + "doc_file": f, + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]}}', + }, + follow=True, + ) self.assertEqual(response.status_code, 200) # Replace Document - d = Document.objects.get(title='File Doc') - test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) - f = SimpleUploadedFile('test_image.png', BytesIO(test_image.tobytes()).read(), 'image/png') - response = self.client.post( - reverse('document_replace', args=(d.id,)), - data={'doc_file': f} - ) + d = Document.objects.get(title="File Doc") + test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) + f = SimpleUploadedFile("test_image.png", BytesIO(test_image.tobytes()).read(), "image/png") + response = self.client.post(reverse("document_replace", args=(d.id,)), data={"doc_file": f}) self.assertEqual(response.status_code, 302) # Remove document d.delete() def test_non_image_documents_thumbnail(self): - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") try: with open(os.path.join(f"{self.project_root}", "tests/data/text.txt"), "rb") as f: - data = { - 'title': "Non img File Doc", - 'doc_file': f, - 'extension': 'txt' - } - self.client.post(reverse('document_upload'), data=data) - d = Document.objects.get(title='Non img File Doc') + data = {"title": "Non img File Doc", "doc_file": f, "extension": "txt"} + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="Non img File Doc") self.assertIsNone(d.thumbnail_url) finally: - Document.objects.filter(title='Non img File Doc').delete() + Document.objects.filter(title="Non img File Doc").delete() def test_image_documents_thumbnail(self): - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") try: # test image doc with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: data = { - 'title': "img File Doc", - 'doc_file': f, - 'extension': 'gif', + "title": "img File Doc", + "doc_file": f, + "extension": "gif", } - with self.settings(THUMBNAIL_SIZE={'width': 400, 'height': 200}): - self.client.post(reverse('document_upload'), data=data) - d = Document.objects.get(title='img File Doc') + with self.settings(THUMBNAIL_SIZE={"width": 400, "height": 200}): + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="img File Doc") self.assertIsNotNone(d.thumbnail_url) thumb_file = os.path.join( settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" @@ -303,40 +274,36 @@ def test_image_documents_thumbnail(self): file = Image.open(thumb_file) self.assertEqual(file.size, (400, 200)) # check thumbnail qualty and extention - self.assertEqual(file.format, 'JPEG') + self.assertEqual(file.format, "JPEG") # test pdf doc with open(os.path.join(f"{self.project_root}", "tests/data/pdf_doc.pdf"), "rb") as f: data = { - 'title': "Pdf File Doc", - 'doc_file': f, - 'extension': 'pdf', + "title": "Pdf File Doc", + "doc_file": f, + "extension": "pdf", } - self.client.post(reverse('document_upload'), data=data) - d = Document.objects.get(title='Pdf File Doc') + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="Pdf File Doc") self.assertIsNotNone(d.thumbnail_url) thumb_file = os.path.join( settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" ) file = Image.open(thumb_file) # check thumbnail qualty and extention - self.assertEqual(file.format, 'JPEG') + self.assertEqual(file.format, "JPEG") finally: - Document.objects.filter(title='img File Doc').delete() - Document.objects.filter(title='Pdf File Doc').delete() + Document.objects.filter(title="img File Doc").delete() + Document.objects.filter(title="Pdf File Doc").delete() def test_upload_document_form_size_limit(self): form_data = { - 'title': 'GeoNode Map', - 'permissions': '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', } - test_file = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif' - ) + test_file = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") test_file.size = settings.DEFAULT_MAX_UPLOAD_SIZE * 5 # Set as a large file - file_data = {'doc_file': test_file} + file_data = {"doc_file": test_file} with self.assertRaises(FileUploadLimitException): form = DocumentCreateForm(form_data, file_data) @@ -346,50 +313,47 @@ def test_upload_document_form_size_limit(self): f"File size size exceeds {filesizeformat(settings.DEFAULT_MAX_UPLOAD_SIZE)}. " f"Please try again with a smaller file." ) - self.assertEqual(form.errors, {'doc_file': [expected_error]}) + self.assertEqual(form.errors, {"doc_file": [expected_error]}) def test_document_embed(self): """/documents/1 -> Test accessing the embed view of a document""" d = Document.objects.all().first() d.set_default_permissions() - response = self.client.get(reverse('document_embed', args=(str(d.id),))) + response = self.client.get(reverse("document_embed", args=(str(d.id),))) self.assertEqual(response.status_code, 200) def test_access_document_upload_form(self): """Test the form page is returned correctly via GET request /documents/upload""" - log = self.client.login(username='bobby', password='bob') + log = self.client.login(username="bobby", password="bob") self.assertTrue(log) - response = self.client.get(reverse('document_upload')) + response = self.client.get(reverse("document_upload")) self.assertEqual(response.status_code, 405) def test_document_isuploaded(self): """/documents/upload -> Test uploading a document""" - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') + f = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") m = Map.objects.first() - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") response = self.client.post( f"{reverse('document_upload')}?no__redirect=true", data={ - 'doc_file': f, - 'title': 'uploaded_document', - 'q': m.id, - 'type': 'document', - 'permissions': '{"users":{"AnonymousUser": ["view_resourcebase"]}}'}, + "doc_file": f, + "title": "uploaded_document", + "q": m.id, + "type": "document", + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]}}', + }, ) self.assertEqual(response.status_code, 200) # Permissions Tests def test_set_document_permissions(self): - """Verify that the set_document_permissions view is behaving as expected - """ + """Verify that the set_document_permissions view is behaving as expected""" # Get a document to work with document = Document.objects.first() @@ -397,99 +361,93 @@ def test_set_document_permissions(self): document.set_permissions(self.perm_spec) # Test that the Permissions for anonympus user are set correctly - self.assertFalse( - self.anonymous_user.has_perm( - 'view_resourcebase', - document.get_self_resource())) + self.assertFalse(self.anonymous_user.has_perm("view_resourcebase", document.get_self_resource())) # Test that previous permissions for users other than ones specified in # the perm_spec (and the document owner) were removed current_perms = document.get_all_level_info() - self.assertEqual(len(current_perms['users']), 1) + self.assertEqual(len(current_perms["users"]), 1) # Test that the User permissions specified in the perm_spec were # applied properly - for username, perm in self.perm_spec['users'].items(): + for username, perm in self.perm_spec["users"].items(): user = get_user_model().objects.get(username=username) self.assertTrue(user.has_perm(perm, document.get_self_resource())) @patch("geonode.documents.tasks.create_document_thumbnail") def test_ajax_document_permissions(self, create_thumb): - """Verify that the ajax_document_permissions view is behaving as expected - """ + """Verify that the ajax_document_permissions view is behaving as expected""" create_thumb.return_value = True # Setup some document names to work with f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) document = resource_manager.create( - None, - resource_type=Document, - defaults=dict( - files=f, - owner=superuser, - title='theimg', - is_approved=True)) + None, resource_type=Document, defaults=dict(files=f, owner=superuser, title="theimg", is_approved=True) + ) document_id = document.id invalid_document_id = 20 # Test that an invalid document is handled for properly response = self.client.post( - reverse( - 'resource_permissions', args=( - invalid_document_id,)), data=json.dumps( - self.perm_spec), content_type="application/json") + reverse("resource_permissions", args=(invalid_document_id,)), + data=json.dumps(self.perm_spec), + content_type="application/json", + ) self.assertEqual(response.status_code, 401) # Test that GET returns permissions - response = self.client.get(reverse('resource_permissions', args=(document_id,))) - assert ('permissions' in ensure_string(response.content)) + response = self.client.get(reverse("resource_permissions", args=(document_id,))) + assert "permissions" in ensure_string(response.content) # Test that a user is required to have # documents.change_dataset_permissions # First test un-authenticated response = self.client.post( - reverse('resource_permissions', args=(document_id,)), + reverse("resource_permissions", args=(document_id,)), data=json.dumps(self.perm_spec), - content_type="application/json") + content_type="application/json", + ) self.assertEqual(response.status_code, 401) # Next Test with a user that does NOT have the proper perms - logged_in = self.client.login(username='bobby', password='bob') + logged_in = self.client.login(username="bobby", password="bob") self.assertEqual(logged_in, True) response = self.client.post( - reverse('resource_permissions', args=(document_id,)), + reverse("resource_permissions", args=(document_id,)), data=json.dumps(self.perm_spec), - content_type="application/json") + content_type="application/json", + ) self.assertEqual(response.status_code, 401) # Login as a user with the proper permission and test the endpoint - logged_in = self.client.login(username='admin', password='admin') + logged_in = self.client.login(username="admin", password="admin") self.assertEqual(logged_in, True) response = self.client.post( - reverse('resource_permissions', args=(document_id,)), + reverse("resource_permissions", args=(document_id,)), data=json.dumps(self.perm_spec), - content_type="application/json") + content_type="application/json", + ) # Test that the method returns 200 self.assertEqual(response.status_code, 200) def test_batch_edit(self): Model = Document - view = 'document_batch_metadata' + view = "document_batch_metadata" resources = Model.objects.all()[:3] - ids = ','.join(str(element.pk) for element in resources) + ids = ",".join(str(element.pk) for element in resources) # test non-admin access self.client.login(username="bobby", password="bob") response = self.client.get(reverse(view)) self.assertTrue(response.status_code in (401, 403)) # test group change group = Group.objects.first() - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") response = self.client.post( reverse(view), - data={'group': group.pk, 'ids': ids, 'regions': 1}, + data={"group": group.pk, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) @@ -499,7 +457,7 @@ def test_batch_edit(self): owner = get_user_model().objects.first() response = self.client.post( reverse(view), - data={'owner': owner.pk, 'ids': ids, 'regions': 1}, + data={"owner": owner.pk, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) @@ -509,7 +467,7 @@ def test_batch_edit(self): license = License.objects.first() response = self.client.post( reverse(view), - data={'license': license.pk, 'ids': ids, 'regions': 1}, + data={"license": license.pk, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) @@ -519,7 +477,7 @@ def test_batch_edit(self): region = Region.objects.first() response = self.client.post( reverse(view), - data={'region': region.pk, 'ids': ids, 'regions': 1}, + data={"region": region.pk, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) @@ -527,104 +485,101 @@ def test_batch_edit(self): if resource.regions.all(): self.assertTrue(region in resource.regions.all()) # test language change - language = 'eng' + language = "eng" response = self.client.post( reverse(view), - data={'language': language, 'ids': ids, 'regions': 1}, + data={"language": language, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) for resource in resources: self.assertEqual(resource.language, language) # test keywords change - keywords = 'some,thing,new' + keywords = "some,thing,new" response = self.client.post( reverse(view), - data={'keywords': keywords, 'ids': ids, 'regions': 1}, + data={"keywords": keywords, "ids": ids, "regions": 1}, ) self.assertEqual(response.status_code, 302) resources = Model.objects.filter(id__in=[r.pk for r in resources]) for resource in resources: for word in resource.keywords.all(): - self.assertTrue(word.name in keywords.split(',')) + self.assertTrue(word.name in keywords.split(",")) class DocumentModerationTestCase(GeoNodeBaseTestSupport): - def setUp(self): super().setUp() - self.user = 'admin' - self.passwd = 'admin' - create_models(type=b'document') - create_models(type=b'map') + self.user = "admin" + self.passwd = "admin" + create_models(type=b"document") + create_models(type=b"map") self.project_root = os.path.abspath(os.path.dirname(__file__)) self.document_upload_url = f"{(reverse('document_upload'))}?no__redirect=true" self.u = get_user_model().objects.get(username=self.user) - self.u.email = 'test@email.com' + self.u.email = "test@email.com" self.u.is_active = True self.u.save() def _get_input_path(self): base_path = gisdata.GOOD_DATA - return os.path.join(base_path, 'vector', 'readme.txt') + return os.path.join(base_path, "vector", "readme.txt") def test_document_upload_redirect(self): with self.settings(ADMIN_MODERATE_UPLOADS=False): self.client.login(username=self.user, password=self.passwd) - dname = 'document title' + dname = "document title" with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: data = { - 'title': dname, - 'doc_file': f, - 'resource': '', - 'extension': 'gif', - 'permissions': '{}', + "title": dname, + "doc_file": f, + "resource": "", + "extension": "gif", + "permissions": "{}", } resp = self.client.post(self.document_upload_url, data=data) self.assertEqual(resp.status_code, 200, resp.content) - content = json.loads(resp.content.decode('utf-8')) + content = json.loads(resp.content.decode("utf-8")) self.assertTrue(content["success"]) self.assertIn("url", content) class DocumentsNotificationsTestCase(NotificationsTestsHelper): - def setUp(self): - self.user = 'admin' - self.passwd = 'admin' - create_models(type=b'document') + self.user = "admin" + self.passwd = "admin" + create_models(type=b"document") self.anonymous_user = get_anonymous_user() self.u = get_user_model().objects.get(username=self.user) - self.u.email = 'test@email.com' + self.u.email = "test@email.com" self.u.is_active = True self.u.is_superuser = True self.u.save() self.setup_notifications_for(DocumentsAppConfig.NOTIFICATIONS, self.u) - self.norman = get_user_model().objects.get(username='norman') - self.norman.email = 'norman@email.com' + self.norman = get_user_model().objects.get(username="norman") + self.norman.email = "norman@email.com" self.norman.is_active = True self.norman.save() self.setup_notifications_for(DocumentsAppConfig.NOTIFICATIONS, self.norman) def testDocumentsNotifications(self): with self.settings( - EMAIL_ENABLE=True, - NOTIFICATION_ENABLED=True, - NOTIFICATIONS_BACKEND="pinax.notifications.backends.email.EmailBackend", - PINAX_NOTIFICATIONS_QUEUE_ALL=False): + EMAIL_ENABLE=True, + NOTIFICATION_ENABLED=True, + NOTIFICATIONS_BACKEND="pinax.notifications.backends.email.EmailBackend", + PINAX_NOTIFICATIONS_QUEUE_ALL=False, + ): self.clear_notifications_queue() self.client.login(username=self.user, password=self.passwd) - _d = Document.objects.create( - title='test notifications', - owner=self.norman) - self.assertTrue(self.check_notification_out('document_created', self.u)) + _d = Document.objects.create(title="test notifications", owner=self.norman) + self.assertTrue(self.check_notification_out("document_created", self.u)) # Ensure "resource.owner" won't be notified for having created its own document - self.assertFalse(self.check_notification_out('document_created', self.norman)) + self.assertFalse(self.check_notification_out("document_created", self.norman)) self.clear_notifications_queue() - _d.title = 'test notifications 2' + _d.title = "test notifications 2" _d.save(notify=True) - self.assertTrue(self.check_notification_out('document_updated', self.u)) + self.assertTrue(self.check_notification_out("document_updated", self.u)) self.clear_notifications_queue() lct = ContentType.objects.get_for_model(_d) @@ -632,25 +587,21 @@ def testDocumentsNotifications(self): if "pinax.ratings" in settings.INSTALLED_APPS: self.clear_notifications_queue() from pinax.ratings.models import Rating - rating = Rating(user=self.norman, - content_type=lct, - object_id=_d.id, - content_object=_d, - rating=5) + + rating = Rating(user=self.norman, content_type=lct, object_id=_d.id, content_object=_d, rating=5) rating.save() - self.assertTrue(self.check_notification_out('document_rated', self.u)) + self.assertTrue(self.check_notification_out("document_rated", self.u)) class DocumentResourceLinkTestCase(GeoNodeBaseTestSupport): - def setUp(self): - create_models(b'document') - create_models(b'map') - create_models(b'dataset') + create_models(b"document") + create_models(b"map") + create_models(b"dataset") self.test_file = io.BytesIO( - b'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' - b'\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;' + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" + b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" ) def test_create_document_with_links(self): @@ -658,13 +609,9 @@ def test_create_document_with_links(self): f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - d = Document.objects.create( - files=f, - owner=superuser, - title='theimg' - ) + d = Document.objects.create(files=f, owner=superuser, title="theimg") - self.assertEqual(Document.objects.get(pk=d.id).title, 'theimg') + self.assertEqual(Document.objects.get(pk=d.id).title, "theimg") maps = list(Map.objects.all()) layers = list(Dataset.objects.all()) @@ -681,11 +628,7 @@ def test_create_document_with_links(self): for resource in resources: ct = ContentType.objects.get_for_model(resource) - _d = DocumentResourceLink.objects.get( - document_id=d.id, - content_type=ct.id, - object_id=resource.id - ) + _d = DocumentResourceLink.objects.get(document_id=d.id, content_type=ct.id, object_id=resource.id) self.assertEqual(_d.object_id, resource.id) # update document links @@ -699,73 +642,58 @@ def test_create_document_with_links(self): for resource in layers: ct = ContentType.objects.get_for_model(resource) - _d = DocumentResourceLink.objects.get( - document_id=d.id, - content_type=ct.id, - object_id=resource.id - ) + _d = DocumentResourceLink.objects.get(document_id=d.id, content_type=ct.id, object_id=resource.id) self.assertEqual(_d.object_id, resource.id) for resource in maps: ct = ContentType.objects.get_for_model(resource) with self.assertRaises(DocumentResourceLink.DoesNotExist): - DocumentResourceLink.objects.get( - document_id=d.id, - content_type=ct.id, - object_id=resource.id - ) + DocumentResourceLink.objects.get(document_id=d.id, content_type=ct.id, object_id=resource.id) class DocumentViewTestCase(GeoNodeBaseTestSupport): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] def setUp(self): - self.not_admin = get_user_model().objects.create(username='r-lukaku', is_active=True) - self.not_admin.set_password('very-secret') + self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True) + self.not_admin.set_password("very-secret") self.not_admin.save() self.files = [f"{settings.MEDIA_ROOT}/img.gif"] self.test_doc = resource_manager.create( None, resource_type=Document, - defaults=dict( - files=self.files, - owner=self.not_admin, - title='test', - is_approved=True)) + defaults=dict(files=self.files, owner=self.not_admin, title="test", is_approved=True), + ) self.perm_spec = {"users": {"AnonymousUser": []}} - self.doc_link_url = reverse('document_link', args=(self.test_doc.pk,)) + self.doc_link_url = reverse("document_link", args=(self.test_doc.pk,)) def test_that_keyword_multiselect_is_disabled_for_non_admin_users(self): """ Test that keyword multiselect widget is disabled when the user is not an admin when FREETEXT_KEYWORDS_READONLY=True """ - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=True): response = self.client.get(url) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) - self.assertTrue(response.context['form']['keywords'].field.disabled) + self.assertTrue(response.context["form"]["keywords"].field.disabled) def test_that_featured_enabling_and_disabling_for_users(self): # Non Admins - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) response = self.client.get(url) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) - self.assertTrue(response.context['form']['featured'].field.disabled) + self.assertTrue(response.context["form"]["featured"].field.disabled) # Admin - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertFalse(response.context['form']['featured'].field.disabled) + self.assertFalse(response.context["form"]["featured"].field.disabled) def test_that_keyword_multiselect_is_not_disabled_for_admin_users(self): """ @@ -774,73 +702,79 @@ def test_that_keyword_multiselect_is_not_disabled_for_admin_users(self): admin = self.not_admin admin.is_superuser = True admin.save() - self.client.login(username=admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) response = self.client.get(url) self.assertTrue(admin.is_superuser) self.assertEqual(response.status_code, 200) - self.assertFalse(response.context['form']['keywords'].field.disabled) + self.assertFalse(response.context["form"]["keywords"].field.disabled) def test_that_non_admin_user_can_create_write_to_map_without_keyword(self): """ Test that non admin users can write to maps without creating/editing keywords when FREETEXT_KEYWORDS_READONLY=True """ - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=True): - response = self.client.post(url, data={ - "resource-owner": self.not_admin.id, - "resource-title": "doc", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - }) + response = self.client.post( + url, + data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + }, + ) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) self.test_doc.refresh_from_db() - self.assertEqual('doc', self.test_doc.title) + self.assertEqual("doc", self.test_doc.title) def test_that_non_admin_user_cannot_create_edit_keyword(self): """ Test that non admin users cannot edit/create keywords when FREETEXT_KEYWORDS_READONLY=True """ - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=True): - response = self.client.post(url, data={'resource-keywords': 'wonderful-keyword'}) + response = self.client.post(url, data={"resource-keywords": "wonderful-keyword"}) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 401) - self.assertEqual(response.content, b'Unauthorized: Cannot edit/create Free-text Keywords') + self.assertEqual(response.content, b"Unauthorized: Cannot edit/create Free-text Keywords") def test_that_keyword_multiselect_is_enabled_for_non_admin_users_when_freetext_keywords_readonly_istrue(self): """ Test that keyword multiselect widget is not disabled when the user is not an admin and FREETEXT_KEYWORDS_READONLY=False """ - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=False): response = self.client.get(url) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) - self.assertFalse(response.context['form']['keywords'].field.disabled) + self.assertFalse(response.context["form"]["keywords"].field.disabled) def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_readonly_istrue(self): """ Test that non admin users can edit/create keywords when FREETEXT_KEYWORDS_READONLY=False """ - self.client.login(username=self.not_admin.username, password='very-secret') - url = reverse('document_metadata', args=(self.test_doc.pk,)) + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=False): - response = self.client.post(url, data={ - "resource-owner": self.not_admin.id, - "resource-title": "doc", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - 'resource-keywords': 'wonderful-keyword' - }) + response = self.client.post( + url, + data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-keywords": "wonderful-keyword", + }, + ) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) self.test_doc.refresh_from_db() @@ -852,7 +786,7 @@ def test_document_link_with_permissions(self): response = self.client.get(self.doc_link_url) self.assertEqual(response.status_code, 401) # Access resource with user logged-in - self.client.login(username=self.not_admin.username, password='very-secret') + self.client.login(username=self.not_admin.username, password="very-secret") response = self.client.get(self.doc_link_url) self.assertEqual(response.status_code, 404) # test document link with external url @@ -863,5 +797,6 @@ def test_document_link_with_permissions(self): doc_url="http://geonode.org/map.pdf", owner=self.not_admin, title="GeoNode Map Doc", - )) - self.assertEqual(doc.href, 'http://geonode.org/map.pdf') + ), + ) + self.assertEqual(doc.href, "http://geonode.org/map.pdf") diff --git a/geonode/documents/translation.py b/geonode/documents/translation.py index 96fa3879218..0fc9d834253 100644 --- a/geonode/documents/translation.py +++ b/geonode/documents/translation.py @@ -23,12 +23,12 @@ class DocumentTranslationOptions(TranslationOptions): fields = ( - 'title', - 'abstract', - 'purpose', - 'constraints_other', - 'supplemental_information', - 'data_quality_statement', + "title", + "abstract", + "purpose", + "constraints_other", + "supplemental_information", + "data_quality_statement", ) diff --git a/geonode/documents/urls.py b/geonode/documents/urls.py index 5f6a5e2fd10..be338923d49 100644 --- a/geonode/documents/urls.py +++ b/geonode/documents/urls.py @@ -23,29 +23,18 @@ from . import views js_info_dict = { - 'packages': ('geonode.documents',), + "packages": ("geonode.documents",), } urlpatterns = [ # 'geonode.documents.views', - url(r'^(?P\d+)/download/?$', - views.document_download, name='document_download'), - url(r'^(?P\d+)/link/?$', - views.document_link, name='document_link'), - url(r'^(?P\d+)/replace$', login_required(DocumentUpdateView.as_view()), - name="document_replace"), - url(r'^(?P\d+)/embed/?$', - views.document_embed, name='document_embed'), - url(r'^upload/?$', login_required( - DocumentUploadView.as_view()), name='document_upload'), - url(r'^(?P[^/]*)/metadata_detail$', views.document_metadata_detail, - name='document_metadata_detail'), - url(r'^(?P\d+)/metadata$', - views.document_metadata, name='document_metadata'), - url( - r'^metadata/batch/$', - views.document_batch_metadata, - name='document_batch_metadata'), - url(r'^(?P\d+)/metadata_advanced$', views.document_metadata_advanced, - name='document_metadata_advanced'), - url(r'^', include('geonode.documents.api.urls')), + url(r"^(?P\d+)/download/?$", views.document_download, name="document_download"), + url(r"^(?P\d+)/link/?$", views.document_link, name="document_link"), + url(r"^(?P\d+)/replace$", login_required(DocumentUpdateView.as_view()), name="document_replace"), + url(r"^(?P\d+)/embed/?$", views.document_embed, name="document_embed"), + url(r"^upload/?$", login_required(DocumentUploadView.as_view()), name="document_upload"), + url(r"^(?P[^/]*)/metadata_detail$", views.document_metadata_detail, name="document_metadata_detail"), + url(r"^(?P\d+)/metadata$", views.document_metadata, name="document_metadata"), + url(r"^metadata/batch/$", views.document_batch_metadata, name="document_batch_metadata"), + url(r"^(?P\d+)/metadata_advanced$", views.document_metadata_advanced, name="document_metadata_advanced"), + url(r"^", include("geonode.documents.api.urls")), ] diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py index 60823fa9183..5957d713eee 100644 --- a/geonode/documents/utils.py +++ b/geonode/documents/utils.py @@ -24,6 +24,7 @@ import os import logging from geonode.storage.manager import storage_manager + # Django functionality from django.http import HttpResponse from django.shortcuts import get_object_or_404 @@ -51,12 +52,10 @@ def delete_orphaned_document_files(): if Document.objects.filter(doc_file__contains=filename).count() == 0: logger.debug(f"Deleting orphaned document {filename}") try: - storage_manager.delete(os.path.join( - os.path.join("documents", "document"), filename)) + storage_manager.delete(os.path.join(os.path.join("documents", "document"), filename)) deleted.append(filename) except NotImplementedError as e: - logger.error( - f"Failed to delete orphaned document '{filename}': {e}") + logger.error(f"Failed to delete orphaned document '{filename}': {e}") return deleted @@ -68,13 +67,13 @@ def get_download_response(request, docid, attachment=False): """ document = get_object_or_404(Document, pk=docid) - if not request.user.has_perm( - 'base.download_resourcebase', - obj=document.get_self_resource()): + if not request.user.has_perm("base.download_resourcebase", obj=document.get_self_resource()): return HttpResponse( loader.render_to_string( - '401.html', context={ - 'error_message': _("You are not allowed to view this document.")}, request=request), status=401) + "401.html", context={"error_message": _("You are not allowed to view this document.")}, request=request + ), + status=401, + ) if attachment: register_event(request, EventType.EVENT_DOWNLOAD, document) filename = slugify(os.path.splitext(os.path.basename(document.title))[0]) @@ -82,10 +81,7 @@ def get_download_response(request, docid, attachment=False): if document.files and storage_manager.exists(document.files[0]): return DownloadResponse( storage_manager.open(document.files[0]).file, - basename=f'{filename}.{document.extension}', - attachment=attachment + basename=f"{filename}.{document.extension}", + attachment=attachment, ) - return HttpResponse( - "File is not available", - status=404 - ) + return HttpResponse("File is not available", status=404) diff --git a/geonode/documents/views.py b/geonode/documents/views.py index d2d8b7deff3..ce54fa812d0 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -48,25 +48,14 @@ from geonode.storage.manager import storage_manager from geonode.resource.manager import resource_manager from geonode.decorators import check_keyword_write_perms -from geonode.security.utils import ( - get_user_visible_groups, - AdvancedSecurityWorkflowManager) -from geonode.base.forms import ( - CategoryForm, - TKeywordForm, - ThesaurusAvailableForm) -from geonode.base.models import ( - Thesaurus, - TopicCategory) +from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory from .utils import get_download_response from .models import Document -from .forms import ( - DocumentForm, - DocumentCreateForm, - DocumentReplaceForm -) +from .forms import DocumentForm, DocumentCreateForm, DocumentReplaceForm logger = logging.getLogger("geonode.documents.views") @@ -75,18 +64,15 @@ _PERMISSION_MSG_DELETE = _("You are not permitted to delete this document") _PERMISSION_MSG_GENERIC = _("You do not have permissions for this document.") _PERMISSION_MSG_MODIFY = _("You are not permitted to modify this document") -_PERMISSION_MSG_METADATA = _( - "You are not permitted to modify this document's metadata") +_PERMISSION_MSG_METADATA = _("You are not permitted to modify this document's metadata") _PERMISSION_MSG_VIEW = _("You are not permitted to view this document") -def _resolve_document(request, docid, permission='base.change_resourcebase', - msg=_PERMISSION_MSG_GENERIC, **kwargs): - ''' +def _resolve_document(request, docid, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ Resolve the document by the provided primary key and check the optional permission. - ''' - return resolve_object(request, Document, {'pk': docid}, - permission=permission, permission_msg=msg, **kwargs) + """ + return resolve_object(request, Document, {"pk": docid}, permission=permission, permission_msg=msg, **kwargs) def document_download(request, docid): @@ -101,45 +87,38 @@ def document_link(request, docid): def document_embed(request, docid): from django.http.response import HttpResponseRedirect + document = get_object_or_404(Document, pk=docid) - if not request.user.has_perm( - 'base.download_resourcebase', - obj=document.get_self_resource()): + if not request.user.has_perm("base.download_resourcebase", obj=document.get_self_resource()): return HttpResponse( loader.render_to_string( - '401.html', context={ - 'error_message': _("You are not allowed to view this document.")}, request=request), status=401) + "401.html", context={"error_message": _("You are not allowed to view this document.")}, request=request + ), + status=401, + ) if document.is_image: if document.doc_url: imageurl = document.doc_url else: - imageurl = reverse('document_link', args=(document.id,)) + imageurl = reverse("document_link", args=(document.id,)) context_dict = { "image_url": imageurl, "resource": document.get_self_resource(), } - return render( - request, - "documents/document_embed.html", - context_dict - ) + return render(request, "documents/document_embed.html", context_dict) if document.doc_url: return HttpResponseRedirect(document.doc_url) else: context_dict = { - "document_link": reverse('document_link', args=(document.id,)), + "document_link": reverse("document_link", args=(document.id,)), "resource": document.get_self_resource(), } - return render( - request, - "documents/document_embed.html", - context_dict - ) + return render(request, "documents/document_embed.html", context_dict) class DocumentUploadView(CreateView): - http_method_names = ['post'] + http_method_names = ["post"] form_class = DocumentCreateForm def post(self, request, *args, **kwargs): @@ -150,34 +129,31 @@ def post(self, request, *args, **kwargs): exception_response = geonode_exception_handler(e, {}) return HttpResponse( json.dumps(exception_response.data), - content_type='application/json', - status=exception_response.status_code) + content_type="application/json", + status=exception_response.status_code, + ) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['ALLOWED_DOC_TYPES'] = ALLOWED_DOC_TYPES + context["ALLOWED_DOC_TYPES"] = ALLOWED_DOC_TYPES return context def form_invalid(self, form): messages.error(self.request, f"{form.errors}") - if self.request.GET.get('no__redirect', False): + if self.request.GET.get("no__redirect", False): plaintext_errors = [] for field in form.errors.values(): plaintext_errors.append(field.data[0].message) - out = {'success': False} - out['message'] = '.'.join(plaintext_errors) + out = {"success": False} + out["message"] = ".".join(plaintext_errors) status_code = 400 - return HttpResponse( - json.dumps(out), - content_type='application/json', - status=status_code) + return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) else: form.name = None form.title = None form.doc_file = None form.doc_url = None - return self.render_to_response( - self.get_context_data(request=self.request, form=form)) + return self.render_to_response(self.get_context_data(request=self.request, form=form)) def form_valid(self, form): """ @@ -185,7 +161,7 @@ def form_valid(self, form): """ doc_form = form.cleaned_data - file = doc_form.pop('doc_file', None) + file = doc_form.pop("doc_file", None) if file: tempdir = mkdtemp() dirname = os.path.basename(tempdir) @@ -196,9 +172,10 @@ def form_valid(self, form): resource_type=Document, defaults=dict( owner=self.request.user, - doc_url=doc_form.pop('doc_url', None), - title=doc_form.pop('title', file.name), - files=[storage_path]) + doc_url=doc_form.pop("doc_url", None), + title=doc_form.pop("title", file.name), + files=[storage_path], + ), ) if tempdir != os.path.dirname(storage_path): shutil.rmtree(tempdir, ignore_errors=True) @@ -207,9 +184,8 @@ def form_valid(self, form): None, resource_type=Document, defaults=dict( - owner=self.request.user, - doc_url=doc_form.pop('doc_url', None), - title=doc_form.pop('title', None)) + owner=self.request.user, doc_url=doc_form.pop("doc_url", None), title=doc_form.pop("title", None) + ), ) self.object.handle_moderated_uploads() @@ -224,17 +200,18 @@ def form_valid(self, form): bbox = None url = hookset.document_detail_url(self.object) - out = {'success': False} + out = {"success": False} - if getattr(settings, 'EXIF_ENABLED', False): + if getattr(settings, "EXIF_ENABLED", False): try: from geonode.documents.exif.utils import exif_extract_metadata_doc + exif_metadata = exif_extract_metadata_doc(self.object) if exif_metadata: - date = exif_metadata.get('date', None) - keywords.extend(exif_metadata.get('keywords', [])) - bbox = exif_metadata.get('bbox', None) - abstract = exif_metadata.get('abstract', None) + date = exif_metadata.get("date", None) + keywords.extend(exif_metadata.get("keywords", [])) + bbox = exif_metadata.get("bbox", None) + abstract = exif_metadata.get("abstract", None) except Exception: logger.debug("Exif extraction failed.") @@ -247,34 +224,32 @@ def form_valid(self, form): abstract=abstract, date=date, date_type="Creation", - bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon() if bbox else None + bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon() if bbox else None, ), - notify=True) + notify=True, + ) resource_manager.set_thumbnail(self.object.uuid, instance=self.object, overwrite=False) register_event(self.request, EventType.EVENT_UPLOAD, self.object) - if self.request.GET.get('no__redirect', False): - out['success'] = True - out['url'] = url - if out['success']: + if self.request.GET.get("no__redirect", False): + out["success"] = True + out["url"] = url + if out["success"]: status_code = 200 else: status_code = 400 - return HttpResponse( - json.dumps(out), - content_type='application/json', - status=status_code) + return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) else: return HttpResponseRedirect(url) class DocumentUpdateView(UpdateView): - template_name = 'documents/document_replace.html' - pk_url_kwarg = 'docid' + template_name = "documents/document_replace.html" + pk_url_kwarg = "docid" form_class = DocumentReplaceForm queryset = Document.objects.all() - context_object_name = 'document' + context_object_name = "document" def post(self, request, *args, **kwargs): self.object = self.get_object() @@ -284,12 +259,13 @@ def post(self, request, *args, **kwargs): exception_response = geonode_exception_handler(e, {}) return HttpResponse( json.dumps(exception_response.data), - content_type='application/json', - status=exception_response.status_code) + content_type="application/json", + status=exception_response.status_code, + ) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['ALLOWED_DOC_TYPES'] = ALLOWED_DOC_TYPES + context["ALLOWED_DOC_TYPES"] = ALLOWED_DOC_TYPES return context def form_valid(self, form): @@ -298,18 +274,14 @@ def form_valid(self, form): """ doc_form = form.cleaned_data - file = doc_form.pop('doc_file', None) + file = doc_form.pop("doc_file", None) if file: tempdir = mkdtemp() dirname = os.path.basename(tempdir) filepath = storage_manager.save(f"{dirname}/{file.name}", file) storage_path = storage_manager.path(filepath) self.object = resource_manager.update( - self.object.uuid, - instance=self.object, - vals=dict( - owner=self.request.user, - files=[storage_path]) + self.object.uuid, instance=self.object, vals=dict(owner=self.request.user, files=[storage_path]) ) if tempdir != os.path.dirname(storage_path): shutil.rmtree(tempdir, ignore_errors=True) @@ -321,18 +293,10 @@ def form_valid(self, form): @login_required @check_keyword_write_perms -def document_metadata( - request, - docid, - template='documents/document_metadata.html', - ajax=True): +def document_metadata(request, docid, template="documents/document_metadata.html", ajax=True): document = None try: - document = _resolve_document( - request, - docid, - 'base.change_resourcebase_metadata', - _PERMISSION_MSG_METADATA) + document = _resolve_document(request, docid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: @@ -348,105 +312,99 @@ def document_metadata( current_keywords = [keyword.name for keyword in document.keywords.all()] if request.method == "POST": - document_form = DocumentForm( + document_form = DocumentForm(request.POST, instance=document, prefix="resource", user=request.user) + category_form = CategoryForm( request.POST, - instance=document, - prefix="resource", - user=request.user) - category_form = CategoryForm(request.POST, prefix="category_choice_field", initial=int( - request.POST["category_choice_field"]) if "category_choice_field" in request.POST and - request.POST["category_choice_field"] else None) + prefix="category_choice_field", + initial=int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None, + ) - if hasattr(settings, 'THESAURUS'): + if hasattr(settings, "THESAURUS"): tkeywords_form = TKeywordForm(request.POST) else: - tkeywords_form = ThesaurusAvailableForm(request.POST, prefix='tkeywords') + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") else: document_form = DocumentForm(instance=document, prefix="resource", user=request.user) document_form.disable_keywords_widget_for_non_superuser(request.user) category_form = CategoryForm( - prefix="category_choice_field", - initial=topic_category.id if topic_category else None) + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) # Keywords from THESAURUS management doc_tkeywords = document.tkeywords.all() - if hasattr(settings, 'THESAURUS') and settings.THESAURUS: - warnings.warn('The settings for Thesaurus has been moved to Model, \ - this feature will be removed in next releases', DeprecationWarning) - tkeywords_list = '' - lang = 'en' # TODO: use user's language + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + tkeywords_list = "" + lang = "en" # TODO: use user's language if doc_tkeywords and len(doc_tkeywords) > 0: - tkeywords_ids = doc_tkeywords.values_list('id', flat=True) - if hasattr(settings, 'THESAURUS') and settings.THESAURUS: + tkeywords_ids = doc_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: el = settings.THESAURUS - thesaurus_name = el['name'] + thesaurus_name = el["name"] try: t = Thesaurus.objects.get(identifier=thesaurus_name) for tk in t.thesaurus.filter(pk__in=tkeywords_ids): tkl = tk.keyword.filter(lang=lang) if len(tkl) > 0: - tkl_ids = ",".join( - map(str, tkl.values_list('id', flat=True))) - tkeywords_list += f",{tkl_ids}" if len( - tkeywords_list) > 0 else tkl_ids + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids except Exception: tb = traceback.format_exc() logger.error(tb) tkeywords_form = TKeywordForm(instance=document) else: - tkeywords_form = ThesaurusAvailableForm(prefix='tkeywords') + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") # set initial values for thesaurus form for tid in tkeywords_form.fields: values = [] values = [keyword.id for keyword in doc_tkeywords if int(tid) == keyword.thesaurus.id] tkeywords_form.fields[tid].initial = values - if request.method == "POST" and document_form.is_valid( - ) and category_form.is_valid() and tkeywords_form.is_valid(): - new_poc = document_form.cleaned_data['poc'] - new_author = document_form.cleaned_data['metadata_author'] - new_keywords = current_keywords if request.keyword_readonly else document_form.cleaned_data['keywords'] - new_regions = document_form.cleaned_data['regions'] + if request.method == "POST" and document_form.is_valid() and category_form.is_valid() and tkeywords_form.is_valid(): + new_poc = document_form.cleaned_data["poc"] + new_author = document_form.cleaned_data["metadata_author"] + new_keywords = current_keywords if request.keyword_readonly else document_form.cleaned_data["keywords"] + new_regions = document_form.cleaned_data["regions"] new_category = None - if category_form and 'category_choice_field' in category_form.cleaned_data and \ - category_form.cleaned_data['category_choice_field']: - new_category = TopicCategory.objects.get( - id=int(category_form.cleaned_data['category_choice_field'])) + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) if new_poc is None: if poc is None: - poc_form = ProfileForm( - request.POST, - prefix="poc", - instance=poc) + poc_form = ProfileForm(request.POST, prefix="poc", instance=poc) else: poc_form = ProfileForm(request.POST, prefix="poc") if poc_form.is_valid(): - if len(poc_form.cleaned_data['profile']) == 0: + if len(poc_form.cleaned_data["profile"]) == 0: # FIXME use form.add_error in django > 1.7 - errors = poc_form._errors.setdefault( - 'profile', ErrorList()) - errors.append( - _('You must set a point of contact for this resource')) + errors = poc_form._errors.setdefault("profile", ErrorList()) + errors.append(_("You must set a point of contact for this resource")) if poc_form.has_changed and poc_form.is_valid(): new_poc = poc_form.save() if new_author is None: if metadata_author is None: - author_form = ProfileForm(request.POST, prefix="author", - instance=metadata_author) + author_form = ProfileForm(request.POST, prefix="author", instance=metadata_author) else: author_form = ProfileForm(request.POST, prefix="author") if author_form.is_valid(): - if len(author_form.cleaned_data['profile']) == 0: + if len(author_form.cleaned_data["profile"]) == 0: # FIXME use form.add_error in django > 1.7 - errors = author_form._errors.setdefault( - 'profile', ErrorList()) - errors.append( - _('You must set an author for this resource')) + errors = author_form._errors.setdefault("profile", ErrorList()) + errors.append(_("You must set an author for this resource")) if author_form.has_changed and author_form.is_valid(): new_author = author_form.save() @@ -459,10 +417,10 @@ def document_metadata( vals=dict( poc=new_poc or document.poc, metadata_author=new_author or document.metadata_author, - category=new_category + category=new_category, ), notify=True, - extra_metadata=json.loads(document_form.cleaned_data['extra_metadata']) + extra_metadata=json.loads(document_form.cleaned_data["extra_metadata"]), ) resource_manager.set_thumbnail(document.uuid, instance=document, overwrite=False) @@ -478,14 +436,12 @@ def document_metadata( # Keywords from THESAURUS management # Rewritten to work with updated autocomplete if not tkeywords_form.is_valid(): - return HttpResponse(json.dumps({'message': "Invalid thesaurus keywords"}, status_code=400)) + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) - thesaurus_setting = getattr(settings, 'THESAURUS', None) + thesaurus_setting = getattr(settings, "THESAURUS", None) if thesaurus_setting: - tkeywords_data = tkeywords_form.cleaned_data['tkeywords'] - tkeywords_data = tkeywords_data.filter( - thesaurus__identifier=thesaurus_setting['name'] - ) + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) document.tkeywords.set(tkeywords_data) elif Thesaurus.objects.all().exists(): fields = tkeywords_form.cleaned_data @@ -496,89 +452,81 @@ def document_metadata( logger.error(tb) vals = {} - if 'group' in document_form.changed_data: - vals['group'] = document_form.cleaned_data.get('group') - if any([x in document_form.changed_data for x in ['is_approved', 'is_published']]): - vals['is_approved'] = document_form.cleaned_data.get('is_approved', document.is_approved) - vals['is_published'] = document_form.cleaned_data.get('is_published', document.is_published) + if "group" in document_form.changed_data: + vals["group"] = document_form.cleaned_data.get("group") + if any([x in document_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = document_form.cleaned_data.get("is_approved", document.is_approved) + vals["is_published"] = document_form.cleaned_data.get("is_published", document.is_published) resource_manager.update( document.uuid, instance=document, notify=True, vals=vals, - extra_metadata=json.loads(document_form.cleaned_data['extra_metadata']) + extra_metadata=json.loads(document_form.cleaned_data["extra_metadata"]), ) - return HttpResponse(json.dumps({'message': message})) - elif request.method == "POST" and (not document_form.is_valid( - ) or not category_form.is_valid() or not tkeywords_form.is_valid()): - errors_list = {**document_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} - logger.error(f"GeoApp Metadata form is not valid: {errors_list}") - out = { - 'success': False, - "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()] + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not document_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **document_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), } - return HttpResponse( - json.dumps(out), - content_type='application/json', - status=400) + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) # - POST Request Ends here - # Request.GET if poc is not None: - document_form.fields['poc'].initial = poc.id + document_form.fields["poc"].initial = poc.id poc_form = ProfileForm(prefix="poc") poc_form.hidden = True if metadata_author is not None: - document_form.fields['metadata_author'].initial = metadata_author.id + document_form.fields["metadata_author"].initial = metadata_author.id author_form = ProfileForm(prefix="author") author_form.hidden = True metadata_author_groups = get_user_visible_groups(request.user) if not AdvancedSecurityWorkflowManager.is_allowed_to_publish(request.user, document): - document_form.fields['is_published'].widget.attrs.update({'disabled': 'true'}) + document_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) if not AdvancedSecurityWorkflowManager.is_allowed_to_approve(request.user, document): - document_form.fields['is_approved'].widget.attrs.update({'disabled': 'true'}) + document_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) register_event(request, EventType.EVENT_VIEW_METADATA, document) - return render(request, template, context={ - "resource": document, - "document": document, - "document_form": document_form, - "poc_form": poc_form, - "author_form": author_form, - "category_form": category_form, - "tkeywords_form": tkeywords_form, - "metadata_author_groups": metadata_author_groups, - "TOPICCATEGORY_MANDATORY": getattr(settings, 'TOPICCATEGORY_MANDATORY', False), - "GROUP_MANDATORY_RESOURCES": getattr(settings, 'GROUP_MANDATORY_RESOURCES', False), - "UI_MANDATORY_FIELDS": list( - set(getattr(settings, 'UI_DEFAULT_MANDATORY_FIELDS', [])) - | - set(getattr(settings, 'UI_REQUIRED_FIELDS', [])) - ) - }) + return render( + request, + template, + context={ + "resource": document, + "document": document, + "document_form": document_form, + "poc_form": poc_form, + "author_form": author_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + }, + ) @login_required def document_metadata_advanced(request, docid): - return document_metadata( - request, - docid, - template='documents/document_metadata_advanced.html') + return document_metadata(request, docid, template="documents/document_metadata_advanced.html") -def document_metadata_detail( - request, - docid, - template='documents/document_metadata_detail.html'): +def document_metadata_detail(request, docid, template="documents/document_metadata_detail.html"): try: - document = _resolve_document( - request, - docid, - 'view_resourcebase', - _PERMISSION_MSG_METADATA) + document = _resolve_document(request, docid, "view_resourcebase", _PERMISSION_MSG_METADATA) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: @@ -592,15 +540,11 @@ def document_metadata_detail( group = GroupProfile.objects.get(slug=document.group.name) except ObjectDoesNotExist: group = None - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL register_event(request, EventType.EVENT_VIEW_METADATA, document) - return render(request, template, context={ - "resource": document, - "group": group, - 'SITEURL': site_url - }) + return render(request, template, context={"resource": document, "group": group, "SITEURL": site_url}) @login_required def document_batch_metadata(request): - return batch_modify(request, 'Document') + return batch_modify(request, "Document") diff --git a/geonode/favorite/models.py b/geonode/favorite/models.py index 87008a1ad41..211188f631f 100644 --- a/geonode/favorite/models.py +++ b/geonode/favorite/models.py @@ -31,14 +31,12 @@ class FavoriteManager(models.Manager): - def favorites_for_user(self, user): return self.filter(user=user) def _favorite_ct_for_user(self, user, model): content_type = ContentType.objects.get_for_model(model) - result = self.favorites_for_user(user).filter( - content_type=content_type).prefetch_related('content_object') + result = self.favorites_for_user(user).filter(content_type=content_type).prefetch_related("content_object") return result def favorite_documents_for_user(self, user): @@ -60,10 +58,7 @@ def favorite_for_user_and_content_object(self, user, content_object): impl note: can only be 0 or 1, per the class's unique_together. """ content_type = ContentType.objects.get_for_model(type(content_object)) - result = self.filter( - user=user, - content_type=content_type, - object_id=content_object.pk) + result = self.filter(user=user, content_type=content_type, object_id=content_object.pk) if result and len(result) > 0: return result[0] @@ -71,19 +66,19 @@ def favorite_for_user_and_content_object(self, user, content_object): return None def bulk_favorite_objects(self, user): - 'get the actual favorite objects for a user as a dict by content_type' + "get the actual favorite objects for a user as a dict by content_type" favs = {} for m in (Document, Map, Dataset, get_user_model()): ct = ContentType.objects.get_for_model(m) f = self.favorites_for_user(user).filter(content_type=ct) - favs[ct.name] = m.objects.filter(id__in=f.values('object_id')) + favs[ct.name] = m.objects.filter(id__in=f.values("object_id")) return favs def create_favorite(self, content_object, user): if isinstance(content_object, ResourceBase): geoapp_types = get_geonode_app_types() if content_object.resource_type in geoapp_types: - content_type = ContentType.objects.get(model='geoapp') + content_type = ContentType.objects.get(model="geoapp") else: content_type = ContentType.objects.get(model=content_object.resource_type) else: @@ -101,16 +96,16 @@ class Favorite(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") created_on = models.DateTimeField(auto_now_add=True) objects = FavoriteManager() class Meta: - verbose_name = 'favorite' - verbose_name_plural = 'favorites' - unique_together = (('user', 'content_type', 'object_id'),) + verbose_name = "favorite" + verbose_name_plural = "favorites" + unique_together = (("user", "content_type", "object_id"),) def __str__(self): if self.content_object: diff --git a/geonode/favorite/tests.py b/geonode/favorite/tests.py index e3df317b3d3..48195b835ed 100644 --- a/geonode/favorite/tests.py +++ b/geonode/favorite/tests.py @@ -26,16 +26,12 @@ from .models import Favorite from geonode.documents.models import Document -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models, - create_single_dataset) +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset class FavoriteTest(GeoNodeBaseTestSupport): - type = 'document' + type = "document" """ Tests geonode.favorite app/module @@ -50,8 +46,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): super().tearDownClass() - remove_models(cls.get_obj_ids, type=cls.get_type, - integration=cls.get_integration) + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) def setUp(self): super().setUp() @@ -80,13 +75,11 @@ def test_favorite(self): self.assertEqual(favorites_for_user.count(), 2) # test document favorites for user. - document_favorites = Favorite.objects.favorite_documents_for_user( - test_user) + document_favorites = Favorite.objects.favorite_documents_for_user(test_user) self.assertEqual(document_favorites.count(), 2) # test layer favorites for user. - dataset_favorites = Favorite.objects.favorite_datasets_for_user( - test_user) + dataset_favorites = Favorite.objects.favorite_datasets_for_user(test_user) self.assertEqual(dataset_favorites.count(), 0) # test map favorites for user. @@ -98,8 +91,7 @@ def test_favorite(self): self.assertEqual(user_favorites.count(), 0) # test favorite for user and a specific content object. - user_content_favorite = Favorite.objects.favorite_for_user_and_content_object( - test_user, test_document_1) + user_content_favorite = Favorite.objects.favorite_for_user_and_content_object(test_user, test_document_1) self.assertEqual(user_content_favorite.object_id, test_document_1.id) # test bulk favorites. @@ -110,18 +102,18 @@ def test_favorite(self): def test_given_resource_base_object_will_assign_subtype_as_content_type(self): test_user = get_user_model().objects.first() - ''' + """ If the input object is a ResourceBase, in favorite content type, should be saved he subtype content type (Doc, Dataset, Map or GeoApp) - ''' - create_single_dataset('foo_dataset') - resource = ResourceBase.objects.get(title='foo_dataset') + """ + create_single_dataset("foo_dataset") + resource = ResourceBase.objects.get(title="foo_dataset") created_fav = Favorite.objects.create_favorite(resource, test_user) - self.assertEqual('dataset', created_fav.content_type.model) + self.assertEqual("dataset", created_fav.content_type.model) - ''' + """ If the input object is a subtype, should save the relative content type - ''' + """ test_document_1 = Document.objects.first() self.assertIsNotNone(test_document_1) Favorite.objects.create_favorite(test_document_1, test_user) @@ -130,10 +122,4 @@ def test_given_resource_base_object_will_assign_subtype_as_content_type(self): self.assertEqual(fav.content_type, ct) def _get_response(self, input_url, input_args): - return self.client.post( - reverse( - input_url, - args=input_args - ), - content_type="application/json" - ) + return self.client.post(reverse(input_url, args=input_args), content_type="application/json") diff --git a/geonode/geoapps/__init__.py b/geonode/geoapps/__init__.py index 9990f003eea..35ab8aa62a1 100644 --- a/geonode/geoapps/__init__.py +++ b/geonode/geoapps/__init__.py @@ -21,16 +21,41 @@ class GeoNodeAppsConfig(NotificationsAppConfigBase): - name = 'geonode.geoapps' - type = 'GEONODE_APP' + name = "geonode.geoapps" + type = "GEONODE_APP" - NOTIFICATIONS = (("geoapp_created", _("App Created"), _("A App was created"),), - ("geoapp_updated", _("App Updated"), _("A App was updated"),), - ("geoapp_approved", _("App Approved"), _("A App was approved by a Manager"),), - ("geoapp_published", _("App Published"), _("A App was published"),), - ("geoapp_deleted", _("App Deleted"), _("A App was deleted"),), - ("geoapp_rated", _("Rating for App"), _("A rating was given to an App"),), - ) + NOTIFICATIONS = ( + ( + "geoapp_created", + _("App Created"), + _("A App was created"), + ), + ( + "geoapp_updated", + _("App Updated"), + _("A App was updated"), + ), + ( + "geoapp_approved", + _("App Approved"), + _("A App was approved by a Manager"), + ), + ( + "geoapp_published", + _("App Published"), + _("A App was published"), + ), + ( + "geoapp_deleted", + _("App Deleted"), + _("A App was deleted"), + ), + ( + "geoapp_rated", + _("Rating for App"), + _("A rating was given to an App"), + ), + ) -default_app_config = 'geonode.geoapps.GeoNodeAppsConfig' +default_app_config = "geonode.geoapps.GeoNodeAppsConfig" diff --git a/geonode/geoapps/admin.py b/geonode/geoapps/admin.py index 879def34118..f0217db4cb2 100644 --- a/geonode/geoapps/admin.py +++ b/geonode/geoapps/admin.py @@ -26,18 +26,45 @@ class GeoAppAdminForm(ResourceBaseAdminForm): - class Meta(ResourceBaseAdminForm.Meta): model = GeoApp - fields = '__all__' + fields = "__all__" class GeoAppAdmin(TabbedTranslationAdmin): - list_display_links = ('title',) - list_display = ('id', 'title', 'type', 'owner', 'category', 'group', 'is_approved', 'is_published',) - list_editable = ('owner', 'category', 'group', 'is_approved', 'is_published',) - list_filter = ('title', 'owner', 'category', 'group', 'is_approved', 'is_published',) - search_fields = ('title', 'abstract', 'purpose', 'is_approved', 'is_published',) + list_display_links = ("title",) + list_display = ( + "id", + "title", + "type", + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + list_editable = ( + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + list_filter = ( + "title", + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + search_fields = ( + "title", + "abstract", + "purpose", + "is_approved", + "is_published", + ) form = GeoAppAdminForm def delete_queryset(self, request, queryset): @@ -47,6 +74,7 @@ def delete_queryset(self, request, queryset): """ for obj in queryset: from geonode.resource.manager import resource_manager + resource_manager.delete(obj.uuid, instance=obj) diff --git a/geonode/geoapps/api/permissions.py b/geonode/geoapps/api/permissions.py index 22ee01ba5a3..1c48cbe0d2e 100644 --- a/geonode/geoapps/api/permissions.py +++ b/geonode/geoapps/api/permissions.py @@ -27,8 +27,9 @@ class GeoAppPermissionsFilter(BaseFilterBackend): A filter backend that limits results to those where the requesting user has read object level permissions. """ + shortcut_kwargs = { - 'accept_global_perms': True, + "accept_global_perms": True, } def filter_queryset(self, request, queryset, view): @@ -39,19 +40,15 @@ def filter_queryset(self, request, queryset, view): from geonode.security.utils import get_visible_resources user = request.user - resources = get_objects_for_user( - user, - 'base.view_resourcebase', - **self.shortcut_kwargs - ) + resources = get_objects_for_user(user, "base.view_resourcebase", **self.shortcut_kwargs) _allowed_ids = get_visible_resources( resources, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, - private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES - ).values_list('id', flat=True) + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ).values_list("id", flat=True) obj_with_perms = [_app.id for _app in GeoApp.objects.filter(id__in=_allowed_ids)] diff --git a/geonode/geoapps/api/serializers.py b/geonode/geoapps/api/serializers.py index a0740f87002..86654832399 100644 --- a/geonode/geoapps/api/serializers.py +++ b/geonode/geoapps/api/serializers.py @@ -29,24 +29,20 @@ class GeoAppSerializer(ResourceBaseSerializer): - def __init__(self, *args, **kwargs): # Instantiate the superclass normally super().__init__(*args, **kwargs) class Meta: model = GeoApp - name = 'geoapp' - view_name = 'geoapps-list' - fields = ( - 'pk', 'uuid', 'data', 'name', - 'executions' - ) + name = "geoapp" + view_name = "geoapps-list" + fields = ("pk", "uuid", "data", "name", "executions") def extra_update_checks(self, validated_data): _user_profiles = {} for _key, _value in validated_data.items(): - if _key in ('owner', 'poc', 'metadata_owner'): + if _key in ("owner", "poc", "metadata_owner"): _user_profiles[_key] = _value for _key, _value in _user_profiles.items(): validated_data.pop(_key) @@ -57,26 +53,25 @@ def extra_update_checks(self, validated_data): raise InvalidGeoAppException(f"The specified '{_key}' does not exist!") def extra_create_checks(self, validated_data): - if 'name' not in validated_data or \ - 'owner' not in validated_data: + if "name" not in validated_data or "owner" not in validated_data: raise InvalidGeoAppException("No valid data: 'name' and 'owner' are mandatory fields!") - if self.Meta.model.objects.filter(name=validated_data['name']).count(): + if self.Meta.model.objects.filter(name=validated_data["name"]).count(): raise DuplicateGeoAppException("A GeoApp with the same 'name' already exists!") self.extra_update_checks(validated_data) def validate(self, data): - request = self.context.get('request') + request = self.context.get("request") if request: - data['owner'] = request.user + data["owner"] = request.user return data def _sanitize_validated_data(self, validated_data, instance=None): # Extract users' profiles _user_profiles = {} for _key, _value in validated_data.items(): - if _key in ('owner', 'poc', 'metadata_owner'): + if _key in ("owner", "poc", "metadata_owner"): _user_profiles[_key] = _value for _key, _value in _user_profiles.items(): validated_data.pop(_key) @@ -92,15 +87,12 @@ def _create_and_update(self, validated_data, instance=None): # Extract JSON blob _data = {} - if 'blob' in validated_data: - _data = validated_data.pop('blob') + if "blob" in validated_data: + _data = validated_data.pop("blob") # Create a new instance if not instance: - _instance = resource_manager.create( - None, - resource_type=self.Meta.model, - defaults=validated_data) + _instance = resource_manager.create(None, resource_type=self.Meta.model, defaults=validated_data) else: _instance = instance @@ -111,29 +103,27 @@ def _create_and_update(self, validated_data, instance=None): raise GeneralGeoAppException(e) # Let's finalize the instance and notify the users - validated_data['blob'] = _data - return resource_manager.update( - _instance.uuid, - instance=_instance, - vals=validated_data, - notify=True) + validated_data["blob"] = _data + return resource_manager.update(_instance.uuid, instance=_instance, vals=validated_data, notify=True) def create(self, validated_data): # Sanity checks - _mandatory_fields = ['name', 'owner', 'resource_type'] + _mandatory_fields = ["name", "owner", "resource_type"] if not all([_x in validated_data for _x in _mandatory_fields]): raise InvalidGeoAppException(f"No valid data: {_mandatory_fields} are mandatory fields!") - if self.Meta.model.objects.filter(name=validated_data['name']).count(): + if self.Meta.model.objects.filter(name=validated_data["name"]).count(): raise DuplicateGeoAppException("A GeoApp with the same 'name' already exists!") return self._create_and_update(self._sanitize_validated_data(validated_data)) def update(self, instance, validated_data): if instance.resource_type: - validated_data['resource_type'] = instance.resource_type + validated_data["resource_type"] = instance.resource_type # Sanity checks - _mandatory_fields = ['resource_type', ] + _mandatory_fields = [ + "resource_type", + ] if not all([_x in validated_data for _x in _mandatory_fields]): raise InvalidGeoAppException(f"No valid data: {_mandatory_fields} are mandatory fields!") diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index bc0f7705173..d37f1f18161 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -34,24 +34,20 @@ class GeoAppsApiTests(APITestCase): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] def setUp(self): - create_models(b'document') - create_models(b'map') - create_models(b'dataset') - self.admin = get_user_model().objects.get(username='admin') - self.bobby = get_user_model().objects.get(username='bobby') - self.norman = get_user_model().objects.get(username='norman') + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + self.admin = get_user_model().objects.get(username="admin") + self.bobby = get_user_model().objects.get(username="bobby") + self.norman = get_user_model().objects.get(username="norman") self.gep_app = GeoApp.objects.create( title="Test GeoApp", owner=self.bobby, resource_type="geostory", - blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}' + blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}', ) self.gep_app.set_default_permissions() @@ -59,35 +55,25 @@ def test_geoapps_list(self): """ Ensure we can access the GeoApps list. """ - url = reverse('geoapps-list') + url = reverse("geoapps-list") # Anonymous - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # Pagination - self.assertEqual(len(response.data['geoapps']), 1) - self.assertTrue('data' not in response.data['geoapps'][0]) + self.assertEqual(len(response.data["geoapps"]), 1) + self.assertTrue("data" not in response.data["geoapps"][0]) - response = self.client.get( - f"{url}?include[]=data", format='json') + response = self.client.get(f"{url}?include[]=data", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 1) + self.assertEqual(response.data["total"], 1) # Pagination - self.assertEqual(len(response.data['geoapps']), 1) - self.assertTrue('data' in response.data['geoapps'][0]) + self.assertEqual(len(response.data["geoapps"]), 1) + self.assertTrue("data" in response.data["geoapps"][0]) self.assertEqual( - json.loads(response.data['geoapps'][0]['data']), - { - "test_data": { - "test": [ - 'test_1', - 'test_2', - 'test_3' - ] - } - } + json.loads(response.data["geoapps"][0]["data"]), {"test_data": {"test": ["test_1", "test_2", "test_3"]}} ) def test_geoapps_crud(self): @@ -95,89 +81,48 @@ def test_geoapps_crud(self): Ensure we can create/update GeoApps. """ # Bobby - self.assertTrue(self.client.login(username='bobby', password='bob')) + self.assertTrue(self.client.login(username="bobby", password="bob")) # Create url = f"{reverse('geoapps-list')}?include[]=data" - data = { - "name": "Test Create", - "title": "Test Create", - "resource_type": "geostory", - "owner": "bobby" - } - response = self.client.post(url, data=data, format='json') + data = {"name": "Test Create", "title": "Test Create", "resource_type": "geostory", "owner": "bobby"} + response = self.client.post(url, data=data, format="json") self.assertEqual(response.status_code, 201) # 201 - Created - response = self.client.get(url, format='json') + response = self.client.get(url, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 2) + self.assertEqual(response.data["total"], 2) # Pagination - self.assertEqual(len(response.data['geoapps']), 2) + self.assertEqual(len(response.data["geoapps"]), 2) # Update: PATCH - url = reverse('geoapps-detail', kwargs={'pk': self.gep_app.pk}) - data = { - "blob": { - "test_data": { - "test": [ - 'test_4', - 'test_5', - 'test_6' - ] - } - } - } + url = reverse("geoapps-detail", kwargs={"pk": self.gep_app.pk}) + data = {"blob": {"test_data": {"test": ["test_4", "test_5", "test_6"]}}} response = self.client.patch(url, data=json.dumps(data), content_type="application/json") self.assertEqual(response.status_code, 200) - response = self.client.get( - f"{url}?include[]=data", format='json') + response = self.client.get(f"{url}?include[]=data", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # Pagination - self.assertTrue('data' in response.data['geoapp']) - self.assertEqual( - response.data['geoapp']['resource_type'], - 'geostory' - ) - self.assertEqual( - response.data['geoapp']['data'], - { - "test_data": { - "test": [ - 'test_4', - 'test_5', - 'test_6' - ] - } - } - ) + self.assertTrue("data" in response.data["geoapp"]) + self.assertEqual(response.data["geoapp"]["resource_type"], "geostory") + self.assertEqual(response.data["geoapp"]["data"], {"test_data": {"test": ["test_4", "test_5", "test_6"]}}) # Update: POST - data = { - "test_data": { - "test": [ - 'test_1', - 'test_2', - 'test_3' - ] - } - } + data = {"test_data": {"test": ["test_1", "test_2", "test_3"]}} _app = GeoApp.objects.first() - _app.set_permissions( - {'users': {self.bobby: ['base.add_resourcebase', 'base.delete_resourcebase']}} - ) + _app.set_permissions({"users": {self.bobby: ["base.add_resourcebase", "base.delete_resourcebase"]}}) - response = self.client.post(url, data=json.dumps(data), format='json') + response = self.client.post(url, data=json.dumps(data), format="json") self.assertEqual(response.status_code, 405) # 405 – Method not allowed # Delete - response = self.client.delete(url, format='json') + response = self.client.delete(url, format="json") self.assertEqual(response.status_code, 405) # 405 - Method Not Allowed - response = self.client.get( - f"{url}?include[]=data", format='json') + response = self.client.get(f"{url}?include[]=data", format="json") self.assertEqual(response.status_code, 200) def test_extra_create_checks_with_no_owner(self): @@ -209,7 +154,9 @@ def test_create_with_no_owner(self): self.assertEqual(exp.exception.category, "geoapp_api") self.assertEqual(exp.exception.default_code, "geoapp_exception") - self.assertEqual(str(exp.exception.detail), "No valid data: ['name', 'owner', 'resource_type'] are mandatory fields!") + self.assertEqual( + str(exp.exception.detail), "No valid data: ['name', 'owner', 'resource_type'] are mandatory fields!" + ) def test_update_with_no_owner(self): serializer = GeoAppSerializer() diff --git a/geonode/geoapps/api/urls.py b/geonode/geoapps/api/urls.py index 4c5588b85da..a39cde0df48 100644 --- a/geonode/geoapps/api/urls.py +++ b/geonode/geoapps/api/urls.py @@ -20,6 +20,6 @@ from . import views -router.register(r'geoapps', views.GeoAppViewSet, 'geoapps') +router.register(r"geoapps", views.GeoAppViewSet, "geoapps") urlpatterns = [] diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index 21b11c404ec..d5cd99eabcb 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -40,13 +40,20 @@ class GeoAppViewSet(DynamicModelViewSet): """ API endpoint that allows geoapps to be viewed or edited. """ - http_method_names = ['get', 'patch', 'post', 'put'] + + http_method_names = ["get", "patch", "post", "put"] authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})] + permission_classes = [ + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ] filter_backends = [ - DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, - ExtentFilter, GeoAppPermissionsFilter + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + GeoAppPermissionsFilter, ] - queryset = GeoApp.objects.all().order_by('-created') + queryset = GeoApp.objects.all().order_by("-created") serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/geoapps/forms.py b/geonode/geoapps/forms.py index 856429d404c..818f3917c5d 100644 --- a/geonode/geoapps/forms.py +++ b/geonode/geoapps/forms.py @@ -22,31 +22,24 @@ class GeoAppForm(ResourceBaseForm): - class Meta(ResourceBaseForm.Meta): model = GeoApp - exclude = ResourceBaseForm.Meta.exclude + ( - 'zoom', - 'projection', - 'center_x', - 'center_y', - 'data' - ) + exclude = ResourceBaseForm.Meta.exclude + ("zoom", "projection", "center_x", "center_y", "data") def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['regions'].choices = get_tree_data() + self.fields["regions"].choices = get_tree_data() for field in self.fields: help_text = self.fields[field].help_text self.fields[field].help_text = None - if help_text != '': + if help_text != "": self.fields[field].widget.attrs.update( { - 'class': 'has-external-popover', - 'data-content': help_text, - 'placeholder': help_text, - 'data-placement': 'right', - 'data-container': 'body', - 'data-html': 'true' + "class": "has-external-popover", + "data-content": help_text, + "placeholder": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", } ) diff --git a/geonode/geoapps/models.py b/geonode/geoapps/models.py index 2f55664b6c4..db57d96f3e3 100644 --- a/geonode/geoapps/models.py +++ b/geonode/geoapps/models.py @@ -38,13 +38,13 @@ class GeoApp(ResourceBase): """ PERMISSIONS = { - 'write': [ - 'change_geoapp_data', - 'change_geoapp_style', + "write": [ + "change_geoapp_data", + "change_geoapp_style", ] } - name = models.TextField(_('Name'), null=False, blank=False) + name = models.TextField(_("Name"), null=False, blank=False) last_modified = models.DateTimeField(auto_now_add=True) # The last time the geoapp was modified. @@ -72,7 +72,7 @@ def center(self): def type(self): _ct = self.polymorphic_ctype _child = _ct.model_class().objects.filter(pk=self.id).first() - if _child and hasattr(_child, 'app_type'): + if _child and hasattr(_child, "app_type"): return _child.app_type return None @@ -82,9 +82,7 @@ def is_public(self): Returns True if anonymous (public) user can view geoapp. """ user = get_anonymous_user() - return user.has_perm( - 'base.view_resourcebase', - obj=self.resourcebase_ptr) + return user.has_perm("base.view_resourcebase", obj=self.resourcebase_ptr) @property def keywords_list(self): @@ -99,4 +97,4 @@ def get_absolute_url(self): @property def embed_url(self): - return reverse('geoapp_embed', kwargs={'geoappid': self.pk}) + return reverse("geoapp_embed", kwargs={"geoappid": self.pk}) diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index 8a1cfa54f67..c42c1733af6 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -27,22 +27,14 @@ from geonode.resource.manager import resource_manager from geonode.tests.base import GeoNodeBaseTestSupport -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models class GeoAppTests(GeoNodeBaseTestSupport): - """Tests geonode.geoapps module - """ + """Tests geonode.geoapps module""" - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] @classmethod def setUpClass(cls): @@ -57,86 +49,103 @@ def tearDownClass(cls): def setUp(self): super().setUp() - self.bobby = get_user_model().objects.get(username='bobby') + self.bobby = get_user_model().objects.get(username="bobby") self.geo_app = resource_manager.create( None, resource_type=GeoApp, defaults=dict( - title="Testing GeoApp", - owner=self.bobby, - blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}' - ) + title="Testing GeoApp", owner=self.bobby, blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}' + ), ) - self.user = get_user_model().objects.get(username='admin') + self.user = get_user_model().objects.get(username="admin") self.geoapp = GeoApp.objects.create( - name="name", - title="geoapp_titlte", - thumbnail_url='initial', - owner=self.user + name="name", title="geoapp_titlte", thumbnail_url="initial", owner=self.user ) self.sut = GeoAppForm def test_resource_form_is_invalid_extra_metadata_not_json_format(self): self.client.login(username="admin", password="admin") url = reverse("geoapp_metadata", args=(self.geoapp.id,)) - response = self.client.post(url, data={ - "resource-owner": self.geoapp.owner.id, - "resource-title": "geoapp_title", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - "resource-extra_metadata": "not-a-json" - }) - expected = {"success": False, "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"]} + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"], + } self.assertDictEqual(expected, response.json()) @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): self.client.login(username="admin", password="admin") url = reverse("geoapp_metadata", args=(self.geoapp.id,)) - response = self.client.post(url, data={ - "resource-owner": self.geoapp.owner.id, - "resource-title": "geoapp_title", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - "resource-extra_metadata": "[{'key': 'value'}]" - }) - expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource geoapp"]} + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource geoapp"], + } self.assertDictEqual(expected, response.json()) def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): self.client.login(username="admin", password="admin") url = reverse("geoapp_metadata", args=(self.geoapp.id,)) - response = self.client.post(url, data={ - "resource-owner": self.geoapp.owner.id, - "resource-title": "geoapp_title", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' - }) - expected = "extra_metadata: Missing keys: \'field_label\', \'field_name\', \'field_value\', \'filter_header\' at index 0 " - self.assertIn(expected, response.json()['errors'][0]) - - @override_settings(EXTRA_METADATA_SCHEMA={ - "geoapp": { - "id": int, - "filter_header": object, - "field_name": object, - "field_label": object, - "field_value": object + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + expected = ( + "extra_metadata: Missing keys: 'field_label', 'field_name', 'field_value', 'filter_header' at index 0 " + ) + self.assertIn(expected, response.json()["errors"][0]) + + @override_settings( + EXTRA_METADATA_SCHEMA={ + "geoapp": { + "id": int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object, + } } - }) + ) def test_resource_form_is_valid_extra_metadata(self): - form = self.sut(data={ - "owner": self.geoapp.owner.id, - "title": "geoapp_title", - "date": "2022-01-24 16:38 pm", - "date_type": "creation", - "language": "eng", - "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' - }, user=self.user) + form = self.sut( + data={ + "owner": self.geoapp.owner.id, + "title": "geoapp_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + user=self.user, + ) self.assertTrue(form.is_valid()) def test_geoapp_category_is_correctly_assigned_in_metadata_upload(self): @@ -144,20 +153,23 @@ def test_geoapp_category_is_correctly_assigned_in_metadata_upload(self): url = reverse("geoapp_metadata", args=(self.geoapp.id,)) # assign a category to the GeoApp - category = TopicCategory.objects.order_by('identifier').first() + category = TopicCategory.objects.order_by("identifier").first() self.geoapp.category = category self.geoapp.save() # retrieving the new one - new_category = TopicCategory.objects.order_by('identifier').last() - - response = self.client.post(url, data={ - "resource-owner": self.geoapp.owner.id, - "resource-title": "geoapp_title", - "resource-date": "2022-01-24 16:38 pm", - "resource-date_type": "creation", - "resource-language": "eng", - "category_choice_field": new_category.id, - }) + new_category = TopicCategory.objects.order_by("identifier").last() + + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "category_choice_field": new_category.id, + }, + ) self.geoapp.refresh_from_db() self.assertEqual(200, response.status_code) self.assertEqual(new_category.identifier, self.geoapp.category.identifier) @@ -166,12 +178,7 @@ def test_geoapp_copy(self): self.client.login(username="admin", password="admin") geoapp_copy = None try: - geoapp_copy = resource_manager.copy( - self.geoapp, - defaults=dict( - title="Testing GeoApp 2" - ) - ) + geoapp_copy = resource_manager.copy(self.geoapp, defaults=dict(title="Testing GeoApp 2")) self.assertIsNotNone(geoapp_copy) self.assertEqual(geoapp_copy.title, "Testing GeoApp 2") finally: diff --git a/geonode/geoapps/urls.py b/geonode/geoapps/urls.py index d81933683da..59ac3221c8b 100644 --- a/geonode/geoapps/urls.py +++ b/geonode/geoapps/urls.py @@ -21,18 +21,15 @@ from . import views js_info_dict = { - 'packages': ('geonode.geoapps', ), + "packages": ("geonode.geoapps",), } urlpatterns = [ # 'geonode.geoapps.views', - url(r'^new$', views.new_geoapp, name="new_geoapp"), - url(r'^(?P\d+)/metadata$', views.geoapp_metadata, name='geoapp_metadata'), - url(r'^(?P[^/]*)/metadata_detail$', - views.geoapp_metadata_detail, name='geoapp_metadata_detail'), - url(r'^(?P\d+)/metadata_advanced$', - views.geoapp_metadata_advanced, name='geoapp_metadata_advanced'), - url(r'^(?P[^/]+)/embed$', views.geoapp_edit, - {'template': 'apps/app_embed.html'}, name='geoapp_embed'), - url(r'^', include('geonode.geoapps.api.urls')), + url(r"^new$", views.new_geoapp, name="new_geoapp"), + url(r"^(?P\d+)/metadata$", views.geoapp_metadata, name="geoapp_metadata"), + url(r"^(?P[^/]*)/metadata_detail$", views.geoapp_metadata_detail, name="geoapp_metadata_detail"), + url(r"^(?P\d+)/metadata_advanced$", views.geoapp_metadata_advanced, name="geoapp_metadata_advanced"), + url(r"^(?P[^/]+)/embed$", views.geoapp_edit, {"template": "apps/app_embed.html"}, name="geoapp_embed"), + url(r"^", include("geonode.geoapps.api.urls")), ] diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index 652bf314d7d..cae13037529 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -39,21 +39,13 @@ from geonode.monitoring.models import EventType from geonode.base.auth import get_or_create_token from geonode.security.views import _perms_info_json -from geonode.security.utils import ( - get_user_visible_groups, - AdvancedSecurityWorkflowManager) +from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager from geonode.geoapps.models import GeoApp from geonode.resource.manager import resource_manager from geonode.decorators import check_keyword_write_perms -from geonode.base.forms import ( - CategoryForm, - TKeywordForm, - ThesaurusAvailableForm) -from geonode.base.models import ( - Thesaurus, - TopicCategory -) +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory from geonode.utils import resolve_object from .forms import GeoAppForm @@ -64,24 +56,21 @@ _PERMISSION_MSG_GENERIC = _("You do not have permissions for this app.") _PERMISSION_MSG_LOGIN = _("You must be logged in to save this app") _PERMISSION_MSG_SAVE = _("You are not permitted to save or edit this app.") -_PERMISSION_MSG_METADATA = _( - "You are not allowed to modify this app's metadata.") +_PERMISSION_MSG_METADATA = _("You are not allowed to modify this app's metadata.") _PERMISSION_MSG_VIEW = _("You are not allowed to view this app.") _PERMISSION_MSG_UNKNOWN = _("An unknown error has occured.") -def _resolve_geoapp(request, id, permission='base.change_resourcebase', - msg=_PERMISSION_MSG_GENERIC, **kwargs): - ''' +def _resolve_geoapp(request, id, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ Resolve the GeoApp by the provided typename and check the optional permission. - ''' + """ - return resolve_object(request, GeoApp, {"pk": id}, permission=permission, - permission_msg=msg, **kwargs) + return resolve_object(request, GeoApp, {"pk": id}, permission=permission, permission_msg=msg, **kwargs) @login_required -def new_geoapp(request, template='apps/app_new.html'): +def new_geoapp(request, template="apps/app_new.html"): access_token = None if request and request.user: @@ -91,10 +80,10 @@ def new_geoapp(request, template='apps/app_new.html'): else: access_token = None - if request.method == 'GET': + if request.method == "GET": _ctx = { - 'user': request.user, - 'access_token': access_token, + "user": request.user, + "access_token": access_token, } return render(request, template, context=_ctx) @@ -102,17 +91,13 @@ def new_geoapp(request, template='apps/app_new.html'): @xframe_options_sameorigin -def geoapp_edit(request, geoappid, template='apps/app_edit.html'): +def geoapp_edit(request, geoappid, template="apps/app_edit.html"): """ The view that returns the app composer opened to the app with the given app ID. """ try: - geoapp_obj = _resolve_geoapp( - request, - geoappid, - 'base.view_resourcebase', - _PERMISSION_MSG_VIEW) + geoapp_obj = _resolve_geoapp(request, geoappid, "base.view_resourcebase", _PERMISSION_MSG_VIEW) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: @@ -124,8 +109,7 @@ def geoapp_edit(request, geoappid, template='apps/app_edit.html'): permissions_json = _perms_info_json(geoapp_obj) perms_list = list( - geoapp_obj.get_self_resource().get_user_perms(request.user) - .union(geoapp_obj.get_user_perms(request.user)) + geoapp_obj.get_self_resource().get_user_perms(request.user).union(geoapp_obj.get_user_perms(request.user)) ) group = None @@ -136,16 +120,11 @@ def geoapp_edit(request, geoappid, template='apps/app_edit.html'): group = None r = geoapp_obj - if request.method in ('POST', 'PATCH', 'PUT'): - r = resource_manager.update( - geoapp_obj.uuid, - instance=geoapp_obj, - notify=True) + if request.method in ("POST", "PATCH", "PUT"): + r = resource_manager.update(geoapp_obj.uuid, instance=geoapp_obj, notify=True) resource_manager.set_permissions( - geoapp_obj.uuid, - instance=geoapp_obj, - permissions=ast.literal_eval(permissions_json) + geoapp_obj.uuid, instance=geoapp_obj, permissions=ast.literal_eval(permissions_json) ) resource_manager.set_thumbnail(geoapp_obj.uuid, instance=geoapp_obj, overwrite=False) @@ -160,31 +139,24 @@ def geoapp_edit(request, geoappid, template='apps/app_edit.html'): _config = json.dumps(r.blob) _ctx = { - 'appId': geoappid, - 'appType': geoapp_obj.resource_type, - 'config': _config, - 'user': request.user, - 'access_token': access_token, - 'resource': geoapp_obj, - 'group': group, - 'perms_list': perms_list, + "appId": geoappid, + "appType": geoapp_obj.resource_type, + "config": _config, + "user": request.user, + "access_token": access_token, + "resource": geoapp_obj, + "group": group, + "perms_list": perms_list, "permissions_json": permissions_json, - 'preview': getattr( - settings, - 'GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY', - 'mapstore') + "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"), } return render(request, template, context=_ctx) -def geoapp_metadata_detail(request, geoappid, template='apps/app_metadata_detail.html'): +def geoapp_metadata_detail(request, geoappid, template="apps/app_metadata_detail.html"): try: - geoapp_obj = _resolve_geoapp( - request, - geoappid, - 'view_resourcebase', - _PERMISSION_MSG_METADATA) + geoapp_obj = _resolve_geoapp(request, geoappid, "view_resourcebase", _PERMISSION_MSG_METADATA) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: @@ -198,25 +170,17 @@ def geoapp_metadata_detail(request, geoappid, template='apps/app_metadata_detail group = GroupProfile.objects.get(slug=geoapp_obj.group.name) except ObjectDoesNotExist: group = None - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL register_event(request, EventType.EVENT_VIEW_METADATA, geoapp_obj) - return render(request, template, context={ - "resource": geoapp_obj, - "group": group, - 'SITEURL': site_url - }) + return render(request, template, context={"resource": geoapp_obj, "group": group, "SITEURL": site_url}) @login_required @check_keyword_write_perms -def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=True): +def geoapp_metadata(request, geoappid, template="apps/app_metadata.html", ajax=True): geoapp_obj = None try: - geoapp_obj = _resolve_geoapp( - request, - geoappid, - 'base.change_resourcebase_metadata', - _PERMISSION_MSG_METADATA) + geoapp_obj = _resolve_geoapp(request, geoappid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: @@ -235,119 +199,113 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T topic_thesaurus = geoapp_obj.tkeywords.all() if request.method == "POST": - geoapp_form = GeoAppForm( + geoapp_form = GeoAppForm(request.POST, instance=geoapp_obj, prefix="resource", user=request.user) + category_form = CategoryForm( request.POST, - instance=geoapp_obj, - prefix="resource", - user=request.user) - category_form = CategoryForm(request.POST, prefix="category_choice_field", initial=int( - request.POST["category_choice_field"]) if "category_choice_field" in request.POST and - request.POST["category_choice_field"] else None) + prefix="category_choice_field", + initial=int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None, + ) - if hasattr(settings, 'THESAURUS'): + if hasattr(settings, "THESAURUS"): tkeywords_form = TKeywordForm(request.POST) else: - tkeywords_form = ThesaurusAvailableForm(request.POST, prefix='tkeywords') + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") else: geoapp_form = GeoAppForm(instance=geoapp_obj, prefix="resource", user=request.user) geoapp_form.disable_keywords_widget_for_non_superuser(request.user) category_form = CategoryForm( - prefix="category_choice_field", - initial=topic_category.id if topic_category else None) + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) # Create THESAURUS widgets - lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, 'THESAURUS_DEFAULT_LANG') else 'en' - if hasattr(settings, 'THESAURUS') and settings.THESAURUS: - warnings.warn('The settings for Thesaurus has been moved to Model, \ - this feature will be removed in next releases', DeprecationWarning) + lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en" + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) dataset_tkeywords = geoapp_obj.tkeywords.all() - tkeywords_list = '' + tkeywords_list = "" if dataset_tkeywords and len(dataset_tkeywords) > 0: - tkeywords_ids = dataset_tkeywords.values_list('id', flat=True) - if hasattr(settings, 'THESAURUS') and settings.THESAURUS: + tkeywords_ids = dataset_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: el = settings.THESAURUS - thesaurus_name = el['name'] + thesaurus_name = el["name"] try: t = Thesaurus.objects.get(identifier=thesaurus_name) for tk in t.thesaurus.filter(pk__in=tkeywords_ids): tkl = tk.keyword.filter(lang=lang) if len(tkl) > 0: - tkl_ids = ",".join( - map(str, tkl.values_list('id', flat=True))) - tkeywords_list += f",{tkl_ids}" if len( - tkeywords_list) > 0 else tkl_ids + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids except Exception: tb = traceback.format_exc() logger.error(tb) tkeywords_form = TKeywordForm(instance=geoapp_obj) else: - tkeywords_form = ThesaurusAvailableForm(prefix='tkeywords') + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") # set initial values for thesaurus form for tid in tkeywords_form.fields: values = [] values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id] tkeywords_form.fields[tid].initial = values - if request.method == "POST" and geoapp_form.is_valid( - ) and category_form.is_valid() and tkeywords_form.is_valid(): - new_poc = geoapp_form.cleaned_data.pop('poc') - new_author = geoapp_form.cleaned_data.pop('metadata_author') - new_keywords = current_keywords if request.keyword_readonly else geoapp_form.cleaned_data.pop('keywords') - new_regions = geoapp_form.cleaned_data.pop('regions') + if request.method == "POST" and geoapp_form.is_valid() and category_form.is_valid() and tkeywords_form.is_valid(): + new_poc = geoapp_form.cleaned_data.pop("poc") + new_author = geoapp_form.cleaned_data.pop("metadata_author") + new_keywords = current_keywords if request.keyword_readonly else geoapp_form.cleaned_data.pop("keywords") + new_regions = geoapp_form.cleaned_data.pop("regions") new_category = None - if category_form and 'category_choice_field' in category_form.cleaned_data and \ - category_form.cleaned_data['category_choice_field']: - new_category = TopicCategory.objects.get( - id=int(category_form.cleaned_data['category_choice_field'])) + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) if new_poc is None: if poc is None: - poc_form = ProfileForm( - request.POST, - prefix="poc", - instance=poc) + poc_form = ProfileForm(request.POST, prefix="poc", instance=poc) else: poc_form = ProfileForm(request.POST, prefix="poc") if poc_form.is_valid(): - if len(poc_form.cleaned_data['profile']) == 0: + if len(poc_form.cleaned_data["profile"]) == 0: # FIXME use form.add_error in django > 1.7 - errors = poc_form._errors.setdefault( - 'profile', ErrorList()) - errors.append( - _('You must set a point of contact for this resource')) + errors = poc_form._errors.setdefault("profile", ErrorList()) + errors.append(_("You must set a point of contact for this resource")) poc = None if poc_form.has_changed and poc_form.is_valid(): new_poc = poc_form.save() if new_author is None: if metadata_author is None: - author_form = ProfileForm(request.POST, prefix="author", - instance=metadata_author) + author_form = ProfileForm(request.POST, prefix="author", instance=metadata_author) else: author_form = ProfileForm(request.POST, prefix="author") if author_form.is_valid(): - if len(author_form.cleaned_data['profile']) == 0: + if len(author_form.cleaned_data["profile"]) == 0: # FIXME use form.add_error in django > 1.7 - errors = author_form._errors.setdefault( - 'profile', ErrorList()) - errors.append( - _('You must set an author for this resource')) + errors = author_form._errors.setdefault("profile", ErrorList()) + errors.append(_("You must set an author for this resource")) metadata_author = None if author_form.has_changed and author_form.is_valid(): new_author = author_form.save() - geoapp_form.cleaned_data.pop('ptype') + geoapp_form.cleaned_data.pop("ptype") additional_vals = dict( poc=new_poc or geoapp_obj.poc, metadata_author=new_author or geoapp_obj.metadata_author, - category=new_category + category=new_category, ) - geoapp_form.cleaned_data.pop('metadata') - extra_metadata = geoapp_form.cleaned_data.pop('extra_metadata') + geoapp_form.cleaned_data.pop("metadata") + extra_metadata = geoapp_form.cleaned_data.pop("extra_metadata") geoapp_obj = geoapp_form.instance @@ -361,7 +319,7 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T regions=new_regions, vals=_vals, notify=True, - extra_metadata=json.loads(extra_metadata) + extra_metadata=json.loads(extra_metadata), ) resource_manager.set_thumbnail(geoapp_obj.uuid, instance=geoapp_obj, overwrite=False) @@ -375,14 +333,12 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T # Keywords from THESAURUS management # Rewritten to work with updated autocomplete if not tkeywords_form.is_valid(): - return HttpResponse(json.dumps({'message': "Invalid thesaurus keywords"}, status_code=400)) + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) - thesaurus_setting = getattr(settings, 'THESAURUS', None) + thesaurus_setting = getattr(settings, "THESAURUS", None) if thesaurus_setting: - tkeywords_data = tkeywords_form.cleaned_data['tkeywords'] - tkeywords_data = tkeywords_data.filter( - thesaurus__identifier=thesaurus_setting['name'] - ) + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) geoapp_obj.tkeywords.set(tkeywords_data) elif Thesaurus.objects.all().exists(): fields = tkeywords_form.cleaned_data @@ -393,73 +349,67 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T logger.error(tb) vals = {} - if 'group' in geoapp_form.changed_data: - vals['group'] = geoapp_form.cleaned_data.get('group') - if any([x in geoapp_form.changed_data for x in ['is_approved', 'is_published']]): - vals['is_approved'] = geoapp_form.cleaned_data.get('is_approved', geoapp_obj.is_approved) - vals['is_published'] = geoapp_form.cleaned_data.get('is_published', geoapp_obj.is_published) - resource_manager.update( - geoapp_obj.uuid, - instance=geoapp_obj, - notify=True, - vals=vals - ) - return HttpResponse(json.dumps({'message': message})) - elif request.method == "POST" and (not geoapp_form.is_valid( - ) or not category_form.is_valid() or not tkeywords_form.is_valid()): - errors_list = {**geoapp_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} - logger.error(f"GeoApp Metadata form is not valid: {errors_list}") - out = { - 'success': False, - "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()] + if "group" in geoapp_form.changed_data: + vals["group"] = geoapp_form.cleaned_data.get("group") + if any([x in geoapp_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = geoapp_form.cleaned_data.get("is_approved", geoapp_obj.is_approved) + vals["is_published"] = geoapp_form.cleaned_data.get("is_published", geoapp_obj.is_published) + resource_manager.update(geoapp_obj.uuid, instance=geoapp_obj, notify=True, vals=vals) + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not geoapp_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **geoapp_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), } - return HttpResponse( - json.dumps(out), - content_type='application/json', - status=400) + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) # - POST Request Ends here - # Request.GET if poc is not None: - geoapp_form.fields['poc'].initial = poc.id + geoapp_form.fields["poc"].initial = poc.id poc_form = ProfileForm(prefix="poc") poc_form.hidden = True if metadata_author is not None: - geoapp_form.fields['metadata_author'].initial = metadata_author.id + geoapp_form.fields["metadata_author"].initial = metadata_author.id author_form = ProfileForm(prefix="author") author_form.hidden = True metadata_author_groups = get_user_visible_groups(request.user) if not AdvancedSecurityWorkflowManager.is_allowed_to_publish(request.user, geoapp_obj): - geoapp_form.fields['is_published'].widget.attrs.update({'disabled': 'true'}) + geoapp_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) if not AdvancedSecurityWorkflowManager.is_allowed_to_approve(request.user, geoapp_obj): - geoapp_form.fields['is_approved'].widget.attrs.update({'disabled': 'true'}) + geoapp_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) register_event(request, EventType.EVENT_VIEW_METADATA, geoapp_obj) - return render(request, template, context={ - "resource": geoapp_obj, - "geoapp": geoapp_obj, - "geoapp_form": geoapp_form, - "poc_form": poc_form, - "author_form": author_form, - "category_form": category_form, - "tkeywords_form": tkeywords_form, - "metadata_author_groups": metadata_author_groups, - "TOPICCATEGORY_MANDATORY": getattr(settings, 'TOPICCATEGORY_MANDATORY', False), - "GROUP_MANDATORY_RESOURCES": getattr(settings, 'GROUP_MANDATORY_RESOURCES', False), - "UI_MANDATORY_FIELDS": list( - set(getattr(settings, 'UI_DEFAULT_MANDATORY_FIELDS', [])) - | - set(getattr(settings, 'UI_REQUIRED_FIELDS', [])) - ) - }) + return render( + request, + template, + context={ + "resource": geoapp_obj, + "geoapp": geoapp_obj, + "geoapp_form": geoapp_form, + "poc_form": poc_form, + "author_form": author_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + }, + ) @login_required def geoapp_metadata_advanced(request, geoappid): - return geoapp_metadata( - request, - geoappid, - template='apps/app_metadata_advanced.html') + return geoapp_metadata(request, geoappid, template="apps/app_metadata_advanced.html") diff --git a/geonode/geoserver/__init__.py b/geonode/geoserver/__init__.py index 071aad3a8ec..175d879e008 100644 --- a/geonode/geoserver/__init__.py +++ b/geonode/geoserver/__init__.py @@ -53,16 +53,23 @@ def set_resource_links(*args, **kwargs): set_resource_default_links(layer, layer) catalogue_post_save(instance=layer, sender=layer.__class__) except Exception: - logger.exception( - f"[ERROR] Dataset [{_lyr_name}] couldn't be updated" - ) + logger.exception(f"[ERROR] Dataset [{_lyr_name}] couldn't be updated") class GeoserverAppConfig(NotificationsAppConfigBase): - name = 'geonode.geoserver' - NOTIFICATIONS = (("dataset_uploaded", _("Dataset Uploaded"), _("A layer was uploaded"),), - ("dataset_rated", _("Rating for Dataset"), _("A rating was given to a layer"),), - ) + name = "geonode.geoserver" + NOTIFICATIONS = ( + ( + "dataset_uploaded", + _("Dataset Uploaded"), + _("A layer was uploaded"), + ), + ( + "dataset_rated", + _("Rating for Dataset"), + _("A rating was given to a layer"), + ), + ) def ready(self): super().ready() @@ -70,9 +77,10 @@ def ready(self): # Connect the post_migrate signal with the _set_resource_links # method to update links for each resource from django.db.models import signals + signals.post_migrate.connect(set_resource_links, sender=self) -default_app_config = 'geonode.geoserver.GeoserverAppConfig' +default_app_config = "geonode.geoserver.GeoserverAppConfig" -BACKEND_PACKAGE = 'geonode.geoserver' +BACKEND_PACKAGE = "geonode.geoserver" diff --git a/geonode/geoserver/context_processors.py b/geonode/geoserver/context_processors.py index 55bb92d4a30..35525dd900d 100644 --- a/geonode/geoserver/context_processors.py +++ b/geonode/geoserver/context_processors.py @@ -28,26 +28,12 @@ def geoserver_urls(request): GEOSERVER_LOCAL_URL=ogc_server_settings.LOCATION, GEOSERVER_PUBLIC_LOCATION=ogc_server_settings.public_url, GEOSERVER_BASE_URL=ogc_server_settings.public_url, - UPLOADER_URL=reverse('data_upload'), - LAYER_ANCILLARY_FILES_UPLOAD_URL=reverse('dataset_upload'), - MAPFISH_PRINT_ENABLED=getattr(ogc_server_settings, 'MAPFISH_PRINT_ENABLED', False), - PRINT_NG_ENABLED=getattr(ogc_server_settings, 'PRINT_NG_ENABLED', False), - GEONODE_SECURITY_ENABLED=getattr(ogc_server_settings, 'GEONODE_SECURITY_ENABLED', False), - TIME_ENABLED=getattr( - settings, - 'UPLOADER', - dict()).get( - 'OPTIONS', - dict()).get( - 'TIME_ENABLED', - False), - MOSAIC_ENABLED=getattr( - settings, - 'UPLOADER', - dict()).get( - 'OPTIONS', - dict()).get( - 'MOSAIC_ENABLED', - False), + UPLOADER_URL=reverse("data_upload"), + LAYER_ANCILLARY_FILES_UPLOAD_URL=reverse("dataset_upload"), + MAPFISH_PRINT_ENABLED=getattr(ogc_server_settings, "MAPFISH_PRINT_ENABLED", False), + PRINT_NG_ENABLED=getattr(ogc_server_settings, "PRINT_NG_ENABLED", False), + GEONODE_SECURITY_ENABLED=getattr(ogc_server_settings, "GEONODE_SECURITY_ENABLED", False), + TIME_ENABLED=getattr(settings, "UPLOADER", dict()).get("OPTIONS", dict()).get("TIME_ENABLED", False), + MOSAIC_ENABLED=getattr(settings, "UPLOADER", dict()).get("OPTIONS", dict()).get("MOSAIC_ENABLED", False), ) return defaults diff --git a/geonode/geoserver/createlayer/forms.py b/geonode/geoserver/createlayer/forms.py index aadeb2f300e..3c25832f1f5 100644 --- a/geonode/geoserver/createlayer/forms.py +++ b/geonode/geoserver/createlayer/forms.py @@ -21,9 +21,9 @@ from django.utils.translation import ugettext_lazy as _ GEOMETRY_TYPES = ( - ('Point', _('Points')), - ('LineString', _('Lines')), - ('Polygon', _('Polygons')), + ("Point", _("Points")), + ("LineString", _("Lines")), + ("Polygon", _("Polygons")), ) @@ -31,21 +31,15 @@ class NewDatasetForm(forms.Form): """ A form to create an empty layer in PostGIS. """ - name = forms.CharField(label=_('Dataset name'), max_length=255) - title = forms.CharField(label=_('Dataset title'), max_length=255) - geometry_type = forms.ChoiceField(label=_('Geometry type'), choices=GEOMETRY_TYPES) + + name = forms.CharField(label=_("Dataset name"), max_length=255) + title = forms.CharField(label=_("Dataset title"), max_length=255) + geometry_type = forms.ChoiceField(label=_("Geometry type"), choices=GEOMETRY_TYPES) permissions = forms.CharField( - widget=forms.HiddenInput( - attrs={ - 'name': 'permissions', - 'id': 'permissions'}), - required=False) + widget=forms.HiddenInput(attrs={"name": "permissions", "id": "permissions"}), required=False + ) attributes = forms.CharField( - widget=forms.HiddenInput( - attrs={ - 'name': 'attributes', - 'id': 'attributes'}), - required=False, - empty_value='{}') + widget=forms.HiddenInput(attrs={"name": "attributes", "id": "attributes"}), required=False, empty_value="{}" + ) diff --git a/geonode/geoserver/createlayer/tests.py b/geonode/geoserver/createlayer/tests.py index 7be6a57c555..87d40a4d56d 100644 --- a/geonode/geoserver/createlayer/tests.py +++ b/geonode/geoserver/createlayer/tests.py @@ -37,7 +37,8 @@ class CreateLayerCoreTest(GeoNodeBaseTestSupport): """ Test createlayer application. """ - fixtures = ['initial_data.json', 'bobby'] + + fixtures = ["initial_data.json", "bobby"] def setUp(self): super().setUp() @@ -60,22 +61,17 @@ def test_dataset_creation(self): """ Try creating a layer. """ - internal_apps_tests = os.environ.get('TEST_RUN_INTERNAL_APPS', None) + internal_apps_tests = os.environ.get("TEST_RUN_INTERNAL_APPS", None) if not internal_apps_tests: internal_apps_tests = settings.internal_apps_tests else: - dataset_name = 'point_dataset' - dataset_title = 'A layer for points' - print(settings.DATABASES['datastore']) - print('#######') - print(settings.OGC_SERVER['default']['DATASTORE']) - - create_dataset( - dataset_name, - dataset_title, - 'bobby', - 'Point' - ) + dataset_name = "point_dataset" + dataset_title = "A layer for points" + print(settings.DATABASES["datastore"]) + print("#######") + print(settings.OGC_SERVER["default"]["DATASTORE"]) + + create_dataset(dataset_name, dataset_title, "bobby", "Point") cat = gs_catalog @@ -97,7 +93,7 @@ def test_dataset_creation(self): self.assertEqual(resource.projection, layer.srid) # check if layer detail page is accessible with client - response = self.client.get(reverse('dataset_embed', args=(f'geonode:{dataset_name}',))) + response = self.client.get(reverse("dataset_embed", args=(f"geonode:{dataset_name}",))) self.assertEqual(response.status_code, 200) finally: cat.delete(gs_dataset) @@ -107,18 +103,14 @@ def test_dataset_creation_with_wrong_geometry_type(self): Try creating a layer with uncorrect geometry type. """ with self.assertRaises(GeoNodeException): - create_dataset( - 'wrong_geom_dataset', - 'A layer with wrong geometry', - 'bobby', - 'wrong_geometry') + create_dataset("wrong_geom_dataset", "A layer with wrong geometry", "bobby", "wrong_geometry") @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_creation_with_attributes(self): """ Try creating a layer with attributes. """ - internal_apps_tests = os.environ.get('TEST_RUN_INTERNAL_APPS', None) + internal_apps_tests = os.environ.get("TEST_RUN_INTERNAL_APPS", None) if not internal_apps_tests: internal_apps_tests = settings.internal_apps_tests @@ -132,16 +124,10 @@ def test_dataset_creation_with_attributes(self): } """ - dataset_name = 'attributes_dataset' - dataset_title = 'A layer with attributes' + dataset_name = "attributes_dataset" + dataset_title = "A layer with attributes" - create_dataset( - dataset_name, - dataset_title, - 'bobby', - 'Point', - attributes - ) + create_dataset(dataset_name, dataset_title, "bobby", "Point", attributes) cat = gs_catalog gs_dataset = cat.get_layer(dataset_name) diff --git a/geonode/geoserver/createlayer/urls.py b/geonode/geoserver/createlayer/urls.py index 908e57a0f4a..273136dba5a 100644 --- a/geonode/geoserver/createlayer/urls.py +++ b/geonode/geoserver/createlayer/urls.py @@ -20,6 +20,4 @@ from django.conf.urls import url from . import views -urlpatterns = [ - url(r'$', views.dataset_create, name='dataset_create') -] +urlpatterns = [url(r"$", views.dataset_create, name="dataset_create")] diff --git a/geonode/geoserver/createlayer/utils.py b/geonode/geoserver/createlayer/utils.py index 76fec7bd231..6f22749b36e 100644 --- a/geonode/geoserver/createlayer/utils.py +++ b/geonode/geoserver/createlayer/utils.py @@ -30,10 +30,7 @@ from geonode.layers.models import Dataset from geonode.layers.utils import get_valid_name from geonode.resource.manager import resource_manager -from geonode.geoserver.helpers import ( - gs_catalog, - ogc_server_settings, - create_geoserver_db_featurestore) +from geonode.geoserver.helpers import gs_catalog, ogc_server_settings, create_geoserver_db_featurestore logger = logging.getLogger(__name__) @@ -47,15 +44,15 @@ def create_dataset(name, title, owner_name, geometry_type, attributes=None): Create an empty layer in GeoServer and register it in GeoNode. """ # first validate parameters - if geometry_type not in ('Point', 'LineString', 'Polygon'): - msg = 'geometry must be Point, LineString or Polygon' + if geometry_type not in ("Point", "LineString", "Polygon"): + msg = "geometry must be Point, LineString or Polygon" logger.error(msg) raise GeoNodeException(msg) name = get_valid_name(name) # we can proceed - logger.debug('Creating the layer in GeoServer') + logger.debug("Creating the layer in GeoServer") workspace, datastore = create_gs_dataset(name, title, geometry_type, attributes) - logger.debug('Creating the layer in GeoNode') + logger.debug("Creating the layer in GeoNode") return create_gn_dataset(workspace, datastore, name, title, owner_name) @@ -72,21 +69,22 @@ def create_gn_dataset(workspace, datastore, name, title, owner_name): name=name, workspace=workspace.name, store=datastore.name, - subtype='vector', - alternate=f'{workspace.name}:{name}', + subtype="vector", + alternate=f"{workspace.name}:{name}", title=title, owner=owner, - srid='EPSG:4326', + srid="EPSG:4326", bbox_polygon=Polygon.from_bbox(BBOX), ll_bbox_polygon=Polygon.from_bbox(BBOX), - data_quality_statement=DATA_QUALITY_MESSAGE - )) + data_quality_statement=DATA_QUALITY_MESSAGE, + ), + ) to_update = {} if settings.ADMIN_MODERATE_UPLOADS: - to_update['is_approved'] = to_update['was_approved'] = False + to_update["is_approved"] = to_update["was_approved"] = False if settings.RESOURCE_PUBLISHING: - to_update['is_published'] = to_update['was_published'] = False + to_update["is_published"] = to_update["was_published"] = False resource_manager.update(layer.uuid, instance=layer, vals=to_update) resource_manager.set_thumbnail(None, instance=layer) @@ -121,9 +119,9 @@ def get_attributes(geometry_type, json_attrs=None): lattrs = [] gattr = [] - gattr.append('the_geom') - gattr.append(f'com.vividsolutions.jts.geom.{geometry_type}') - gattr.append({'nillable': False}) + gattr.append("the_geom") + gattr.append(f"com.vividsolutions.jts.geom.{geometry_type}") + gattr.append({"nillable": False}) lattrs.append(gattr) if json_attrs: jattrs = json.loads(json_attrs) @@ -132,20 +130,20 @@ def get_attributes(geometry_type, json_attrs=None): attr_name = slugify(jattr[0]) attr_type = jattr[1].lower() if len(attr_name) == 0: - msg = f'You must provide an attribute name for attribute of type {attr_type}' + msg = f"You must provide an attribute name for attribute of type {attr_type}" logger.error(msg) raise GeoNodeException(msg) - if attr_type not in ('float', 'date', 'string', 'integer'): - msg = f'{attr_type} is not a valid type for attribute {attr_name}' + if attr_type not in ("float", "date", "string", "integer"): + msg = f"{attr_type} is not a valid type for attribute {attr_name}" logger.error(msg) raise GeoNodeException(msg) - if attr_type == 'date': - attr_type = f'java.util.{(attr_type[:1].upper() + attr_type[1:])}' + if attr_type == "date": + attr_type = f"java.util.{(attr_type[:1].upper() + attr_type[1:])}" else: - attr_type = f'java.lang.{(attr_type[:1].upper() + attr_type[1:])}' + attr_type = f"java.lang.{(attr_type[:1].upper() + attr_type[1:])}" lattr.append(attr_name) lattr.append(attr_type) - lattr.append({'nillable': True}) + lattr.append({"nillable": True}) lattrs.append(lattr) return lattrs @@ -154,7 +152,7 @@ def get_or_create_datastore(cat, workspace=None, charset="UTF-8"): """ Get a PostGIS database store or create it in GeoServer if does not exist. """ - dsname = ogc_server_settings.datastore_db['NAME'] + dsname = ogc_server_settings.datastore_db["NAME"] ds = create_geoserver_db_featurestore(store_name=dsname, workspace=workspace) return ds @@ -175,8 +173,8 @@ def create_gs_dataset(name, title, geometry_type, attributes=None): datastore = get_or_create_datastore(cat, workspace) # check if datastore is of PostGIS type - if datastore.type != 'PostGIS': - msg = ("To use the createlayer application you must use PostGIS") + if datastore.type != "PostGIS": + msg = "To use the createlayer application you must use PostGIS" logger.error(msg) raise GeoNodeException(msg) @@ -198,7 +196,8 @@ def create_gs_dataset(name, title, geometry_type, attributes=None): f"{att_name}" f"{binding}" f"{nillable}" - "") + "" + ) attributes_block += "" # TODO implement others srs and not only EPSG:4326 @@ -211,17 +210,16 @@ def create_gs_dataset(name, title, geometry_type, attributes=None): f"{BBOX[0]}{BBOX[2]}{BBOX[1]}{BBOX[3]}" f"EPSG:4326" f"{attributes_block}" - "") - - url = ( - f'{ogc_server_settings.rest}/workspaces/{workspace.name}/datastores/{datastore.name}/featuretypes' + "" ) - headers = {'Content-Type': 'application/xml'} + + url = f"{ogc_server_settings.rest}/workspaces/{workspace.name}/datastores/{datastore.name}/featuretypes" + headers = {"Content-Type": "application/xml"} _user, _password = ogc_server_settings.credentials req = requests.post(url, data=xml, headers=headers, auth=(_user, _password)) if req.status_code != 201: - logger.error(f'Request status code was: {req.status_code}') - logger.error(f'Response was: {req.text}') + logger.error(f"Request status code was: {req.status_code}") + logger.error(f"Response was: {req.text}") raise Exception(f"Dataset could not be created in GeoServer {req.text}") cat.reload() diff --git a/geonode/geoserver/createlayer/views.py b/geonode/geoserver/createlayer/views.py index 1d07e46846a..0a165172da7 100644 --- a/geonode/geoserver/createlayer/views.py +++ b/geonode/geoserver/createlayer/views.py @@ -31,33 +31,33 @@ @login_required -def dataset_create(request, template='createlayer/dataset_create.html'): +def dataset_create(request, template="createlayer/dataset_create.html"): """ Create an empty layer. """ error = None - if request.method == 'POST': + if request.method == "POST": form = NewDatasetForm(request.POST) if form.is_valid(): try: - name = form.cleaned_data['name'] + name = form.cleaned_data["name"] name = slugify(name.replace(".", "_")) - title = form.cleaned_data['title'] - geometry_type = form.cleaned_data['geometry_type'] - attributes = form.cleaned_data['attributes'] + title = form.cleaned_data["title"] + geometry_type = form.cleaned_data["geometry_type"] + attributes = form.cleaned_data["attributes"] permissions = DEFAULT_PERMS_SPEC layer = create_dataset(name, title, request.user.username, geometry_type, attributes) layer.set_permissions(json.loads(permissions), created=True) return redirect(layer) except Exception as e: - error = f'{e} ({type(e)})' + error = f"{e} ({type(e)})" else: form = NewDatasetForm() ctx = { - 'form': form, - 'is_dataset': True, - 'error': error, + "form": form, + "is_dataset": True, + "error": error, } return render(request, template, context=ctx) diff --git a/geonode/geoserver/geofence.py b/geonode/geoserver/geofence.py index d8c385c64e4..fb1d4f424f5 100644 --- a/geonode/geoserver/geofence.py +++ b/geonode/geoserver/geofence.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -ogc_server_settings = settings.OGC_SERVER['default'] +ogc_server_settings = settings.OGC_SERVER["default"] class GeofenceException(Exception): @@ -58,10 +58,20 @@ class Rule: LIMIT = "LIMIT" CM_MIXED = "MIXED" - def __init__(self, priority, workspace, layer, access: (str, bool), - user=None, group=None, - service=None, request=None, subfield=None, - geo_limit=None, catalog_mode=None) -> None: + def __init__( + self, + priority, + workspace, + layer, + access: (str, bool), + user=None, + group=None, + service=None, + request=None, + subfield=None, + geo_limit=None, + catalog_mode=None, + ) -> None: self.fields = {} # access may be either a boolean or ALLOW/DENY/LIMIT @@ -71,37 +81,33 @@ def __init__(self, priority, workspace, layer, access: (str, bool), access = Rule.DENY for field, value in ( - ('priority', priority), - - ('userName', user), - ('roleName', group), - - ('service', service), - ('request', request), - ('subfield', subfield), - - ('workspace', workspace), - ('layer', layer), - - ('access', access), + ("priority", priority), + ("userName", user), + ("roleName", group), + ("service", service), + ("request", request), + ("subfield", subfield), + ("workspace", workspace), + ("layer", layer), + ("access", access), ): - if value is not None and value != '*': + if value is not None and value != "*": self.fields[field] = value limits = {} for field, value in ( - ('allowedArea', geo_limit), - ('catalogMode', catalog_mode), + ("allowedArea", geo_limit), + ("catalogMode", catalog_mode), ): if value is not None: limits[field] = value if limits: - self.fields['limits'] = limits + self.fields["limits"] = limits def get_object(self): logger.debug(f"Creating Rule object: {self.fields}") - return {'Rule': self.fields} + return {"Rule": self.fields} class Batch: @@ -181,22 +187,18 @@ class Batch: def __init__(self, log_name=None) -> None: self.operations = [] - self.log_name = f'"{log_name}"' if log_name else '' + self.log_name = f'"{log_name}"' if log_name else "" def __str__(self) -> str: return super().__str__() def add_delete_rule(self, rule_id: int): - self.operations.append({ - '@service': 'rules', - '@type': 'delete', - '@id': rule_id - }) + self.operations.append({"@service": "rules", "@type": "delete", "@id": rule_id}) def add_insert_rule(self, rule: Rule): operation = { - '@service': 'rules', - '@type': 'insert', + "@service": "rules", + "@type": "insert", } operation.update(rule.get_object()) self.operations.append(operation) @@ -206,11 +208,7 @@ def get_batch_length(self): def get_object(self): logger.debug(f"Creating Batch object {self.log_name} with {len(self.operations)} operations") - return { - 'Batch': { - 'operations': self.operations - } - } + return {"Batch": {"operations": self.operations}} class GeofenceClient: @@ -229,16 +227,14 @@ def __init__(self, baseurl: str, username: str, pw: str) -> None: def invalidate_cache(self): r = requests.put( - f'{self.baseurl.rstrip("/")}/geofence/ruleCache/invalidate', - auth=HTTPBasicAuth(self.username, self.pw)) + f'{self.baseurl.rstrip("/")}/geofence/ruleCache/invalidate', auth=HTTPBasicAuth(self.username, self.pw) + ) if r.status_code != 200: logger.debug("Could not invalidate cache") raise GeofenceException("Could not invalidate cache") - def get_rules(self, page=None, entries=None, - workspace=None, workspace_any=None, - layer=None, layer_any=None): + def get_rules(self, page=None, entries=None, workspace=None, workspace_any=None, layer=None, layer_any=None): if (page is None and entries is not None) or (page is not None and entries is None): raise GeofenceException(f"Bad page/entries combination {page}/{entries}") @@ -250,13 +246,13 @@ def get_rules(self, page=None, entries=None, params = {} if entries: - params.update({'page': page, 'entries': entries}) + params.update({"page": page, "entries": entries}) for param, value in ( - ('workspace', workspace), - ('workspaceAny', workspace_any), - ('layer', layer), - ('layerAny', layer_any), + ("workspace", workspace), + ("workspaceAny", workspace_any), + ("layer", layer), + ("layerAny", layer_any), ): if value is not None: params[param] = value @@ -265,10 +261,11 @@ def get_rules(self, page=None, entries=None, r = requests.get( url, - headers={'Content-type': 'application/json'}, + headers={"Content-type": "application/json"}, auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 10), - verify=False) + timeout=ogc_server_settings.get("TIMEOUT", 10), + verify=False, + ) if r.status_code != 200: logger.debug(f"Could not retrieve GeoFence Rules from {url} -- code:{r.status_code} - {r.text}") @@ -288,17 +285,18 @@ def get_rules_count(self): """ r = requests.get( f'{self.baseurl.rstrip("/")}/geofence/rules/count.json', - headers={'Content-type': 'application/json'}, + headers={"Content-type": "application/json"}, auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 10), - verify=False) + timeout=ogc_server_settings.get("TIMEOUT", 10), + verify=False, + ) if r.status_code != 200: logger.debug(f"Could not retrieve GeoFence Rules count: [{r.status_code}] - {r.text}") raise GeofenceException(f"Could not retrieve GeoFence Rules count: [{r.status_code}]") response = r.json() - return response['count'] + return response["count"] except Exception as e: logger.debug("Error while retrieving GeoFence rules count", exc_info=e) @@ -316,8 +314,9 @@ def insert_rule(self, rule: Rule): # headers={'Content-type': 'application/json'}, json=rule.get_object(), auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 60), - verify=False) + timeout=ogc_server_settings.get("TIMEOUT", 60), + verify=False, + ) if r.status_code not in (200, 201): logger.debug(f"Could not insert rule: [{r.status_code}] - {r.content}") @@ -329,7 +328,7 @@ def insert_rule(self, rule: Rule): def run_batch(self, batch: Batch): if batch.get_batch_length() == 0: - logger.debug(f'Skipping batch execution {batch.log_name}') + logger.debug(f"Skipping batch execution {batch.log_name}") return try: @@ -341,8 +340,9 @@ def run_batch(self, batch: Batch): f'{self.baseurl.rstrip("/")}/geofence/batch/exec', json=batch.get_object(), auth=HTTPBasicAuth(self.username, self.pw), - timeout=ogc_server_settings.get('TIMEOUT', 60), - verify=False) + timeout=ogc_server_settings.get("TIMEOUT", 60), + verify=False, + ) if r.status_code != 200: logger.debug(f"Error while running batch {batch.log_name}: [{r.status_code}] - {r.content}") @@ -357,28 +357,26 @@ def run_batch(self, batch: Batch): def purge_all_rules(self): """purge all existing GeoFence Cache Rules""" rules_objs = self.get_rules() - rules = rules_objs['rules'] + rules = rules_objs["rules"] - batch = Batch('Purge All') + batch = Batch("Purge All") for rule in rules: - batch.add_delete_rule(rule['id']) + batch.add_delete_rule(rule["id"]) logger.debug(f"Going to remove all {len(rules)} rules in geofence") self.run_batch(batch) def purge_layer_rules(self, layer_name: str, workspace: str = None): """purge existing GeoFence Cache Rules related to a specific Layer""" - gs_rules = self.get_rules( - workspace=workspace, workspace_any=False, - layer=layer_name, layer_any=False) + gs_rules = self.get_rules(workspace=workspace, workspace_any=False, layer=layer_name, layer_any=False) - batch = Batch(f'Purge {workspace}:{layer_name}') + batch = Batch(f"Purge {workspace}:{layer_name}") - if gs_rules and gs_rules['rules']: + if gs_rules and gs_rules["rules"]: logger.debug(f"Going to remove {len(gs_rules['rules'])} rules for layer '{layer_name}'") - for r in gs_rules['rules']: - if r['layer'] and r['layer'] == layer_name: - batch.add_delete_rule(r['id']) + for r in gs_rules["rules"]: + if r["layer"] and r["layer"] == layer_name: + batch.add_delete_rule(r["id"]) else: logger.debug(f"Bad rule retrieved for dataset '{layer_name}': {r}") self.run_batch(batch) diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index 2500e8ea3f4..4d83c12f5e0 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -52,8 +52,13 @@ from geoserver.catalog import Catalog, FailedRequestError from geoserver.resource import FeatureType, Coverage -from geoserver.store import CoverageStore, DataStore, datastore_from_index, \ - coveragestore_from_index, wmsstore_from_index +from geoserver.store import ( + CoverageStore, + DataStore, + datastore_from_index, + coveragestore_from_index, + wmsstore_from_index, +) from geoserver.support import DimensionInfo from geoserver.workspace import Workspace from gsimporter import Client @@ -74,51 +79,53 @@ http_client, get_legend_url, is_monochromatic_image, - set_resource_default_links) + set_resource_default_links, +) from .geofence import GeofenceClient logger = logging.getLogger(__name__) -temp_style_name_regex = r'[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}_ms_.*' +temp_style_name_regex = r"[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}_ms_.*" LAYER_SUBTYPES = { "dataStore": "vector", "coverageStore": "raster", "remoteStore": "remote", - "vectorTimeSeries": "vector_time" + "vectorTimeSeries": "vector_time", } WPS_ACCEPTABLE_FORMATS = [ - ('application/json', 'vector'), - ('application/arcgrid', 'raster'), - ('image/tiff', 'raster'), - ('image/png', 'raster'), - ('image/jpeg', 'raster'), - ('application/wfs-collection-1.0', 'vector'), - ('application/wfs-collection-1.1', 'vector'), - ('application/zip', 'vector'), - ('text/csv', 'vector') + ("application/json", "vector"), + ("application/arcgrid", "raster"), + ("image/tiff", "raster"), + ("image/png", "raster"), + ("image/jpeg", "raster"), + ("application/wfs-collection-1.0", "vector"), + ("application/wfs-collection-1.1", "vector"), + ("application/zip", "vector"), + ("text/csv", "vector"), ] -DEFAULT_STYLE_NAME = ['generic', 'line', 'point', 'polygon', 'raster'] +DEFAULT_STYLE_NAME = ["generic", "line", "point", "polygon", "raster"] -if not hasattr(settings, 'OGC_SERVER'): +if not hasattr(settings, "OGC_SERVER"): msg = ( - 'Please configure OGC_SERVER when enabling geonode.geoserver.' - ' More info can be found at ' - 'http://docs.geonode.org/en/2.10.x/basic/settings/index.html#ogc-server') + "Please configure OGC_SERVER when enabling geonode.geoserver." + " More info can be found at " + "http://docs.geonode.org/en/2.10.x/basic/settings/index.html#ogc-server" + ) raise ImproperlyConfigured(msg) def check_geoserver_is_up(): """Verifies all geoserver is running, - this is needed to be able to upload. + this is needed to be able to upload. """ url = f"{ogc_server_settings.LOCATION}" req, content = http_client.get(url, user=_user) - msg = f'Cannot connect to the GeoServer at {url}\nPlease make sure you have started it.' + msg = f"Cannot connect to the GeoServer at {url}\nPlease make sure you have started it." logger.debug(req) assert req.status_code == 200, msg @@ -129,7 +136,8 @@ def _add_sld_boilerplate(symbolizer): elements to make it a valid SLD which applies that symbolizer to all features, including format strings to allow interpolating a "name" variable in. """ - return """ + return ( + """ @@ -140,13 +148,16 @@ def _add_sld_boilerplate(symbolizer): %(name)s -""" + symbolizer + """ +""" + + symbolizer + + """ """ + ) _raster_template = """ @@ -206,18 +217,15 @@ def _add_sld_boilerplate(symbolizer): raster=_add_sld_boilerplate(_raster_template), polygon=_add_sld_boilerplate(_polygon_template), line=_add_sld_boilerplate(_line_template), - point=_add_sld_boilerplate(_point_template) + point=_add_sld_boilerplate(_point_template), ) -STYLES_VERSION = { - "1.0.0": "sld10", - "1.1.0": "sld11" -} +STYLES_VERSION = {"1.0.0": "sld10", "1.1.0": "sld11"} def _extract_style_version_from_sld(sld): """ - Assume: SLD as a byte + Assume: SLD as a byte """ root = objectify.fromstring(sld) try: @@ -237,23 +245,20 @@ def extract_name_from_sld(gs_catalog, sld, sld_file=None): with open(sld, "rb") as sld_file: sld = sld_file.read() if isinstance(sld, str): - sld = sld.encode('utf-8') + sld = sld.encode("utf-8") dom = etree.XML(sld, parser=etree.XMLParser(resolve_entities=False)) elif sld_file and isfile(sld_file): with open(sld_file, "rb") as sld_file: sld = sld_file.read() if isinstance(sld, str): - sld = sld.encode('utf-8') + sld = sld.encode("utf-8") dom = dlxml.parse(sld) except Exception: logger.exception("The uploaded SLD file is not valid XML") - raise Exception( - "The uploaded SLD file is not valid XML") + raise Exception("The uploaded SLD file is not valid XML") - named_dataset = dom.findall( - "{http://www.opengis.net/sld}NamedLayer") - user_dataset = dom.findall( - "{http://www.opengis.net/sld}UserLayer") + named_dataset = dom.findall("{http://www.opengis.net/sld}NamedLayer") + user_dataset = dom.findall("{http://www.opengis.net/sld}UserLayer") el = None if named_dataset and len(named_dataset) > 0: @@ -285,8 +290,7 @@ def extract_name_from_sld(gs_catalog, sld, sld_file=None): if sld_file: return splitext(basename(sld_file))[0] else: - raise Exception( - "Please provide a name, unable to extract one from the SLD.") + raise Exception("Please provide a name, unable to extract one from the SLD.") return el[0].text @@ -339,7 +343,7 @@ def get_sld_for(gs_catalog, layer): # Detect geometry type if it is a FeatureType res = gs_dataset.resource if gs_dataset else None - if res and res.resource_type == 'featureType': + if res and res.resource_type == "featureType": res.fetch() ft = res.store.get_resources(name=res.name) ft.fetch() @@ -358,11 +362,7 @@ def get_sld_for(gs_catalog, layer): # style. if name in _style_templates: fg, bg, mark = next(_style_contexts) - return _style_templates[name] % dict( - name=layer.name, - fg=fg, - bg=bg, - mark=mark) + return _style_templates[name] % dict(name=layer.name, fg=fg, bg=bg, mark=mark) else: return gs_style @@ -376,8 +376,8 @@ def set_dataset_style(saved_dataset, title, sld, base_file=None): sld = sld_file.read() elif isinstance(sld, str): - sld = sld.strip('b\'\n') - sld = re.sub(r'(\\r)|(\\n)', '', sld).encode("UTF-8") + sld = sld.strip("b'\n") + sld = re.sub(r"(\\r)|(\\n)", "", sld).encode("UTF-8") etree.XML(sld, parser=etree.XMLParser(resolve_entities=False)) elif base_file and isfile(base_file): with open(base_file, "rb") as sld_file: @@ -389,8 +389,7 @@ def set_dataset_style(saved_dataset, title, sld, base_file=None): # Check Dataset's available styles match = None - styles = list(saved_dataset.styles.all()) + [ - saved_dataset.default_style] + styles = list(saved_dataset.styles.all()) + [saved_dataset.default_style] for style in styles: if style and style.name == saved_dataset.name: match = style @@ -399,44 +398,46 @@ def set_dataset_style(saved_dataset, title, sld, base_file=None): style = None if match is None: try: - style = gs_catalog.get_style(saved_dataset.name, workspace=saved_dataset.workspace) or \ - gs_catalog.get_style(saved_dataset.name) + style = gs_catalog.get_style(saved_dataset.name, workspace=saved_dataset.workspace) or gs_catalog.get_style( + saved_dataset.name + ) if not style: style = gs_catalog.create_style( - saved_dataset.name, sld, overwrite=False, raw=True, workspace=saved_dataset.workspace) + saved_dataset.name, sld, overwrite=False, raw=True, workspace=saved_dataset.workspace + ) except Exception as e: logger.exception(e) else: try: _sld_format = _extract_style_version_from_sld(sld) style = gs_catalog.create_style( - saved_dataset.name, sld, - overwrite=True, raw=True, style_format=_sld_format, - workspace=saved_dataset.workspace) + saved_dataset.name, + sld, + overwrite=True, + raw=True, + style_format=_sld_format, + workspace=saved_dataset.workspace, + ) except Exception as e: logger.exception(e) if layer and style: _old_styles = [] - _old_styles.append(gs_catalog.get_style( - name=saved_dataset.name)) - _old_styles.append(gs_catalog.get_style( - name=f"{saved_dataset.workspace}_{saved_dataset.name}")) + _old_styles.append(gs_catalog.get_style(name=saved_dataset.name)) + _old_styles.append(gs_catalog.get_style(name=f"{saved_dataset.workspace}_{saved_dataset.name}")) if layer.default_style and layer.default_style.name: - _old_styles.append(gs_catalog.get_style( - name=layer.default_style.name)) - _old_styles.append(gs_catalog.get_style( - name=layer.default_style.name, - workspace=layer.default_style.workspace)) + _old_styles.append(gs_catalog.get_style(name=layer.default_style.name)) + _old_styles.append( + gs_catalog.get_style(name=layer.default_style.name, workspace=layer.default_style.workspace) + ) layer.default_style = style gs_catalog.save(layer) for _s in _old_styles: try: gs_catalog.delete(_s) Link.objects.filter( - resource=saved_dataset.resourcebase_ptr, - name='Legend', - url__contains=f'STYLE={_s.name}').delete() + resource=saved_dataset.resourcebase_ptr, name="Legend", url__contains=f"STYLE={_s.name}" + ).delete() except Exception as e: logger.debug(e) set_styles(saved_dataset, gs_catalog) @@ -449,8 +450,8 @@ def cascading_delete(dataset_name=None, catalog=None): resource = None workspace = None try: - if dataset_name.find(':') != -1 and len(dataset_name.split(':')) == 2: - workspace, name = dataset_name.split(':') + if dataset_name.find(":") != -1 and len(dataset_name.split(":")) == 2: + workspace, name = dataset_name.split(":") ws = cat.get_workspace(workspace) store = None try: @@ -462,21 +463,20 @@ def cascading_delete(dataset_name=None, catalog=None): for layer in layers: store = get_store(cat, layer.store, workspace=ws) except FailedRequestError: - logger.debug( - 'the store was not found in geoserver') + logger.debug("the store was not found in geoserver") else: - logger.debug( - 'the store was not found in geoserver') + logger.debug("the store was not found in geoserver") if ws is None or store is None: - logger.debug( - 'cascading delete was called on a layer where the workspace was not found') + logger.debug("cascading delete was called on a layer where the workspace was not found") resource = cat.get_resource(name=name, store=store, workspace=workspace) else: resource = cat.get_resource(name=dataset_name) except OSError as e: if e.errno == errno.ECONNREFUSED: - msg = (f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' - f'to save information for layer "{dataset_name}"') + msg = ( + f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' + f'to save information for layer "{dataset_name}"' + ) logger.error(msg) return None else: @@ -491,8 +491,7 @@ def cascading_delete(dataset_name=None, catalog=None): # If there is no associated resource, # this method can not delete anything. # Let's return and make a note in the log. - logger.debug( - 'cascading_delete was called with a non existent resource') + logger.debug("cascading_delete was called with a non existent resource") return resource_name = resource.name lyr = None @@ -518,7 +517,7 @@ def cascading_delete(dataset_name=None, catalog=None): if s is not None and s.name not in _default_style_names: try: logger.debug(f"Trying to delete Style [{s.name}]") - cat.delete(s, purge='true') + cat.delete(s, purge="true") except Exception as e: # Trying to delete a shared style will fail # We'll catch the exception and log it. @@ -532,18 +531,21 @@ def cascading_delete(dataset_name=None, catalog=None): except Exception: pass - if store.resource_type == 'dataStore' and 'dbtype' in store.connection_parameters and \ - store.connection_parameters['dbtype'] == 'postgis': + if ( + store.resource_type == "dataStore" + and "dbtype" in store.connection_parameters + and store.connection_parameters["dbtype"] == "postgis" + ): delete_from_postgis(resource_name, store) else: # AF: for the time being this one mitigates the issue #8671 # until we find a suitable solution for the GeoTools ImageMosaic plugin # ref: https://github.com/geotools/geotools/blob/main/modules/plugin/imagemosaic/src/main/java/org/geotools/gce/imagemosaic/catalog/AbstractGTDataStoreGranuleCatalog.java#L753 - if store.resource_type == 'coverageStore' and store.type != 'ImageMosaic': + if store.resource_type == "coverageStore" and store.type != "ImageMosaic": try: logger.debug(f" - Going to purge the {store.resource_type} : {store.href}") cat.reset() # this resets the coverage readers and unlocks the files - cat.delete(store, purge='all', recurse=True) + cat.delete(store, purge="all", recurse=True) # cat.reload() # this preservers the integrity of geoserver except Exception as e: # Trying to recursively purge a store may fail @@ -568,11 +570,11 @@ def delete_from_postgis(dataset_name, store): # we will assume that store/database may change (when using shard for example) # but user and password are the ones from settings (DATASTORE_URL) db = ogc_server_settings.datastore_db - db_name = store.connection_parameters['database'] - user = db['USER'] - password = db['PASSWORD'] - host = store.connection_parameters['host'] - port = store.connection_parameters['port'] + db_name = store.connection_parameters["database"] + user = db["USER"] + password = db["PASSWORD"] + host = store.connection_parameters["host"] + port = store.connection_parameters["port"] conn = None try: conn = psycopg2.connect(dbname=db_name, user=user, host=host, port=port, password=password) @@ -580,10 +582,7 @@ def delete_from_postgis(dataset_name, store): cur.execute(f"SELECT DropGeometryTable ('{dataset_name}')") conn.commit() except Exception as e: - logger.error( - "Error deleting PostGIS table %s:%s", - dataset_name, - str(e)) + logger.error("Error deleting PostGIS table %s:%s", dataset_name, str(e)) finally: try: if conn: @@ -593,26 +592,27 @@ def delete_from_postgis(dataset_name, store): def gs_slurp( - ignore_errors=False, - verbosity=1, - console=None, - owner=None, - workspace=None, - store=None, - filter=None, - skip_unadvertised=False, - skip_geonode_registered=False, - remove_deleted=False, - permissions=None, - execute_signals=False): + ignore_errors=False, + verbosity=1, + console=None, + owner=None, + workspace=None, + store=None, + filter=None, + skip_unadvertised=False, + skip_geonode_registered=False, + remove_deleted=False, + permissions=None, + execute_signals=False, +): """Configure the layers available in GeoServer in GeoNode. - It returns a list of dictionaries with the name of the layer, - the result of the operation and the errors and traceback if it failed. + It returns a list of dictionaries with the name of the layer, + the result of the operation and the errors and traceback if it failed. """ from geonode.resource.manager import resource_manager if console is None: - console = open(os.devnull, 'w') + console = open(os.devnull, "w") if verbosity > 0: print("Inspecting the available layers in GeoServer ...", file=console) @@ -647,11 +647,11 @@ def gs_slurp( # enabled = true, if --skip-unadvertised: advertised = true, but # disregard the filter parameter in the case of deleting layers try: - resources_for_delete_compare = [ - k for k in resources_for_delete_compare if k.enabled in {"true", True}] + resources_for_delete_compare = [k for k in resources_for_delete_compare if k.enabled in {"true", True}] if skip_unadvertised: resources_for_delete_compare = [ - k for k in resources_for_delete_compare if k.advertised in {"true", True}] + k for k in resources_for_delete_compare if k.advertised in {"true", True} + ] except Exception: if ignore_errors: pass @@ -684,11 +684,10 @@ def gs_slurp( raise # filter out layers already registered in geonode - dataset_names = Dataset.objects.values_list('alternate', flat=True) + dataset_names = Dataset.objects.values_list("alternate", flat=True) if skip_geonode_registered: try: - resources = [k for k in resources - if f'{k.workspace.name}:{k.name}' not in dataset_names] + resources = [k for k in resources if f"{k.workspace.name}:{k.name}" not in dataset_names] except Exception: if ignore_errors: pass @@ -704,14 +703,14 @@ def gs_slurp( msg = "Found %d layers, starting processing" % number print(msg, file=console) output = { - 'stats': { - 'failed': 0, - 'updated': 0, - 'created': 0, - 'deleted': 0, + "stats": { + "failed": 0, + "updated": 0, + "created": 0, + "deleted": 0, }, - 'layers': [], - 'deleted_datasets': [] + "layers": [], + "deleted_datasets": [], } start = datetime.datetime.now(timezone.get_current_timezone()) for i, resource in enumerate(resources): @@ -732,10 +731,10 @@ def gs_slurp( store=the_store.name, subtype=get_dataset_storetype(the_store.resource_type), alternate=f"{workspace.name}:{resource.name}", - title=resource.title or _('No title provided'), - abstract=resource.abstract or _('No abstract provided'), - owner=owner - ) + title=resource.title or _("No title provided"), + abstract=resource.abstract or _("No abstract provided"), + owner=owner, + ), ) created = True # Hide the resource until finished @@ -749,37 +748,29 @@ def gs_slurp( raise else: logger.exception(e) - layer.srid = 'EPSG:4326' + layer.srid = "EPSG:4326" layer.set_ll_bbox_polygon([ll_bbox[0], ll_bbox[2], ll_bbox[1], ll_bbox[3]]) # sync permissions in GeoFence perm_spec = json.loads(_perms_info_json(layer)) - resource_manager.set_permissions( - layer.uuid, - permissions=perm_spec) + resource_manager.set_permissions(layer.uuid, permissions=perm_spec) # recalculate the layer statistics set_attributes_from_geoserver(layer, overwrite=True) # in some cases we need to explicitily save the resource to execute the signals # (for sure when running updatelayers) - resource_manager.update( - layer.uuid, - instance=layer, - notify=execute_signals) + resource_manager.update(layer.uuid, instance=layer, notify=execute_signals) # Creating the Thumbnail - resource_manager.set_thumbnail( - layer.uuid, - overwrite=True, check_bbox=False - ) + resource_manager.set_thumbnail(layer.uuid, overwrite=True, check_bbox=False) except Exception as e: # Hide the resource until finished if layer: layer.set_processing_state("FAILED") if ignore_errors: - status = 'failed' + status = "failed" exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: @@ -788,7 +779,7 @@ def gs_slurp( raise Exception(f"Failed to process {resource.name}") from e if layer is None: if ignore_errors: - status = 'failed' + status = "failed" exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: @@ -802,20 +793,20 @@ def gs_slurp( else: layer.set_permissions(permissions) - status = 'created' - output['stats']['created'] += 1 + status = "created" + output["stats"]["created"] += 1 else: - status = 'updated' - output['stats']['updated'] += 1 + status = "updated" + output["stats"]["updated"] += 1 msg = f"[{status}] Dataset {name} ({(i + 1)}/{number})" - info = {'name': name, 'status': status} - if status == 'failed': - output['stats']['failed'] += 1 - info['traceback'] = traceback - info['exception_type'] = exception_type - info['error'] = error - output['layers'].append(info) + info = {"name": name, "status": status} + if status == "failed": + output["stats"]["failed"] += 1 + info["traceback"] = traceback + info["exception_type"] = exception_type + info["error"] = error + output["layers"].append(info) if verbosity > 0: print(msg, file=console) @@ -823,16 +814,11 @@ def gs_slurp( q = Dataset.objects.filter() if workspace_for_delete_compare is not None: if isinstance(workspace_for_delete_compare, Workspace): - q = q.filter( - workspace__exact=workspace_for_delete_compare.name) + q = q.filter(workspace__exact=workspace_for_delete_compare.name) else: q = q.filter(workspace__exact=workspace_for_delete_compare) if store is not None: - if isinstance( - store, - CoverageStore) or isinstance( - store, - DataStore): + if isinstance(store, CoverageStore) or isinstance(store, DataStore): q = q.filter(store__exact=store.name) else: q = q.filter(store__exact=store) @@ -846,10 +832,8 @@ def gs_slurp( deleted_datasets = [] for layer in q: logger.debug( - "GeoNode Dataset info: name: %s, workspace: %s, store: %s", - layer.name, - layer.workspace, - layer.store) + "GeoNode Dataset info: name: %s, workspace: %s, store: %s", layer.name, layer.workspace, layer.store + ) dataset_found_in_geoserver = False for resource in resources_for_delete_compare: # if layer.name matches a GeoServer resource, check also that @@ -860,18 +844,20 @@ def gs_slurp( "Matches GeoServer layer: name: %s, workspace: %s, store: %s", resource.name, resource.workspace.name, - resource.store.name) + resource.store.name, + ) dataset_found_in_geoserver = True if not dataset_found_in_geoserver: - logger.debug( - "----- Dataset %s not matched, marked for deletion ---------------", - layer.name) + logger.debug("----- Dataset %s not matched, marked for deletion ---------------", layer.name) deleted_datasets.append(layer) number_deleted = len(deleted_datasets) if verbosity > 0: - msg = "\nFound %d layers to delete, starting processing" % number_deleted if number_deleted > 0 else \ - "\nFound %d layers to delete" % number_deleted + msg = ( + "\nFound %d layers to delete, starting processing" % number_deleted + if number_deleted > 0 + else "\nFound %d layers to delete" % number_deleted + ) print(msg, file=console) for i, layer in enumerate(deleted_datasets): @@ -879,36 +865,34 @@ def gs_slurp( "GeoNode Dataset to delete: name: %s, workspace: %s, store: %s", layer.name, layer.workspace, - layer.store) + layer.store, + ) try: # delete ratings, and taggit tags: ct = ContentType.objects.get_for_model(layer) - OverallRating.objects.filter( - content_type=ct, - object_id=layer.id).delete() + OverallRating.objects.filter(content_type=ct, object_id=layer.id).delete() layer.keywords.clear() layer.delete() - output['stats']['deleted'] += 1 + output["stats"]["deleted"] += 1 status = "delete_succeeded" except Exception: status = "delete_failed" msg = f"[{status}] Dataset {layer.name} ({(i + 1)}/{number_deleted})" - info = {'name': layer.name, 'status': status} + info = {"name": layer.name, "status": status} if status == "delete_failed": exception_type, error, traceback = sys.exc_info() - info['traceback'] = traceback - info['exception_type'] = exception_type - info['error'] = error - output['deleted_datasets'].append(info) + info["traceback"] = traceback + info["exception_type"] = exception_type + info["error"] = error + output["deleted_datasets"].append(info) if verbosity > 0: print(msg, file=console) finish = datetime.datetime.now(timezone.get_current_timezone()) td = finish - start - output['stats']['duration_sec'] = td.microseconds / \ - 1000000 + td.seconds + td.days * 24 * 3600 + output["stats"]["duration_sec"] = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 return output @@ -918,33 +902,29 @@ def get_stores(store_type=None): store_list = [] for store in stores: store.fetch() - stype = store.dom.find('type').text.lower() + stype = store.dom.find("type").text.lower() if store_type and store_type.lower() == stype: - store_list.append({'name': store.name, 'type': stype}) + store_list.append({"name": store.name, "type": stype}) elif store_type is None: - store_list.append({'name': store.name, 'type': stype}) + store_list.append({"name": store.name, "type": stype}) return store_list -def set_attributes( - layer, - attribute_map, - overwrite=False, - attribute_stats=None): - """ *layer*: a geonode.layers.models.Dataset instance - *attribute_map*: a list of 2-lists specifying attribute names and types, - example: [ ['id', 'Integer'], ... ] - *overwrite*: replace existing attributes with new values if name/type matches. - *attribute_stats*: dictionary of return values from get_attribute_statistics(), - of the form to get values by referencing attribute_stats[][]. +def set_attributes(layer, attribute_map, overwrite=False, attribute_stats=None): + """*layer*: a geonode.layers.models.Dataset instance + *attribute_map*: a list of 2-lists specifying attribute names and types, + example: [ ['id', 'Integer'], ... ] + *overwrite*: replace existing attributes with new values if name/type matches. + *attribute_stats*: dictionary of return values from get_attribute_statistics(), + of the form to get values by referencing attribute_stats[][]. """ # we need 3 more items; description, attribute_label, and display_order attribute_map_dict = { - 'field': 0, - 'ftype': 1, - 'description': 2, - 'label': 3, - 'display_order': 4, + "field": 0, + "ftype": 1, + "description": 2, + "label": 3, + "display_order": 4, } for attribute in attribute_map: if len(attribute) == 2: @@ -959,14 +939,11 @@ def set_attributes( if field == la.attribute: lafound = True # store description and attribute_label in attribute_map - attribute[attribute_map_dict['description']] = la.description - attribute[attribute_map_dict['label']] = la.attribute_label - attribute[attribute_map_dict['display_order']] = la.display_order + attribute[attribute_map_dict["description"]] = la.description + attribute[attribute_map_dict["label"]] = la.attribute_label + attribute[attribute_map_dict["display_order"]] = la.display_order if overwrite or not lafound: - logger.debug( - "Going to delete [%s] for [%s]", - la.attribute, - layer.name) + logger.debug("Going to delete [%s] for [%s]", la.attribute, layer.name) la.delete() # Add new layer attributes if they doesn't exist already @@ -988,21 +965,20 @@ def set_attributes( la.attribute_label = label la.display_order = iter iter += 1 - if (not attribute_stats or layer.name not in attribute_stats or - field not in attribute_stats[layer.name]): + if not attribute_stats or layer.name not in attribute_stats or field not in attribute_stats[layer.name]: result = None else: result = attribute_stats[layer.name][field] if result: logger.debug("Generating layer attribute statistics") - la.count = result['Count'] - la.min = result['Min'] - la.max = result['Max'] - la.average = result['Average'] - la.median = result['Median'] - la.stddev = result['StandardDeviation'] - la.sum = result['Sum'] - la.unique_values = result['unique_values'] + la.count = result["Count"] + la.min = result["Min"] + la.max = result["Max"] + la.average = result["Average"] + la.median = result["Median"] + la.stddev = result["StandardDeviation"] + la.sum = result["Sum"] + la.unique_values = result["unique_values"] la.last_stats_updated = datetime.datetime.now(timezone.get_current_timezone()) try: la.save() @@ -1018,23 +994,26 @@ def set_attributes_from_geoserver(layer, overwrite=False): then store in GeoNode database using Attribute model """ attribute_map = [] - if getattr(layer, 'remote_service') and layer.remote_service: + if getattr(layer, "remote_service") and layer.remote_service: server_url = layer.remote_service.service_url - if layer.remote_service.operations.get('GetCapabilities', None) and layer.remote_service.operations.get('GetCapabilities').get('methods'): - for _method in layer.remote_service.operations.get('GetCapabilities').get('methods'): - if _method.get('type', '').upper() == 'GET': - server_url = _method.get('url', server_url) + if layer.remote_service.operations.get("GetCapabilities", None) and layer.remote_service.operations.get( + "GetCapabilities" + ).get("methods"): + for _method in layer.remote_service.operations.get("GetCapabilities").get("methods"): + if _method.get("type", "").upper() == "GET": + server_url = _method.get("url", server_url) break else: server_url = ogc_server_settings.LOCATION - if layer.subtype in ['tileStore', 'remote'] and layer.remote_service.ptype == "gxp_arcrestsource": + if layer.subtype in ["tileStore", "remote"] and layer.remote_service.ptype == "gxp_arcrestsource": dft_url = f"{server_url}{(layer.alternate or layer.typename)}?f=json" try: # The code below will fail if http_client cannot be imported req, body = http_client.get(dft_url, user=_user) body = json.loads(body) - attribute_map = [[n["name"], _esri_types[n["type"]]] - for n in body["fields"] if n.get("name") and n.get("type")] + attribute_map = [ + [n["name"], _esri_types[n["type"]]] for n in body["fields"] if n.get("name") and n.get("type") + ] except Exception: tb = traceback.format_exc() logger.debug(tb) @@ -1043,12 +1022,7 @@ def set_attributes_from_geoserver(layer, overwrite=False): typename = layer.alternate if layer.alternate else layer.typename dft_url_path = re.sub(r"\/wms\/?$", "/", server_url) dft_query = urlencode( - { - "service": "wfs", - "version": "1.0.0", - "request": "DescribeFeatureType", - "typename": typename - } + {"service": "wfs", "version": "1.0.0", "request": "DescribeFeatureType", "typename": typename} ) dft_url = urljoin(dft_url_path, f"ows?{dft_query}") try: @@ -1057,32 +1031,41 @@ def set_attributes_from_geoserver(layer, overwrite=False): doc = dlxml.fromstring(body.encode()) xsd = "{http://www.w3.org/2001/XMLSchema}" path = f".//{xsd}extension/{xsd}sequence/{xsd}element" - attribute_map = [[n.attrib["name"], n.attrib["type"]] for n in doc.findall( - path) if n.attrib.get("name") and n.attrib.get("type")] + attribute_map = [ + [n.attrib["name"], n.attrib["type"]] + for n in doc.findall(path) + if n.attrib.get("name") and n.attrib.get("type") + ] except Exception: tb = traceback.format_exc() logger.debug(tb) attribute_map = [] # Try WMS instead - dft_url = server_url + "?" + urlencode({ - "service": "wms", - "version": "1.0.0", - "request": "GetFeatureInfo", - "bbox": ','.join([str(x) for x in layer.bbox]), - "LAYERS": layer.alternate, - "QUERY_LAYERS": typename, - "feature_count": 1, - "width": 1, - "height": 1, - "srs": "EPSG:4326", - "info_format": "text/html", - "x": 1, - "y": 1 - }) + dft_url = ( + server_url + + "?" + + urlencode( + { + "service": "wms", + "version": "1.0.0", + "request": "GetFeatureInfo", + "bbox": ",".join([str(x) for x in layer.bbox]), + "LAYERS": layer.alternate, + "QUERY_LAYERS": typename, + "feature_count": 1, + "width": 1, + "height": 1, + "srs": "EPSG:4326", + "info_format": "text/html", + "x": 1, + "y": 1, + } + ) + ) try: req, body = http_client.get(dft_url, user=_user) soup = BeautifulSoup(body, features="lxml") - for field in soup.findAll('th'): + for field in soup.findAll("th"): if field.string is None: field_name = field.contents[0].string else: @@ -1113,18 +1096,13 @@ def set_attributes_from_geoserver(layer, overwrite=False): if field is not None: if Attribute.objects.filter(dataset=layer, attribute=field).exists(): continue - elif is_dataset_attribute_aggregable( - layer.subtype, - field, - ftype): + elif is_dataset_attribute_aggregable(layer.subtype, field, ftype): logger.debug("Generating layer attribute statistics") result = get_attribute_statistics(layer.alternate or layer.typename, field) else: result = None attribute_stats[layer.name][field] = result - set_attributes( - layer, attribute_map, overwrite=overwrite, attribute_stats=attribute_stats - ) + set_attributes(layer, attribute_map, overwrite=overwrite, attribute_stats=attribute_stats) def get_dataset(layer, gs_catalog: Catalog): @@ -1152,18 +1130,15 @@ def clean_styles(layer, gs_catalog: Catalog): gs_catalog.reset() gs_dataset = get_dataset(layer, gs_catalog) logger.debug(f'clean_styles: Retrieving style "{gs_dataset.default_style.name}" for cleanup') - style = gs_catalog.get_style( - name=gs_dataset.default_style.name, - workspace=None, - recursive=True) + style = gs_catalog.get_style(name=gs_dataset.default_style.name, workspace=None, recursive=True) if style: gs_catalog.delete(style, purge=True, recurse=False) - logger.debug(f'clean_styles: Style removed: {gs_dataset.default_style.name}') + logger.debug(f"clean_styles: Style removed: {gs_dataset.default_style.name}") else: - logger.debug(f'clean_styles: Style does not exist: {gs_dataset.default_style.name}') + logger.debug(f"clean_styles: Style does not exist: {gs_dataset.default_style.name}") except Exception as e: - logger.warning(f'Could not clean style for layer {layer.name}', exc_info=e) - logger.debug(f'Could not clean style for layer {layer.name} - STACK INFO', stack_info=True) + logger.warning(f"Could not clean style for layer {layer.name}", exc_info=e) + logger.debug(f"Could not clean style for layer {layer.name} - STACK INFO", stack_info=True) def set_styles(layer, gs_catalog: Catalog): @@ -1175,21 +1150,28 @@ def set_styles(layer, gs_catalog: Catalog): # make sure we are not using a default SLD (which won't be editable) layer.default_style, _gs_default_style = save_style(default_style, layer) try: - if default_style.name != _gs_default_style.name or default_style.workspace != _gs_default_style.workspace: + if ( + default_style.name != _gs_default_style.name + or default_style.workspace != _gs_default_style.workspace + ): logger.debug(f'set_style: Setting default style "{_gs_default_style.name}" for layer "{layer.name}') gs_dataset.default_style = _gs_default_style gs_catalog.save(gs_dataset) if default_style.name not in DEFAULT_STYLE_NAME: - logger.debug(f'set_style: Retrieving no-workspace default style "{default_style.name}" for deletion') + logger.debug( + f'set_style: Retrieving no-workspace default style "{default_style.name}" for deletion' + ) style_to_delete = gs_catalog.get_style(name=default_style.name, workspace=None, recursive=True) if style_to_delete: gs_catalog.delete(style_to_delete, purge=True, recurse=False) - logger.debug(f'set_style: No-ws default style deleted: {default_style.name}') + logger.debug(f"set_style: No-ws default style deleted: {default_style.name}") else: - logger.debug(f'set_style: No-ws default style does not exist: {default_style.name}') + logger.debug(f"set_style: No-ws default style does not exist: {default_style.name}") except Exception as e: - logger.error(f'Error setting default style "{_gs_default_style.name}" for layer "{layer.name}', exc_info=e) + logger.error( + f'Error setting default style "{_gs_default_style.name}" for layer "{layer.name}', exc_info=e + ) style_set.append(layer.default_style) @@ -1197,7 +1179,12 @@ def set_styles(layer, gs_catalog: Catalog): if gs_dataset.styles: alt_styles = gs_dataset.styles for alt_style in alt_styles: - if alt_style and alt_style.name and alt_style.name != layer.default_style.name and alt_style.workspace != layer.default_style.workspace: + if ( + alt_style + and alt_style.name + and alt_style.name != layer.default_style.name + and alt_style.workspace != layer.default_style.workspace + ): _s, _ = save_style(alt_style, layer) style_set.append(_s) except Exception as e: @@ -1211,9 +1198,7 @@ def set_styles(layer, gs_catalog: Catalog): clean_styles(layer, gs_catalog) # Update default style to database - to_update = { - 'default_style': layer.default_style - } + to_update = {"default_style": layer.default_style} Dataset.objects.filter(id=layer.id).update(**to_update) layer.refresh_from_db() @@ -1222,23 +1207,28 @@ def set_styles(layer, gs_catalog: Catalog): logger.debug(f" -- Resource Links[Legend link] for layer {layer.name}...") try: from geonode.base.models import Link - dataset_legends = Link.objects.filter(resource=layer.resourcebase_ptr, name='Legend') - for style in set(list(layer.styles.all()) + [layer.default_style, ]): + + dataset_legends = Link.objects.filter(resource=layer.resourcebase_ptr, name="Legend") + for style in set( + list(layer.styles.all()) + + [ + layer.default_style, + ] + ): if style: - style_name = os.path.basename( - urlparse(style.sld_url).path).split('.')[0] + style_name = os.path.basename(urlparse(style.sld_url).path).split(".")[0] legend_url = get_legend_url(layer, style_name) - if dataset_legends.filter(resource=layer.resourcebase_ptr, name='Legend', url=legend_url).count() < 2: + if dataset_legends.filter(resource=layer.resourcebase_ptr, name="Legend", url=legend_url).count() < 2: Link.objects.update_or_create( resource=layer.resourcebase_ptr, - name='Legend', + name="Legend", url=legend_url, defaults=dict( - extension='png', + extension="png", url=legend_url, - mime='image/png', - link_type='image', - ) + mime="image/png", + link_type="image", + ), ) logger.debug(" -- Resource Links[Legend link]...done!") except Exception as e: @@ -1246,6 +1236,7 @@ def set_styles(layer, gs_catalog: Catalog): try: from .security import set_geowebcache_invalidate_cache + set_geowebcache_invalidate_cache(layer.alternate or layer.typename, cat=gs_catalog) except Exception: tb = traceback.format_exc() @@ -1253,29 +1244,24 @@ def set_styles(layer, gs_catalog: Catalog): def save_style(gs_style, layer): - style_name = os.path.basename( - urlparse(gs_style.body_href).path).split('.')[0] + style_name = os.path.basename(urlparse(gs_style.body_href).path).split(".")[0] sld_name = copy.copy(gs_style.name) sld_body = copy.copy(gs_style.sld_body) _gs_style = None if not gs_style.workspace: logger.debug(f'save_style: Copying style "{sld_name}" to "{layer.workspace}:{layer.name}') - _gs_style = gs_catalog.create_style( - layer.name, sld_body, - raw=True, overwrite=True, - workspace=layer.workspace) + _gs_style = gs_catalog.create_style(layer.name, sld_body, raw=True, overwrite=True, workspace=layer.workspace) else: - logger.debug(f'save_style: Retrieving style "{layer.workspace}:{sld_name}" for layer "{layer.workspace}:{layer.name}') - _gs_style = gs_catalog.get_style( - name=sld_name, - workspace=layer.workspace + logger.debug( + f'save_style: Retrieving style "{layer.workspace}:{sld_name}" for layer "{layer.workspace}:{layer.name}' ) + _gs_style = gs_catalog.get_style(name=sld_name, workspace=layer.workspace) style = None try: style, _ = Style.objects.get_or_create(name=style_name) style.workspace = _gs_style.workspace - style.sld_title = _gs_style.sld_title if _gs_style.style_format != 'css' and _gs_style.sld_title else sld_name + style.sld_title = _gs_style.sld_title if _gs_style.style_format != "css" and _gs_style.sld_title else sld_name style.sld_body = _gs_style.sld_body style.sld_url = _gs_style.body_href style.save() @@ -1292,13 +1278,13 @@ def is_dataset_attribute_aggregable(store_type, field_name, field_type): """ # must be vector layer - if store_type != 'dataStore': + if store_type != "dataStore": return False # must be a numeric data type if field_type not in LAYER_ATTRIBUTE_NUMERIC_DATA_TYPES: return False # must not be an identifier type field - if field_name.lower() in {'id', 'identifier'}: + if field_name.lower() in {"id", "identifier"}: return False return True @@ -1310,7 +1296,7 @@ def get_attribute_statistics(dataset_name, field): for layer attribute """ - logger.debug('Deriving aggregate statistics for attribute %s', field) + logger.debug("Deriving aggregate statistics for attribute %s", field) if not ogc_server_settings.WPS_ENABLED: return None @@ -1319,21 +1305,19 @@ def get_attribute_statistics(dataset_name, field): except Exception: tb = traceback.format_exc() logger.debug(tb) - logger.exception('Error generating layer aggregate statistics') + logger.exception("Error generating layer aggregate statistics") def get_wcs_record(instance, retry=True): - wcs = WebCoverageService(f"{ogc_server_settings.LOCATION}wcs", '1.0.0') + wcs = WebCoverageService(f"{ogc_server_settings.LOCATION}wcs", "1.0.0") key = f"{instance.workspace}:{instance.name}" logger.debug(wcs.contents) if key in wcs.contents: return wcs.contents[key] else: - msg = (f"Dataset '{key}' was not found in WCS service at {ogc_server_settings.public_url}." - ) + msg = f"Dataset '{key}' was not found in WCS service at {ogc_server_settings.public_url}." if retry: - logger.debug( - f"{msg} Waiting a couple of seconds before trying again.") + logger.debug(f"{msg} Waiting a couple of seconds before trying again.") time.sleep(2) return get_wcs_record(instance, retry=False) else: @@ -1342,33 +1326,32 @@ def get_wcs_record(instance, retry=True): def get_coverage_grid_extent(instance): """ - Returns a list of integers with the size of the coverage - extent in pixels + Returns a list of integers with the size of the coverage + extent in pixels """ instance_wcs = get_wcs_record(instance) grid = instance_wcs.grid - return [(int(h) - int(l) + 1) for - h, l in zip(grid.highlimits, grid.lowlimits)] + return [(int(h) - int(l) + 1) for h, l in zip(grid.highlimits, grid.lowlimits)] GEOSERVER_LAYER_TYPES = { - 'vector': FeatureType.resource_type, - 'raster': Coverage.resource_type, + "vector": FeatureType.resource_type, + "raster": Coverage.resource_type, } def cleanup(name, uuid): """Deletes GeoServer and Catalogue records for a given name. - Useful to clean the mess when something goes terribly wrong. - It also verifies if the Django record existed, in which case - it performs no action. + Useful to clean the mess when something goes terribly wrong. + It also verifies if the Django record existed, in which case + it performs no action. """ try: Dataset.objects.get(name=name) except Dataset.DoesNotExist: pass else: - msg = f'Not doing any cleanup because the layer {name} exists in the Django db.' + msg = f"Not doing any cleanup because the layer {name} exists in the Django db." raise GeoNodeException(msg) cat = gs_catalog @@ -1387,8 +1370,7 @@ def cleanup(name, uuid): gs_dataset = None gs_resource = None except FailedRequestError as e: - msg = ('Couldn\'t connect to GeoServer while cleaning up layer ' - '[%s] !!', str(e)) + msg = ("Couldn't connect to GeoServer while cleaning up layer " "[%s] !!", str(e)) logger.warning(msg) if gs_dataset is not None: @@ -1400,7 +1382,7 @@ def cleanup(name, uuid): try: cat.delete(gs_resource) except Exception: - msg = 'Couldn\'t delete GeoServer resource during cleanup()' + msg = "Couldn't delete GeoServer resource during cleanup()" logger.warning(msg) if gs_store is not None: try: @@ -1408,21 +1390,24 @@ def cleanup(name, uuid): except Exception: logger.warning("Couldn't delete GeoServer store during cleanup()") - logger.warning('Deleting dangling Catalogue record for [%s] ' - '(no Django record to match)', name) + logger.warning("Deleting dangling Catalogue record for [%s] " "(no Django record to match)", name) - if 'geonode.catalogue' in settings.INSTALLED_APPS: + if "geonode.catalogue" in settings.INSTALLED_APPS: from geonode.catalogue import get_catalogue + catalogue = get_catalogue() catalogue.remove_record(uuid) - logger.warning('Finished cleanup after failed Catalogue/Django ' - 'import for layer: %s', name) + logger.warning("Finished cleanup after failed Catalogue/Django " "import for layer: %s", name) def create_geoserver_db_featurestore( - store_type=None, store_name=None, - author_name='admin', author_email='admin@geonode.org', - charset="UTF-8", workspace=None): + store_type=None, + store_name=None, + author_name="admin", + author_email="admin@geonode.org", + charset="UTF-8", + workspace=None, +): cat = gs_catalog dsname = store_name or ogc_server_settings.DATASTORE # get or create datastore @@ -1436,51 +1421,50 @@ def create_geoserver_db_featurestore( raise FailedRequestError ds_exists = True except FailedRequestError: - logger.debug( - f'Creating target datastore {dsname}') + logger.debug(f"Creating target datastore {dsname}") ds = cat.create_datastore(dsname, workspace=workspace) db = ogc_server_settings.datastore_db - db_engine = 'postgis' if \ - 'postgis' in db['ENGINE'] else db['ENGINE'] + db_engine = "postgis" if "postgis" in db["ENGINE"] else db["ENGINE"] ds.connection_parameters.update( - {'Evictor run periodicity': 300, - 'Estimated extends': 'true', - 'fetch size': 100000, - 'encode functions': 'false', - 'Expose primary keys': 'true', - 'validate connections': 'true', - 'Support on the fly geometry simplification': 'false', - 'Connection timeout': 10, - 'create database': 'false', - 'Batch insert size': 30, - 'preparedStatements': 'true', - 'min connections': 10, - 'max connections': 100, - 'Evictor tests per run': 3, - 'Max connection idle time': 300, - 'Loose bbox': 'true', - 'Test while idle': 'true', - 'host': db['HOST'], - 'port': db['PORT'] if isinstance( - db['PORT'], str) else str(db['PORT']) or '5432', - 'database': db['NAME'], - 'user': db['USER'], - 'passwd': db['PASSWORD'], - 'dbtype': db_engine} + { + "Evictor run periodicity": 300, + "Estimated extends": "true", + "fetch size": 100000, + "encode functions": "false", + "Expose primary keys": "true", + "validate connections": "true", + "Support on the fly geometry simplification": "false", + "Connection timeout": 10, + "create database": "false", + "Batch insert size": 30, + "preparedStatements": "true", + "min connections": 10, + "max connections": 100, + "Evictor tests per run": 3, + "Max connection idle time": 300, + "Loose bbox": "true", + "Test while idle": "true", + "host": db["HOST"], + "port": db["PORT"] if isinstance(db["PORT"], str) else str(db["PORT"]) or "5432", + "database": db["NAME"], + "user": db["USER"], + "passwd": db["PASSWORD"], + "dbtype": db_engine, + } ) if ds_exists: ds.save_method = "PUT" else: - logger.debug('Updating target datastore % s' % dsname) + logger.debug("Updating target datastore % s" % dsname) try: cat.save(ds) except FailedRequestError as e: - if 'already exists in workspace' not in e.args[0]: + if "already exists in workspace" not in e.args[0]: raise e logger.warning("The store was already present in the workspace selected") - logger.debug('Reloading target datastore % s' % dsname) + logger.debug("Reloading target datastore % s" % dsname) ds = get_store(cat, dsname, workspace=workspace) assert ds.enabled @@ -1511,16 +1495,11 @@ def _create_db_featurestore(name, data, overwrite=False, charset="UTF-8", worksp cat = gs_catalog db = ogc_server_settings.datastore_db # dsname = ogc_server_settings.DATASTORE - dsname = db['NAME'] + dsname = db["NAME"] ds = create_geoserver_db_featurestore(store_name=dsname, workspace=workspace) try: - cat.add_data_to_store(ds, - name, - data, - overwrite=overwrite, - workspace=workspace, - charset=charset) + cat.add_data_to_store(ds, name, data, overwrite=overwrite, workspace=workspace, charset=charset) resource = cat.get_resource(name=name, store=ds, workspace=workspace) assert resource is not None return ds, resource @@ -1546,21 +1525,21 @@ def get_store(cat, name, workspace=None): if workspace: try: - store = cat.get_xml(f'{workspace.datastore_url[:-4]}/{name}.xml') + store = cat.get_xml(f"{workspace.datastore_url[:-4]}/{name}.xml") except FailedRequestError: try: - store = cat.get_xml(f'{workspace.coveragestore_url[:-4]}/{name}.xml') + store = cat.get_xml(f"{workspace.coveragestore_url[:-4]}/{name}.xml") except FailedRequestError: try: - store = cat.get_xml(f'{workspace.wmsstore_url[:-4]}/{name}.xml') + store = cat.get_xml(f"{workspace.wmsstore_url[:-4]}/{name}.xml") except FailedRequestError: raise FailedRequestError(f"No store found named: {name}") if store: - if store.tag == 'dataStore': + if store.tag == "dataStore": store = datastore_from_index(cat, workspace, store) - elif store.tag == 'coverageStore': + elif store.tag == "coverageStore": store = coveragestore_from_index(cat, workspace, store) - elif store.tag == 'wmsStore': + elif store.tag == "wmsStore": store = wmsstore_from_index(cat, workspace, store) return store else: @@ -1572,39 +1551,42 @@ def get_store(cat, name, workspace=None): def fetch_gs_resource(instance, values, tries): _max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 2) try: - gs_resource = gs_catalog.get_resource( - name=instance.name, - store=instance.store, - workspace=instance.workspace) + gs_resource = gs_catalog.get_resource(name=instance.name, store=instance.store, workspace=instance.workspace) except Exception: try: gs_resource = gs_catalog.get_resource( - name=instance.alternate, - store=instance.store, - workspace=instance.workspace) + name=instance.alternate, store=instance.store, workspace=instance.workspace + ) except Exception: try: - gs_resource = gs_catalog.get_resource( - name=instance.alternate or instance.typename) + gs_resource = gs_catalog.get_resource(name=instance.alternate or instance.typename) except Exception: gs_resource = None if gs_resource: if values: - gs_resource.title = values.get('title', '') - gs_resource.abstract = values.get('abstract', '') + gs_resource.title = values.get("title", "") + gs_resource.abstract = values.get("abstract", "") else: values = {} _subtype = gs_resource.store.resource_type - if getattr(gs_resource, 'metadata', None) and gs_resource.metadata.get('time', False) and gs_resource.metadata.get('time').enabled: + if ( + getattr(gs_resource, "metadata", None) + and gs_resource.metadata.get("time", False) + and gs_resource.metadata.get("time").enabled + ): _subtype = "vectorTimeSeries" - values.update(dict(store=gs_resource.store.name, - subtype=_subtype, - alternate=f"{gs_resource.store.workspace.name}:{gs_resource.name}", - title=gs_resource.title or gs_resource.store.name, - abstract=gs_resource.abstract or '', - owner=instance.owner)) + values.update( + dict( + store=gs_resource.store.name, + subtype=_subtype, + alternate=f"{gs_resource.store.workspace.name}:{gs_resource.name}", + title=gs_resource.title or gs_resource.store.name, + abstract=gs_resource.abstract or "", + owner=instance.owner, + ) + ) else: msg = f"There isn't a geoserver resource for this layer: {instance.name}" logger.debug(msg) @@ -1619,92 +1601,71 @@ def wps_execute_dataset_attribute_statistics(dataset_name, field): """Derive aggregate statistics from WPS endpoint""" # generate statistics using WPS - url = urljoin(ogc_server_settings.LOCATION, 'ows') + url = urljoin(ogc_server_settings.LOCATION, "ows") - request = render_to_string('layers/wps_execute_gs_aggregate.xml', { - 'dataset_name': dataset_name, - 'field': field - }) + request = render_to_string("layers/wps_execute_gs_aggregate.xml", {"dataset_name": dataset_name, "field": field}) u = urlsplit(url) headers = { - 'User-Agent': 'OWSLib (https://geopython.github.io/OWSLib)', - 'Content-type': 'text/xml', - 'Accept': 'text/xml', - 'Accept-Language': 'en-US', - 'Accept-Encoding': 'gzip,deflate', - 'Host': u.netloc, + "User-Agent": "OWSLib (https://geopython.github.io/OWSLib)", + "Content-type": "text/xml", + "Accept": "text/xml", + "Accept-Language": "en-US", + "Accept-Encoding": "gzip,deflate", + "Host": u.netloc, } response, content = http_client.request( - url, - method='POST', - data=request, - headers=headers, - user=_user, - timeout=5, - retries=1) + url, method="POST", data=request, headers=headers, user=_user, timeout=5, retries=1 + ) exml = dlxml.fromstring(content.encode()) result = {} - for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']: + for f in ["Min", "Max", "Average", "Median", "StandardDeviation", "Sum"]: fr = exml.find(f) if fr is not None: result[f] = fr.text else: - result[f] = 'NA' + result[f] = "NA" - count = exml.find('Count') + count = exml.find("Count") if count is not None: - result['Count'] = int(count.text) + result["Count"] = int(count.text) else: - result['Count'] = 0 + result["Count"] = 0 - result['unique_values'] = 'NA' + result["unique_values"] = "NA" return result def _stylefilterparams_geowebcache_dataset(dataset_name): - headers = { - "Content-Type": "text/xml" - } - url = f'{ogc_server_settings.LOCATION}gwc/rest/layers/{dataset_name}.xml' + headers = {"Content-Type": "text/xml"} + url = f"{ogc_server_settings.LOCATION}gwc/rest/layers/{dataset_name}.xml" # read GWC configuration - req, content = http_client.get( - url, - headers=headers, - user=_user) + req, content = http_client.get(url, headers=headers, user=_user) if req.status_code != 200: - logger.error( - f"Error {req.status_code} reading Style Filter Params GeoWebCache at {url}" - ) + logger.error(f"Error {req.status_code} reading Style Filter Params GeoWebCache at {url}") return # check/write GWC filter parameters body = None tree = dlxml.fromstring(_) - param_filters = tree.findall('parameterFilters') + param_filters = tree.findall("parameterFilters") if param_filters and len(param_filters) > 0: - if not param_filters[0].findall('styleParameterFilter'): + if not param_filters[0].findall("styleParameterFilter"): style_filters_xml = "STYLES\ " style_filters_elem = dlxml.fromstring(style_filters_xml) param_filters[0].append(style_filters_elem) body = ET.tostring(tree) if body: - req, content = http_client.post( - url, - data=body, - headers=headers, - user=_user) + req, content = http_client.post(url, data=body, headers=headers, user=_user) if req.status_code != 200: - logger.error( - f"Error {req.status_code} writing Style Filter Params GeoWebCache at {url}" - ) + logger.error(f"Error {req.status_code} writing Style Filter Params GeoWebCache at {url}") def _invalidate_geowebcache_dataset(dataset_name, url=None): @@ -1716,17 +1677,11 @@ def _invalidate_geowebcache_dataset(dataset_name, url=None): {dataset_name} """.strip() if not url: - url = f'{ogc_server_settings.LOCATION}gwc/rest/masstruncate' - req, content = http_client.post( - url, - data=body, - headers=headers, - user=_user) + url = f"{ogc_server_settings.LOCATION}gwc/rest/masstruncate" + req, content = http_client.post(url, data=body, headers=headers, user=_user) if req.status_code != 200: - logger.debug( - f"Error {req.status_code} invalidating GeoWebCache at {url}" - ) + logger.debug(f"Error {req.status_code} invalidating GeoWebCache at {url}") def style_update(request, url, workspace=None): @@ -1741,40 +1696,38 @@ def style_update(request, url, workspace=None): request.body, which is in this format: """ affected_datasets = [] - if request.method in ('POST', 'PUT', 'DELETE'): # we need to parse xml + if request.method in ("POST", "PUT", "DELETE"): # we need to parse xml # Need to remove NSx from IE11 if "HTTP_USER_AGENT" in request.META: - if ('Trident/7.0' in request.META['HTTP_USER_AGENT'] and - 'rv:11.0' in request.META['HTTP_USER_AGENT']): - txml = re.sub(r'xmlns:NS[0-9]=""', '', request.body) - txml = re.sub(r'NS[0-9]:', '', txml) + if "Trident/7.0" in request.META["HTTP_USER_AGENT"] and "rv:11.0" in request.META["HTTP_USER_AGENT"]: + txml = re.sub(r'xmlns:NS[0-9]=""', "", request.body) + txml = re.sub(r"NS[0-9]:", "", txml) request._body = txml style_name = os.path.basename(request.path) sld_title = style_name sld_body = None sld_url = url dataset_name = None - if 'name' in request.GET: - style_name = request.GET['name'] + if "name" in request.GET: + style_name = request.GET["name"] sld_body = request.body - elif request.method == 'DELETE': + elif request.method == "DELETE": style_name = os.path.basename(request.path) else: sld_body = request.body - gs_style = gs_catalog.get_style(name=style_name) or gs_catalog.get_style(name=style_name, workspace=workspace) + gs_style = gs_catalog.get_style(name=style_name) or gs_catalog.get_style( + name=style_name, workspace=workspace + ) if gs_style: - sld_title = gs_style.sld_title if gs_style.style_format != 'css' and gs_style.sld_title else style_name + sld_title = gs_style.sld_title if gs_style.style_format != "css" and gs_style.sld_title else style_name sld_body = gs_style.sld_body sld_url = gs_style.body_href else: try: tree = ET.ElementTree(dlxml.fromstring(request.body)) - elm_nameddataset_name = tree.findall( - './/{http://www.opengis.net/sld}Name')[0] - elm_user_style_name = tree.findall( - './/{http://www.opengis.net/sld}Name')[1] - elm_user_style_title = tree.find( - './/{http://www.opengis.net/sld}Title') + elm_nameddataset_name = tree.findall(".//{http://www.opengis.net/sld}Name")[0] + elm_user_style_name = tree.findall(".//{http://www.opengis.net/sld}Name")[1] + elm_user_style_title = tree.find(".//{http://www.opengis.net/sld}Title") dataset_name = elm_nameddataset_name.text if elm_user_style_title is None: sld_title = elm_user_style_name.text @@ -1785,10 +1738,10 @@ def style_update(request, url, workspace=None): logger.warn("Could not recognize Style and Dataset name from Request!") # add style in GN and associate it to layer - if request.method == 'DELETE': + if request.method == "DELETE": if style_name: Style.objects.filter(name=style_name).delete() - if request.method == 'POST': + if request.method == "POST": style = None if style_name and not re.match(temp_style_name_regex, style_name): style, created = Style.objects.get_or_create(name=style_name) @@ -1811,7 +1764,7 @@ def style_update(request, url, workspace=None): style.dataset_styles.add(layer) style.save() affected_datasets.append(layer) - elif request.method == 'PUT': # update style in GN + elif request.method == "PUT": # update style in GN if style_name and not re.match(temp_style_name_regex, style_name): style, created = Style.objects.get_or_create(name=style_name) style.workspace = workspace @@ -1832,9 +1785,8 @@ def style_update(request, url, workspace=None): return affected_datasets -def set_time_info(layer, attribute, end_attribute, presentation, - precision_value, precision_step, enabled=True): - '''Configure the time dimension for a layer. +def set_time_info(layer, attribute, end_attribute, presentation, precision_value, precision_step, enabled=True): + """Configure the time dimension for a layer. :param layer: the layer to configure :param attribute: the attribute used to represent the instant or period @@ -1847,10 +1799,10 @@ def set_time_info(layer, attribute, end_attribute, presentation, :param precision_step: one of 'seconds', 'minutes', 'hours', 'days', 'months', 'years' :param enabled: defaults to True - ''' + """ layer = gs_catalog.get_layer(layer.name) if layer is None: - raise ValueError(f'no such layer: {layer.name}') + raise ValueError(f"no such layer: {layer.name}") resource = layer.resource if layer else None if not resource: resources = gs_catalog.get_resources(stores=[layer.name]) @@ -1859,14 +1811,15 @@ def set_time_info(layer, attribute, end_attribute, presentation, resolution = None if precision_value and precision_step: - resolution = f'{precision_value} {precision_step}' - info = DimensionInfo("time", enabled, presentation, resolution, "ISO8601", - None, attribute=attribute, end_attribute=end_attribute) + resolution = f"{precision_value} {precision_step}" + info = DimensionInfo( + "time", enabled, presentation, resolution, "ISO8601", None, attribute=attribute, end_attribute=end_attribute + ) if resource and resource.metadata: metadata = dict(resource.metadata or {}) else: metadata = dict({}) - metadata['time'] = info + metadata["time"] = info if resource and resource.metadata: resource.metadata = metadata @@ -1875,22 +1828,22 @@ def set_time_info(layer, attribute, end_attribute, presentation, def get_time_info(layer): - '''Get the configured time dimension metadata for the layer as a dict. + """Get the configured time dimension metadata for the layer as a dict. The keys of the dict will be those of the parameters of `set_time_info`. :returns: dict of values or None if not configured - ''' + """ layer = gs_catalog.get_layer(layer.name) if layer is None: - raise ValueError(f'no such layer: {layer.name}') + raise ValueError(f"no such layer: {layer.name}") resource = layer.resource if layer else None if not resource: resources = gs_catalog.get_resources(stores=[layer.name]) if resources: resource = resources[0] - info = resource.metadata.get('time', None) if resource.metadata else None + info = resource.metadata.get("time", None) if resource.metadata else None vals = None if info: value = step = None @@ -1908,34 +1861,22 @@ def get_time_info(layer): return vals -ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default'] +ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] _wms = None _csw = None _user, _password = ogc_server_settings.credentials url = ogc_server_settings.rest -gs_catalog = Catalog(url, _user, _password, - retries=ogc_server_settings.MAX_RETRIES, - backoff_factor=ogc_server_settings.BACKOFF_FACTOR) +gs_catalog = Catalog( + url, _user, _password, retries=ogc_server_settings.MAX_RETRIES, backoff_factor=ogc_server_settings.BACKOFF_FACTOR +) gs_uploader = Client(url, _user, _password) gf_client = GeofenceClient(url, _user, _password) _punc = re.compile(r"[\.:]") # regex for punctuation that confuses restconfig -_foregrounds = [ - "#ffbbbb", - "#bbffbb", - "#bbbbff", - "#ffffbb", - "#bbffff", - "#ffbbff"] -_backgrounds = [ - "#880000", - "#008800", - "#000088", - "#888800", - "#008888", - "#880088"] +_foregrounds = ["#ffbbbb", "#bbffbb", "#bbbbff", "#ffffbb", "#bbffff", "#ffbbff"] +_backgrounds = ["#880000", "#008800", "#000088", "#888800", "#008888", "#880088"] _marks = ["square", "circle", "cross", "x", "triangle"] _style_contexts = zip(cycle(_foregrounds), cycle(_backgrounds), cycle(_marks)) _default_style_names = ["point", "line", "polygon", "raster"] @@ -1951,29 +1892,22 @@ def get_time_info(layer): "esriFieldTypeRaster": "raster", "esriFieldTypeGUID": "xsd:string", "esriFieldTypeGlobalID": "xsd:string", - "esriFieldTypeXML": "xsd:anyType"} + "esriFieldTypeXML": "xsd:anyType", +} def _dump_image_spec(request_body, image_spec): millis = int(round(time.time() * 1000)) try: with tempfile.TemporaryDirectory() as tmp_dir: - _request_body_file_name = os.path.join( - tmp_dir, - f"request_body_{millis}.dump") - _image_spec_file_name = os.path.join( - tmp_dir, - f"image_spec_{millis}.dump") + _request_body_file_name = os.path.join(tmp_dir, f"request_body_{millis}.dump") + _image_spec_file_name = os.path.join(tmp_dir, f"image_spec_{millis}.dump") with open(_request_body_file_name, "w") as _request_body_file: _request_body_file.write(f"{request_body}") - copyfile( - _request_body_file_name, - os.path.join(tempfile.gettempdir(), f"request_body_{millis}.dump")) + copyfile(_request_body_file_name, os.path.join(tempfile.gettempdir(), f"request_body_{millis}.dump")) with open(_image_spec_file_name, "w") as _image_spec_file: _image_spec_file.write(f"{image_spec}") - copyfile( - _image_spec_file_name, - os.path.join(tempfile.gettempdir(), f"image_spec_{millis}.dump")) + copyfile(_image_spec_file_name, os.path.join(tempfile.gettempdir(), f"image_spec_{millis}.dump")) return f"Dumping image_spec to: {os.path.join(tempfile.gettempdir(), f'image_spec_{millis}.dump')}" except Exception as e: logger.exception(e) @@ -1989,26 +1923,42 @@ def mosaic_delete_first_granule(cat, layer): granule_id = f"{layer}.1" - cat.mosaic_delete_granule(coverages['coverages']['coverage'][0]['name'], store, granule_id) + cat.mosaic_delete_granule(coverages["coverages"]["coverage"][0]["name"], store, granule_id) -def set_time_dimension(cat, name, workspace, time_presentation, time_presentation_res, time_presentation_default_value, - time_presentation_reference_value): +def set_time_dimension( + cat, + name, + workspace, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value, +): # configure the layer time dimension as LIST presentation = time_presentation if not presentation: presentation = "LIST" resolution = None - if time_presentation == 'DISCRETE_INTERVAL': + if time_presentation == "DISCRETE_INTERVAL": resolution = time_presentation_res strategy = None if time_presentation_default_value and not time_presentation_default_value == "": strategy = time_presentation_default_value - timeInfo = DimensionInfo("time", "true", presentation, resolution, "ISO8601", None, attribute="time", - strategy=strategy, reference_value=time_presentation_reference_value) + timeInfo = DimensionInfo( + "time", + "true", + presentation, + resolution, + "ISO8601", + None, + attribute="time", + strategy=strategy, + reference_value=time_presentation_reference_value, + ) layer = cat.get_layer(name) resource = layer.resource if layer else None @@ -2021,7 +1971,7 @@ def set_time_dimension(cat, name, workspace, time_presentation, time_presentatio logger.exception(f"No resource could be found on GeoServer with name {name}") raise Exception(f"No resource could be found on GeoServer with name {name}") - resource.metadata = {'time': timeInfo} + resource.metadata = {"time": timeInfo} cat.save(resource) @@ -2032,14 +1982,12 @@ def create_gs_thumbnail(instance, overwrite=False, check_bbox=False): return implementation(instance, overwrite, check_bbox) -def sync_instance_with_geoserver( - instance_id, - *args, **kwargs): +def sync_instance_with_geoserver(instance_id, *args, **kwargs): """ Synchronizes the Django Instance with GeoServer layers. """ - updatebbox = kwargs.get('updatebbox', True) - updatemetadata = kwargs.get('updatemetadata', True) + updatebbox = kwargs.get("updatebbox", True) + updatemetadata = kwargs.get("updatemetadata", True) instance = None try: @@ -2049,7 +1997,7 @@ def sync_instance_with_geoserver( raise if isinstance(instance, ResourceBase): - if hasattr(instance, 'dataset'): + if hasattr(instance, "dataset"): instance = instance.dataset else: return instance @@ -2066,7 +2014,7 @@ def sync_instance_with_geoserver( # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. - _is_remote_instance = hasattr(instance, 'subtype') and getattr(instance, 'subtype') in ['tileStore', 'remote'] + _is_remote_instance = hasattr(instance, "subtype") and getattr(instance, "subtype") in ["tileStore", "remote"] # Let's reset the connections first gs_catalog._cache.clear() @@ -2080,12 +2028,13 @@ def sync_instance_with_geoserver( # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method - if getattr(instance, 'overwrite', False): + if getattr(instance, "overwrite", False): base_file, info = instance.get_base_file() # There is no need to process it if there is no file. if base_file: from geonode.geoserver.upload import geoserver_upload + gs_name, workspace, values, gs_resource = geoserver_upload( instance, base_file.file.path, @@ -2094,7 +2043,7 @@ def sync_instance_with_geoserver( overwrite=True, title=instance.title, abstract=instance.abstract, - charset=instance.charset + charset=instance.charset, ) values, gs_resource = fetch_gs_resource(instance, values, _tries) @@ -2113,7 +2062,7 @@ def sync_instance_with_geoserver( instance.gs_resource = gs_resource # Iterate over values from geoserver. - for key in ['alternate', 'store', 'subtype']: + for key in ["alternate", "store", "subtype"]: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, get_dataset_storetype(values[key])) @@ -2125,20 +2074,23 @@ def sync_instance_with_geoserver( if instance.poc: # gsconfig now utilizes an attribution dictionary gs_resource.attribution = { - 'title': str(instance.poc), - 'width': None, - 'height': None, - 'href': None, - 'url': None, - 'type': None} + "title": str(instance.poc), + "width": None, + "height": None, + "href": None, + "url": None, + "type": None, + } profile = get_user_model().objects.get(username=instance.poc.username) - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL + site_url = ( + settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + ) gs_resource.attribution_link = site_url + profile.get_absolute_url() try: if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: - gs_resource.advertised = 'true' + gs_resource.advertised = "true" if any(instance.keyword_list()): keywords = gs_resource.keywords + instance.keyword_list() @@ -2149,7 +2101,7 @@ def sync_instance_with_geoserver( if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) except Exception as e: - msg = (f'Error while trying to save resource named {gs_resource} in GeoServer, try to use: "{e}"') + msg = f'Error while trying to save resource named {gs_resource} in GeoServer, try to use: "{e}"' e.args = (msg,) logger.warning(e) @@ -2173,12 +2125,14 @@ def sync_instance_with_geoserver( raise else: logger.exception(e) - instance.srid = 'EPSG:4326' + instance.srid = "EPSG:4326" Dataset.objects.filter(id=instance.id).update(srid=instance.srid) instance.set_ll_bbox_polygon([ll_bbox[0], ll_bbox[2], ll_bbox[1], ll_bbox[3]]) if instance.srid: - instance.srid_url = f"http://www.spatialreference.org/ref/{instance.srid.replace(':', '/').lower()}/" + instance.srid_url = ( + f"http://www.spatialreference.org/ref/{instance.srid.replace(':', '/').lower()}/" + ) else: raise GeoNodeException(_("Invalid Projection. Dataset is missing CRS!")) @@ -2186,26 +2140,24 @@ def sync_instance_with_geoserver( to_update = {} if updatemetadata: to_update = { - 'title': instance.title or instance.name, - 'abstract': instance.abstract or "", - 'alternate': instance.alternate + "title": instance.title or instance.name, + "abstract": instance.abstract or "", + "alternate": instance.alternate, } if updatebbox and is_monochromatic_image(instance.thumbnail_url): - to_update['thumbnail_url'] = None + to_update["thumbnail_url"] = None # Save all the modified information in the instance without triggering signals. with transaction.atomic(): - ResourceBase.objects.filter( - id=instance.resourcebase_ptr.id).update( - **to_update) + ResourceBase.objects.filter(id=instance.resourcebase_ptr.id).update(**to_update) # to_update['name'] = instance.name, - to_update['workspace'] = gs_resource.store.workspace.name - to_update['store'] = gs_resource.store.name - to_update['subtype'] = instance.subtype - to_update['typename'] = instance.alternate - to_update['srid'] = instance.srid + to_update["workspace"] = gs_resource.store.workspace.name + to_update["store"] = gs_resource.store.name + to_update["subtype"] = instance.subtype + to_update["typename"] = instance.alternate + to_update["srid"] = instance.srid Dataset.objects.filter(id=instance.id).update(**to_update) # Refresh from DB @@ -2249,7 +2201,7 @@ def write_uploaded_files_to_disk(target_dir, files): result = [] for django_file in files: path = os.path.join(target_dir, django_file.name) - with open(path, 'wb') as fh: + with open(path, "wb") as fh: for chunk in django_file.chunks(): fh.write(chunk) result = path @@ -2287,9 +2239,8 @@ class SpatialFilesLayerType: dataset_type: typing.Optional[str] = None -def get_spatial_files_dataset_type(allowed_extensions, files, charset='UTF-8') -> SpatialFilesLayerType: - """Reutnrs 'vector' or 'raster' whether a file from the allowed extensins has been identified. - """ +def get_spatial_files_dataset_type(allowed_extensions, files, charset="UTF-8") -> SpatialFilesLayerType: + """Reutnrs 'vector' or 'raster' whether a file from the allowed extensins has been identified.""" from geonode.upload.files import get_scan_hint, scan_file allowed_file = select_relevant_files(allowed_extensions, files) @@ -2297,11 +2248,7 @@ def get_spatial_files_dataset_type(allowed_extensions, files, charset='UTF-8') - return None base_file = allowed_file[0] scan_hint = get_scan_hint(allowed_extensions) - spatial_files = scan_file( - base_file, - scan_hint=scan_hint, - charset=charset - ) + spatial_files = scan_file(base_file, scan_hint=scan_hint, charset=charset) the_dataset_type = get_dataset_type(spatial_files) if the_dataset_type not in (FeatureType.resource_type, Coverage.resource_type): return None @@ -2309,14 +2256,14 @@ def get_spatial_files_dataset_type(allowed_extensions, files, charset='UTF-8') - base_file=base_file, scan_hint=scan_hint, spatial_files=spatial_files, - dataset_type='vector' if the_dataset_type == FeatureType.resource_type else 'raster') + dataset_type="vector" if the_dataset_type == FeatureType.resource_type else "raster", + ) return spatial_files_type def get_dataset_type(spatial_files): - """Returns 'FeatureType.resource_type' or 'Coverage.resource_type' accordingly to the provided SpatialFiles - """ + """Returns 'FeatureType.resource_type' or 'Coverage.resource_type' accordingly to the provided SpatialFiles""" if spatial_files.archive is not None: the_dataset_type = FeatureType.resource_type else: @@ -2330,6 +2277,7 @@ def wps_format_is_supported(_format, dataset_type): def ows_endpoint_in_path(path): return ( - re.match(r'.*(? QuerySet: return resource_type.objects.none() def exists(self, uuid: str, /, instance: ResourceBase = None) -> bool: if instance: _real_instance = instance.get_real_instance() - if hasattr(_real_instance, 'subtype') and _real_instance.subtype not in ['tileStore', 'remote']: + if hasattr(_real_instance, "subtype") and _real_instance.subtype not in ["tileStore", "remote"]: try: logger.debug(f"Searching GeoServer for layer '{_real_instance.alternate}'") # Let's reset the connections first @@ -107,18 +103,25 @@ def exists(self, uuid: str, /, instance: ResourceBase = None) -> bool: return False def delete(self, uuid: str, /, instance: ResourceBase = None) -> int: - """Removes the layer from GeoServer - """ + """Removes the layer from GeoServer""" # cascading_delete should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if instance and getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: _real_instance = instance.get_real_instance() - if isinstance(_real_instance, Dataset) and hasattr(_real_instance, 'alternate') and _real_instance.alternate: - if not hasattr(_real_instance, 'remote_service') or _real_instance.remote_service is None or _real_instance.remote_service.method == CASCADED: + if ( + isinstance(_real_instance, Dataset) + and hasattr(_real_instance, "alternate") + and _real_instance.alternate + ): + if ( + not hasattr(_real_instance, "remote_service") + or _real_instance.remote_service is None + or _real_instance.remote_service.method == CASCADED + ): geoserver_cascading_delete.apply_async((_real_instance.alternate,)) elif isinstance(_real_instance, Map): - geoserver_delete_map.apply_async((_real_instance.id, )) + geoserver_delete_map.apply_async((_real_instance.id,)) except Exception as e: logger.exception(e) @@ -129,33 +132,56 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de _resource = _synced_resource or _resource return _resource - def update(self, uuid: str, /, instance: ResourceBase = None, xml_file: str = None, metadata_uploaded: bool = False, - vals: dict = {}, regions: dict = {}, keywords: dict = {}, custom: dict = {}, notify: bool = True, **kwargs) -> ResourceBase: + def update( + self, + uuid: str, + /, + instance: ResourceBase = None, + xml_file: str = None, + metadata_uploaded: bool = False, + vals: dict = {}, + regions: dict = {}, + keywords: dict = {}, + custom: dict = {}, + notify: bool = True, + **kwargs, + ) -> ResourceBase: if instance: if isinstance(instance.get_real_instance(), Dataset): _synced_resource = sync_instance_with_geoserver(instance.id) instance = _synced_resource or instance return instance - def ingest(self, files: typing.List[str], /, uuid: str = None, resource_type: typing.Optional[object] = None, defaults: dict = {}, **kwargs) -> ResourceBase: + def ingest( + self, + files: typing.List[str], + /, + uuid: str = None, + resource_type: typing.Optional[object] = None, + defaults: dict = {}, + **kwargs, + ) -> ResourceBase: instance = ResourceManager._get_instance(uuid) if instance and isinstance(instance.get_real_instance(), Dataset): instance = self.import_dataset( - 'import_dataset', + "import_dataset", instance.uuid, instance=instance, files=files, - user=defaults.get('user', instance.owner), + user=defaults.get("user", instance.owner), defaults=defaults, - action_type='create', - **kwargs) + action_type="create", + **kwargs, + ) return instance - def copy(self, instance: ResourceBase, /, uuid: str = None, owner: settings.AUTH_USER_MODEL = None, defaults: dict = {}) -> ResourceBase: + def copy( + self, instance: ResourceBase, /, uuid: str = None, owner: settings.AUTH_USER_MODEL = None, defaults: dict = {} + ) -> ResourceBase: if uuid and instance: _resource = ResourceManager._get_instance(uuid) if _resource and isinstance(_resource.get_real_instance(), Dataset): - importer_session_opts = defaults.get('importer_session_opts', {}) + importer_session_opts = defaults.get("importer_session_opts", {}) if not importer_session_opts: _src_upload_session = Upload.objects.filter(resource=instance.get_real_instance().resourcebase_ptr) if _src_upload_session.exists(): @@ -163,45 +189,46 @@ def copy(self, instance: ResourceBase, /, uuid: str = None, owner: settings.AUTH if _src_upload_session and _src_upload_session.get_session: try: _src_importer_session = _src_upload_session.get_session.import_session.reload() - importer_session_opts.update({'transforms': _src_importer_session.tasks[0].transforms}) + importer_session_opts.update({"transforms": _src_importer_session.tasks[0].transforms}) except Exception as e: logger.exception(e) return self.import_dataset( - 'import_dataset', + "import_dataset", uuid, instance=_resource, - files=defaults.get('files', None), - user=defaults.get('user', _resource.owner), + files=defaults.get("files", None), + user=defaults.get("user", _resource.owner), defaults=defaults, - action_type='create', - importer_session_opts=importer_session_opts) + action_type="create", + importer_session_opts=importer_session_opts, + ) return _resource def append(self, instance: ResourceBase, vals: dict = {}, *args, **kwargs) -> ResourceBase: if instance and isinstance(instance.get_real_instance(), Dataset): return self.import_dataset( - 'import_dataset', + "import_dataset", instance.uuid, instance=instance, - files=vals.get('files', None), - user=vals.get('user', instance.owner), - action_type='append', - importer_session_opts=vals.get('importer_session_opts', None), - **kwargs + files=vals.get("files", None), + user=vals.get("user", instance.owner), + action_type="append", + importer_session_opts=vals.get("importer_session_opts", None), + **kwargs, ) return instance def replace(self, instance: ResourceBase, vals: dict = {}, *args, **kwargs) -> ResourceBase: if instance and isinstance(instance.get_real_instance(), Dataset): return self.import_dataset( - 'import_dataset', + "import_dataset", instance.uuid, instance=instance, - files=vals.get('files', None), - user=vals.get('user', instance.owner), - action_type='replace', - importer_session_opts=vals.get('importer_session_opts', None), - **kwargs + files=vals.get("files", None), + user=vals.get("user", instance.owner), + action_type="replace", + importer_session_opts=vals.get("importer_session_opts", None), + **kwargs, ) return instance @@ -212,33 +239,34 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non try: _gs_import_session_info = self._execute_resource_import( instance, - kwargs.get('files', None), - kwargs.get('user', instance.owner), - action_type=kwargs.get('action_type', 'create'), - importer_session_opts=kwargs.get('importer_session_opts', None)) + kwargs.get("files", None), + kwargs.get("user", instance.owner), + action_type=kwargs.get("action_type", "create"), + importer_session_opts=kwargs.get("importer_session_opts", None), + ) import_session = _gs_import_session_info.import_session if import_session: if import_session.state == enumerations.STATE_PENDING: task = None native_crs = None - target_crs = 'EPSG:4326' + target_crs = "EPSG:4326" for _task in import_session.tasks: # CRS missing/unknown - if _task.state == 'NO_CRS': + if _task.state == "NO_CRS": task = _task native_crs = _task.layer.srs break if not native_crs: - native_crs = 'EPSG:4326' + native_crs = "EPSG:4326" if task: task.set_srs(native_crs) transform = { - 'type': 'ReprojectTransform', - 'source': native_crs, - 'target': target_crs, + "type": "ReprojectTransform", + "source": native_crs, + "target": target_crs, } - task.remove_transforms([transform], by_field='type', save=False) + task.remove_transforms([transform], by_field="type", save=False) task.add_transforms([transform], save=False) task.save_transforms() # Starting import process @@ -246,28 +274,36 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non import_session = import_session.reload() _gs_import_session_info.import_session = import_session _gs_import_session_info.dataset_name = import_session.tasks[0].layer.name - _name = _gs_import_session_info.dataset_name if import_session.state == enumerations.STATE_COMPLETE else '' - _alternate = f'{_gs_import_session_info.workspace}:{_gs_import_session_info.dataset_name}' if import_session.state == enumerations.STATE_COMPLETE else '' + _name = ( + _gs_import_session_info.dataset_name + if import_session.state == enumerations.STATE_COMPLETE + else "" + ) + _alternate = ( + f"{_gs_import_session_info.workspace}:{_gs_import_session_info.dataset_name}" + if import_session.state == enumerations.STATE_COMPLETE + else "" + ) _to_update = { - 'name': _name, - 'title': instance.title or _gs_import_session_info.dataset_name, - 'files': kwargs.get('files', None), - 'workspace': _gs_import_session_info.workspace, - 'alternate': _alternate, - 'typename': _alternate, - 'store': _gs_import_session_info.target_store or _gs_import_session_info.dataset_name, - 'subtype': _gs_import_session_info.spatial_files_type.dataset_type + "name": _name, + "title": instance.title or _gs_import_session_info.dataset_name, + "files": kwargs.get("files", None), + "workspace": _gs_import_session_info.workspace, + "alternate": _alternate, + "typename": _alternate, + "store": _gs_import_session_info.target_store or _gs_import_session_info.dataset_name, + "subtype": _gs_import_session_info.spatial_files_type.dataset_type, } - if 'defaults' in kwargs: - kwargs['defaults'].update(_to_update) + if "defaults" in kwargs: + kwargs["defaults"].update(_to_update) Dataset.objects.filter(uuid=instance.uuid).update(**_to_update) instance.get_real_instance_class().objects.filter(uuid=instance.uuid).update(**_to_update) # Refresh from DB instance.refresh_from_db() - if kwargs.get('action_type', 'create') == 'create': + if kwargs.get("action_type", "create") == "create": set_styles(instance.get_real_instance(), gs_catalog) set_attributes_from_geoserver(instance.get_real_instance(), overwrite=True) - elif kwargs.get('action_type', 'create') == 'create': + elif kwargs.get("action_type", "create") == "create": logger.exception(Exception(f"Importer Session not valid - STATE: {import_session.state}")) if import_session.state == enumerations.STATE_COMPLETE: instance.set_processing_state(enumerations.STATE_PROCESSED) @@ -277,12 +313,14 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non instance.save(notify=False) except Exception as e: logger.exception(e) - if kwargs.get('action_type', 'create') == 'create': + if kwargs.get("action_type", "create") == "create": instance.delete() instance = None return instance - def _execute_resource_import(self, instance, files: list, user, action_type: str, importer_session_opts: typing.Optional[typing.Dict] = None): + def _execute_resource_import( + self, instance, files: list, user, action_type: str, importer_session_opts: typing.Optional[typing.Dict] = None + ): from geonode.utils import get_allowed_extensions ALLOWED_EXTENSIONS = get_allowed_extensions() @@ -294,13 +332,17 @@ def _execute_resource_import(self, instance, files: list, user, action_type: str if not spatial_files_type: raise Exception(f"No suitable Spatial Files avaialable for 'ALLOWED_EXTENSIONS' = {ALLOWED_EXTENSIONS}.") - upload_session, _ = Upload.objects.get_or_create(resource=instance.get_real_instance().resourcebase_ptr, user=user) + upload_session, _ = Upload.objects.get_or_create( + resource=instance.get_real_instance().resourcebase_ptr, user=user + ) upload_session.resource = instance.get_real_instance().resourcebase_ptr upload_session.save() _name = instance.get_real_instance().name if not _name: - _name = session_opts.get('name', None) or os.path.splitext(os.path.basename(spatial_files_type.base_file))[0] + _name = ( + session_opts.get("name", None) or os.path.splitext(os.path.basename(spatial_files_type.base_file))[0] + ) instance.get_real_instance().name = _name gs_dataset = None @@ -312,31 +354,27 @@ def _execute_resource_import(self, instance, files: list, user, action_type: str _workspace = None _target_store = None if gs_dataset: - _target_store = gs_dataset.resource.store.name if instance.get_real_instance().subtype == 'vector' else None + _target_store = gs_dataset.resource.store.name if instance.get_real_instance().subtype == "vector" else None _workspace = gs_dataset.resource.workspace.name if gs_dataset.resource.workspace else None if not _workspace: - _workspace = session_opts.get('workspace', instance.get_real_instance().workspace) + _workspace = session_opts.get("workspace", instance.get_real_instance().workspace) if not _workspace: _workspace = instance.get_real_instance().workspace or settings.DEFAULT_WORKSPACE if not _target_store: - if instance.get_real_instance().subtype == 'vector' or spatial_files_type.dataset_type == 'vector': - _dsname = ogc_server_settings.datastore_db['NAME'] + if instance.get_real_instance().subtype == "vector" or spatial_files_type.dataset_type == "vector": + _dsname = ogc_server_settings.datastore_db["NAME"] _ds = create_geoserver_db_featurestore(store_name=_dsname, workspace=_workspace) if _ds: - _target_store = session_opts.get('target_store', None) or _dsname + _target_store = session_opts.get("target_store", None) or _dsname # opening Import session for the selected layer # Let's reset the connections first gs_catalog._cache.clear() gs_catalog.reset() # Let's now try the new ingestion - import_session = gs_uploader.start_import( - import_id=upload_session.id, - name=_name, - target_store=_target_store - ) + import_session = gs_uploader.start_import(import_id=upload_session.id, name=_name, target_store=_target_store) upload_session.set_processing_state(enumerations.STATE_PROCESSED) upload_session.import_id = import_session.id @@ -351,7 +389,7 @@ def _execute_resource_import(self, instance, files: list, user, action_type: str spatial_files_type=spatial_files_type, dataset_name=None, workspace=_workspace, - target_store=_target_store + target_store=_target_store, ) import_session.upload_task(files) @@ -359,11 +397,8 @@ def _execute_resource_import(self, instance, files: list, user, action_type: str # Changing layer name, mode and target task.layer.set_target_layer_name(_name) task.set_update_mode(action_type.upper()) - task.set_target( - store_name=_target_store, - workspace=_workspace - ) - transforms = session_opts.get('transforms', None) + task.set_target(store_name=_target_store, workspace=_workspace) + transforms = session_opts.get("transforms", None) if transforms: task.set_transforms(transforms) # Starting import process @@ -380,8 +415,8 @@ def remove_permissions(self, uuid: str, /, instance: ResourceBase = None) -> boo try: if instance and isinstance(instance.get_real_instance(), Dataset): - if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): purge_geofence_dataset_rules(instance.get_real_instance()) set_geofence_invalidate_cache() else: @@ -391,17 +426,28 @@ def remove_permissions(self, uuid: str, /, instance: ResourceBase = None) -> boo return False return True - def set_permissions(self, uuid: str, /, instance: ResourceBase = None, owner: settings.AUTH_USER_MODEL = None, permissions: dict = {}, created: bool = False, - approval_status_changed: bool = False, group_status_changed: bool = False) -> bool: + def set_permissions( + self, + uuid: str, + /, + instance: ResourceBase = None, + owner: settings.AUTH_USER_MODEL = None, + permissions: dict = {}, + created: bool = False, + approval_status_changed: bool = False, + group_status_changed: bool = False, + ) -> bool: _resource = instance or ResourceManager._get_instance(uuid) try: if _resource: _resource = _resource.get_real_instance() - logger.error(f'Fixup GIS Backend Security Rules Accordingly on resource {instance} {isinstance(_resource, Dataset)}') + logger.error( + f"Fixup GIS Backend Security Rules Accordingly on resource {instance} {isinstance(_resource, Dataset)}" + ) if isinstance(_resource, Dataset): - if settings.OGC_SERVER['default'].get("GEOFENCE_SECURITY_ENABLED", False): - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if settings.OGC_SERVER["default"].get("GEOFENCE_SECURITY_ENABLED", False): + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): _disable_cache = [] _owner = owner or _resource.owner if permissions is not None and len(permissions): @@ -409,31 +455,41 @@ def set_permissions(self, uuid: str, /, instance: ResourceBase = None, owner: se purge_geofence_dataset_rules(_resource) # Owner - perms = OWNER_PERMISSIONS.copy() + DATASET_ADMIN_PERMISSIONS.copy() + DOWNLOAD_PERMISSIONS.copy() - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _owner, None, None) + perms = ( + OWNER_PERMISSIONS.copy() + + DATASET_ADMIN_PERMISSIONS.copy() + + DOWNLOAD_PERMISSIONS.copy() + ) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, _owner, None, None + ) # All the other users - if 'users' in permissions and len(permissions['users']) > 0: - for user, perms in permissions['users'].items(): + if "users" in permissions and len(permissions["users"]) > 0: + for user, perms in permissions["users"].items(): _user = get_user_model().objects.get(username=user) if _user != _owner: # Set the GeoFence Rules group_perms = None - if 'groups' in permissions and len(permissions['groups']) > 0: - group_perms = permissions['groups'] + if "groups" in permissions and len(permissions["groups"]) > 0: + group_perms = permissions["groups"] if user == "AnonymousUser": _user = None _group = list(group_perms.keys())[0] if group_perms else None - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _user, _group, group_perms) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, _user, _group, group_perms + ) # All the other groups - if 'groups' in permissions and len(permissions['groups']) > 0: - for group, perms in permissions['groups'].items(): + if "groups" in permissions and len(permissions["groups"]) > 0: + for group, perms in permissions["groups"].items(): _group = Group.objects.get(name=group) # Set the GeoFence Rules - if _group and _group.name and _group.name == 'anonymous': + if _group and _group.name and _group.name == "anonymous": _group = None - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, None, _group, None) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, None, _group, None + ) else: anonymous_can_view = settings.DEFAULT_ANONYMOUS_VIEW_PERMISSION anonymous_can_download = settings.DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION @@ -442,29 +498,45 @@ def set_permissions(self, uuid: str, /, instance: ResourceBase = None, owner: se purge_geofence_dataset_rules(_resource.get_self_resource()) # Owner & Managers - perms = OWNER_PERMISSIONS.copy() + DATASET_ADMIN_PERMISSIONS.copy() + DOWNLOAD_PERMISSIONS.copy() - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _owner, None, None) + perms = ( + OWNER_PERMISSIONS.copy() + + DATASET_ADMIN_PERMISSIONS.copy() + + DOWNLOAD_PERMISSIONS.copy() + ) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, _owner, None, None + ) _resource_groups, _group_managers = _resource.get_group_managers(group=_resource.group) for _group_manager in _group_managers: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, _group_manager, None, None) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, _group_manager, None, None + ) for user_group in _resource_groups: if not skip_registered_members_common_group(user_group): - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, perms, None, user_group, None) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, perms, None, user_group, None + ) # Anonymous if anonymous_can_view: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, VIEW_PERMISSIONS, None, None, None) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, VIEW_PERMISSIONS, None, None, None + ) if anonymous_can_download: - _disable_cache = sync_permissions_and_disable_cache(_disable_cache, _resource, DOWNLOAD_PERMISSIONS, None, None, None) + _disable_cache = sync_permissions_and_disable_cache( + _disable_cache, _resource, DOWNLOAD_PERMISSIONS, None, None, None + ) if _disable_cache: filters, formats = _get_gwc_filters_and_formats(_disable_cache) try: _dataset_workspace = get_dataset_workspace(_resource) - toggle_dataset_cache(f'{_dataset_workspace}:{_resource.name}', filters=filters, formats=formats) + toggle_dataset_cache( + f"{_dataset_workspace}:{_resource.name}", filters=filters, formats=formats + ) except Dataset.DoesNotExist: pass else: @@ -473,12 +545,16 @@ def set_permissions(self, uuid: str, /, instance: ResourceBase = None, owner: se logger.exception(e) return False - post_set_permissions.send_robust(sender=instance, instance=instance) + geofence_rule_assign.send_robust(sender=instance, instance=instance) return True - def set_thumbnail(self, uuid: str, /, instance: ResourceBase = None, overwrite: bool = True, check_bbox: bool = True) -> bool: - if instance and (isinstance(instance.get_real_instance(), Dataset) or isinstance(instance.get_real_instance(), Map)): + def set_thumbnail( + self, uuid: str, /, instance: ResourceBase = None, overwrite: bool = True, check_bbox: bool = True + ) -> bool: + if instance and ( + isinstance(instance.get_real_instance(), Dataset) or isinstance(instance.get_real_instance(), Map) + ): if overwrite or not instance.thumbnail_url: create_gs_thumbnail(instance.get_real_instance(), overwrite=overwrite, check_bbox=check_bbox) return True @@ -492,13 +568,15 @@ def set_style(self, method: str, uuid: str, instance: ResourceBase = None, **kwa if instance and isinstance(instance.get_real_instance(), Dataset): try: - logger.info(f'Creating style for Dataset {instance.get_real_instance()} / {kwargs}') - _sld_file = kwargs.get('sld_file', None) - _tempdir = kwargs.get('tempdir', tempfile.gettempdir()) - if _sld_file and kwargs.get('sld_uploaded', False): + logger.info(f"Creating style for Dataset {instance.get_real_instance()} / {kwargs}") + _sld_file = kwargs.get("sld_file", None) + _tempdir = kwargs.get("tempdir", tempfile.gettempdir()) + if _sld_file and kwargs.get("sld_uploaded", False): geoserver_set_style(instance.get_real_instance().id, _sld_file) else: - geoserver_create_style(instance.get_real_instance().id, instance.get_real_instance().name, _sld_file, _tempdir) + geoserver_create_style( + instance.get_real_instance().id, instance.get_real_instance().name, _sld_file, _tempdir + ) except Exception as e: logger.exception(e) return None @@ -509,8 +587,8 @@ def set_time_info(self, method: str, uuid: str, /, instance: ResourceBase = None if instance and isinstance(instance.get_real_instance(), Dataset): try: - if kwargs.get('time_info', None): - set_time_info(instance.get_real_instance(), **kwargs['time_info']) + if kwargs.get("time_info", None): + set_time_info(instance.get_real_instance(), **kwargs["time_info"]) except Exception as e: logger.exception(e) return None diff --git a/geonode/geoserver/ows.py b/geonode/geoserver/ows.py index 17490d7a41f..cf4735ab259 100644 --- a/geonode/geoserver/ows.py +++ b/geonode/geoserver/ows.py @@ -27,16 +27,13 @@ from lxml import etree from urllib.parse import urlencode, urljoin -from geonode.utils import ( - XML_PARSER, - http_client, - OGC_Servers_Handler) +from geonode.utils import XML_PARSER, http_client, OGC_Servers_Handler logger = logging.getLogger(__name__) -ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default'] +ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] -DEFAULT_EXCLUDE_FORMATS = ['PNG', 'JPEG', 'GIF', 'TIFF'] +DEFAULT_EXCLUDE_FORMATS = ["PNG", "JPEG", "GIF", "TIFF"] def _get_nsmap(original: typing.Dict, key: str): @@ -59,30 +56,29 @@ def _get_nsmap(original: typing.Dict, key: str): def _wcs_get_capabilities(): try: - wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') - wcs_url += '&' if '?' in wcs_url else '?' + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wcs_url += "&" if "?" in wcs_url else "?" - return wcs_url + urlencode({ - 'service': 'WCS', - 'request': 'GetCapabilities', - 'version': '2.0.1', - }) + return wcs_url + urlencode( + { + "service": "WCS", + "request": "GetCapabilities", + "version": "2.0.1", + } + ) def _wcs_describe_coverage(coverage_id): try: - wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') - wcs_url += '&' if '?' in wcs_url else '?' - return wcs_url + urlencode({ - 'service': 'WCS', - 'request': 'describecoverage', - 'version': '2.0.1', - 'coverageid': coverage_id - }) + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wcs_url += "&" if "?" in wcs_url else "?" + return wcs_url + urlencode( + {"service": "WCS", "request": "describecoverage", "version": "2.0.1", "coverageid": coverage_id} + ) def _get_wcs_axis_labels(coverage_id): @@ -177,7 +173,7 @@ def _get_wcs_axis_labels(coverage_id): def _swap(_axis_labels): # WCS 2.0.1 swaps the order of the Lon/Lat axis to Lat/Lon. - if _axis_labels[0].lower() in 'lat': + if _axis_labels[0].lower() in "lat": return [_axis_labels[1], _axis_labels[0]] else: return _axis_labels @@ -185,8 +181,8 @@ def _swap(_axis_labels): try: _describe_coverage_response, _content = http_client.get(_wcs_describe_coverage(coverage_id)) _describe_coverage_response.raise_for_status() - _root = etree.fromstring(_content.encode('UTF-8'), parser=XML_PARSER) - _nsmap = _get_nsmap(_root.nsmap, 'wcs') + _root = etree.fromstring(_content.encode("UTF-8"), parser=XML_PARSER) + _nsmap = _get_nsmap(_root.nsmap, "wcs") _coverage_descriptions = _root.xpath("//wcs:CoverageDescription", namespaces=_nsmap) for _coverage_description in _coverage_descriptions: _envelope = _coverage_description.xpath("gml:boundedBy/gml:Envelope", namespaces=_nsmap) @@ -216,24 +212,24 @@ def _wcs_link(wcs_url, identifier, mime, srid=None, bbox=None, compression=None, subset=Long(-84.0,-78.0)& subset=Lat(11.0,15.0) """ - coverage_id = identifier.replace(':', '__', 1) + coverage_id = identifier.replace(":", "__", 1) wcs_params = { - 'service': 'WCS', - 'request': 'GetCoverage', - 'coverageid': coverage_id, - 'format': mime, - 'version': '2.0.1', + "service": "WCS", + "request": "GetCoverage", + "coverageid": coverage_id, + "format": mime, + "version": "2.0.1", } if compression: - wcs_params['compression'] = compression + wcs_params["compression"] = compression if tile_size: - wcs_params['tileWidth'] = tile_size - wcs_params['tileHeight'] = tile_size + wcs_params["tileWidth"] = tile_size + wcs_params["tileHeight"] = tile_size if srid: - wcs_params['outputCrs'] = srid + wcs_params["outputCrs"] = srid _wcs_params = urlencode(wcs_params) @@ -242,13 +238,19 @@ def _wcs_link(wcs_url, identifier, mime, srid=None, bbox=None, compression=None, if isinstance(bbox, list): _bbox = bbox elif isinstance(bbox, str): - _bbox = ast.literal_eval(f'[{bbox}]') if all([_x in bbox for _x in ['[', ']']]) else ast.literal_eval(f'[{bbox}]') + _bbox = ( + ast.literal_eval(f"[{bbox}]") + if all([_x in bbox for _x in ["[", "]"]]) + else ast.literal_eval(f"[{bbox}]") + ) if _bbox: _axis_labels = _get_wcs_axis_labels(coverage_id) if _axis_labels: - _wcs_params += f'&subset={_axis_labels[0]}({_bbox[0]},{_bbox[2]})&subset={_axis_labels[1]}({_bbox[1]},{_bbox[3]})' - _query_separator = '?' if not wcs_url.endswith('?') else '' - return f'{wcs_url}{_query_separator}{_wcs_params}' + _wcs_params += ( + f"&subset={_axis_labels[0]}({_bbox[0]},{_bbox[2]})&subset={_axis_labels[1]}({_bbox[1]},{_bbox[3]})" + ) + _query_separator = "?" if not wcs_url.endswith("?") else "" + return f"{wcs_url}{_query_separator}{_wcs_params}" def wcs_links(wcs_url, identifier, bbox=None, srid=None): @@ -274,43 +276,45 @@ def wcs_links(wcs_url, identifier, bbox=None, srid=None): def _wfs_get_capabilities(): try: - wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wfs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') - wfs_url += '&' if '?' in wfs_url else '?' + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wfs_url += "&" if "?" in wfs_url else "?" - return wfs_url + urlencode({ - 'service': 'WFS', - 'request': 'GetCapabilities', - 'version': '1.1.0', - }) + return wfs_url + urlencode( + { + "service": "WFS", + "request": "GetCapabilities", + "version": "1.1.0", + } + ) def _wfs_link(wfs_url, identifier, mime, extra_params, bbox=None, srid=None): params = { - 'service': 'WFS', - 'version': '1.0.0', - 'request': 'GetFeature', - 'typename': identifier, - 'outputFormat': mime, + "service": "WFS", + "version": "1.0.0", + "request": "GetFeature", + "typename": identifier, + "outputFormat": mime, } if srid: - params['srs'] = srid + params["srs"] = srid if bbox: - params['bbox'] = bbox + params["bbox"] = bbox params.update(extra_params) - _query_separator = '?' if not wfs_url.endswith('?') else '' - return f'{wfs_url}{_query_separator}{urlencode(params)}' + _query_separator = "?" if not wfs_url.endswith("?") else "" + return f"{wfs_url}{_query_separator}{urlencode(params)}" def wfs_links(wfs_url, identifier, bbox=None, srid=None): types = [ - ("zip", _("Zipped Shapefile"), "SHAPE-ZIP", {'format_options': 'charset:UTF-8'}), + ("zip", _("Zipped Shapefile"), "SHAPE-ZIP", {"format_options": "charset:UTF-8"}), ("gml", _("GML 2.0"), "gml2", {}), ("gml", _("GML 3.1.1"), "text/xml; subtype=gml/3.1.1", {}), ("csv", _("CSV"), "csv", {}), ("excel", _("Excel"), "excel", {}), - ("json", _("GeoJSON"), "json", {'srsName': srid or 'EPSG:4326'}) + ("json", _("GeoJSON"), "json", {"srsName": srid or "EPSG:4326"}), ] output = [] for ext, name, mime, extra_params in types: @@ -321,33 +325,35 @@ def wfs_links(wfs_url, identifier, bbox=None, srid=None): def _wms_get_capabilities(): try: - wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wms_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') - wms_url += '&' if '?' in wms_url else '?' + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wms_url += "&" if "?" in wms_url else "?" - return wms_url + urlencode({ - 'service': 'WMS', - 'request': 'GetCapabilities', - 'version': '1.3.0', - }) + return wms_url + urlencode( + { + "service": "WMS", + "request": "GetCapabilities", + "version": "1.3.0", + } + ) def _wms_link(wms_url, identifier, mime, height, width, srid=None, bbox=None): wms_params = { - 'service': 'WMS', - 'request': 'GetMap', - 'layers': identifier, - 'format': mime, - 'height': height, - 'width': width, + "service": "WMS", + "request": "GetMap", + "layers": identifier, + "format": mime, + "height": height, + "width": width, } if srid: - wms_params['srs'] = srid + wms_params["srs"] = srid if bbox: - wms_params['bbox'] = bbox - _query_separator = '?' if not wms_url.endswith('?') else '' - return f'{wms_url}{_query_separator}{urlencode(wms_params)}' + wms_params["bbox"] = bbox + _query_separator = "?" if not wms_url.endswith("?") else "" + return f"{wms_url}{_query_separator}{urlencode(wms_params)}" def wms_links(wms_url, identifier, bbox, srid, height, width): diff --git a/geonode/geoserver/processing/models.py b/geonode/geoserver/processing/models.py index 4935241e7f2..8540349316e 100644 --- a/geonode/geoserver/processing/models.py +++ b/geonode/geoserver/processing/models.py @@ -23,6 +23,5 @@ # Samples # ##################################################################################### # class SampleGeoServerProcessingTask(AbstractProcessingTask, metaclass=AbstractProcessingTaskMeta): - def execute(self, resource): print(f"Executing {self.name} against {resource}") diff --git a/geonode/geoserver/security.py b/geonode/geoserver/security.py index 18da209bfa8..8a032411846 100644 --- a/geonode/geoserver/security.py +++ b/geonode/geoserver/security.py @@ -45,8 +45,8 @@ def get_highest_priority(): try: rules_count = gf_client.get_rules_count() rules_objs = gf_client.get_rules(page=rules_count - 1, entries=1) - if len(rules_objs['rules']) > 0: - highest_priority = rules_objs['rules'][0]['priority'] + if len(rules_objs["rules"]) > 0: + highest_priority = rules_objs["rules"][0]["priority"] else: highest_priority = 0 return int(highest_priority) @@ -58,7 +58,7 @@ def get_highest_priority(): def purge_geofence_all(): """purge all existing GeoFence Cache Rules""" - if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: gf_client.purge_all_rules() @@ -70,8 +70,9 @@ def purge_geofence_dataset_rules(resource): http://:/geoserver/rest/geofence/rules.json?workspace=geonode&layer={layer} """ workspace = get_dataset_workspace(resource.dataset) - dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ - else resource.dataset.alternate + dataset_name = ( + resource.dataset.name if resource.dataset and hasattr(resource.dataset, "name") else resource.dataset.alternate + ) try: gf_client.purge_layer_rules(dataset_name, workspace=workspace) except Exception as e: @@ -82,7 +83,7 @@ def purge_geofence_dataset_rules(resource): def set_geofence_invalidate_cache(): """invalidate GeoFence Cache Rules""" - if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: try: gf_client.invalidate_cache() return True @@ -94,48 +95,47 @@ def set_geofence_invalidate_cache(): def toggle_dataset_cache(dataset_name, enable=True, filters=None, formats=None): """Disable/enable a GeoServer Tiled Dataset Configuration""" - if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] """ curl -v -u admin:geoserver -XGET \ "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" """ - r = requests.get(f'{url}gwc/rest/layers/{dataset_name}.xml', - auth=HTTPBasicAuth(user, passwd)) + r = requests.get(f"{url}gwc/rest/layers/{dataset_name}.xml", auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): + if r.status_code < 200 or r.status_code > 201: logger.debug(f"Could not Retrieve {dataset_name} Cache.") return False try: xml_content = r.content tree = dlxml.fromstring(xml_content) - gwc_id = tree.find('id') + gwc_id = tree.find("id") tree.remove(gwc_id) - gwc_enabled = tree.find('enabled') + gwc_enabled = tree.find("enabled") if gwc_enabled is None: - gwc_enabled = etree.Element('enabled') + gwc_enabled = etree.Element("enabled") tree.append(gwc_enabled) gwc_enabled.text = str(enable).lower() - gwc_mimeFormats = tree.find('mimeFormats') + gwc_mimeFormats = tree.find("mimeFormats") # Returns an element instance or None if gwc_mimeFormats is not None and len(gwc_mimeFormats): tree.remove(gwc_mimeFormats) if formats is not None: for format in formats: - gwc_format = etree.Element('string') + gwc_format = etree.Element("string") gwc_format.text = format gwc_mimeFormats.append(gwc_format) tree.append(gwc_mimeFormats) - gwc_parameterFilters = tree.find('parameterFilters') + gwc_parameterFilters = tree.find("parameterFilters") if filters is None: tree.remove(gwc_parameterFilters) else: @@ -151,9 +151,9 @@ def toggle_dataset_cache(dataset_name, enable=True, filters=None, formats=None): """ gwc_parameter = etree.Element(k) for parameter_key, parameter_value in v.items(): - gwc_parameter_key = etree.Element('key') + gwc_parameter_key = etree.Element("key") gwc_parameter_key.text = parameter_key - gwc_parameter_value = etree.Element('defaultValue') + gwc_parameter_value = etree.Element("defaultValue") gwc_parameter_value.text = parameter_value gwc_parameter.append(gwc_parameter_key) @@ -165,13 +165,15 @@ def toggle_dataset_cache(dataset_name, enable=True, filters=None, formats=None): -H "Content-type: text/xml" -d @poi.xml \ "http://localhost:8080/geoserver/gwc/rest/layers/tiger:poi.xml" """ - headers = {'Content-type': 'text/xml'} + headers = {"Content-type": "text/xml"} payload = ET.tostring(tree) - r = requests.post(f'{url}gwc/rest/layers/{dataset_name}.xml', - headers=headers, - data=payload, - auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): + r = requests.post( + f"{url}gwc/rest/layers/{dataset_name}.xml", + headers=headers, + data=payload, + auth=HTTPBasicAuth(user, passwd), + ) + if r.status_code < 200 or r.status_code > 201: logger.debug(f"Could not Update {dataset_name} Cache.") return False except Exception: @@ -187,19 +189,18 @@ def toggle_dataset_cache(dataset_name, enable=True, filters=None, formats=None): def delete_dataset_cache(dataset_name): """Delete a GeoServer Tiled Dataset Configuration and all the cache""" - if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] """ curl -v -u admin:geoserver -XDELETE \ "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" """ - r = requests.delete(f'{url}gwc/rest/layers/{dataset_name}.xml', - auth=HTTPBasicAuth(user, passwd)) + r = requests.delete(f"{url}gwc/rest/layers/{dataset_name}.xml", auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): + if r.status_code < 200 or r.status_code > 201: logger.debug(f"Could not Delete {dataset_name} Cache.") return False return True @@ -214,23 +215,24 @@ def set_geowebcache_invalidate_cache(dataset_alternate, cat=None): if dataset_alternate is not None and len(dataset_alternate) and "None" not in dataset_alternate: try: if cat is None or cat.get_layer(dataset_alternate) is not None: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] """ curl -v -u admin:geoserver \ -H "Content-type: text/xml" \ -d "{dataset_alternate}" \ http://localhost:8080/geoserver/gwc/rest/masstruncate """ - headers = {'Content-type': 'text/xml'} + headers = {"Content-type": "text/xml"} payload = f"{dataset_alternate}" r = requests.post( f"{url.rstrip('/')}/gwc/rest/masstruncate", headers=headers, data=payload, - auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): + auth=HTTPBasicAuth(user, passwd), + ) + if r.status_code < 200 or r.status_code > 201: logger.debug(f"Could not Truncate GWC Cache for Dataset '{dataset_alternate}'.") except Exception: tb = traceback.format_exc() @@ -251,10 +253,11 @@ def set_geofence_all(instance): resource = instance.get_self_resource() logger.debug(f"Inside set_geofence_all for instance {instance}") workspace = get_dataset_workspace(resource.dataset) - dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ - else resource.dataset.alternate + dataset_name = ( + resource.dataset.name if resource.dataset and hasattr(resource.dataset, "name") else resource.dataset.alternate + ) logger.debug(f"going to work in workspace {workspace}") - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): try: priority = get_highest_priority() + 1 gf_client.insert_rule(Rule(priority, workspace, dataset_name, Rule.ALLOW)) @@ -272,39 +275,37 @@ def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_per """ Sync Guardian permissions to GeoFence. """ - layer_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate + layer_name = dataset.name if dataset and hasattr(dataset, "name") else dataset.alternate workspace_name = get_dataset_workspace(dataset) # Create new rule-set gf_services = _get_gf_services(dataset, perms) gf_requests = {} - if 'change_dataset_data' not in perms: + if "change_dataset_data" not in perms: _skip_perm = False if user and group_perms: if isinstance(user, str): user = get_user_model().objects.get(username=user) - user_groups = list(user.groups.values_list('name', flat=True)) + user_groups = list(user.groups.values_list("name", flat=True)) for _group, _perm in group_perms.items(): - if 'change_dataset_data' in _perm and _group in user_groups: + if "change_dataset_data" in _perm and _group in user_groups: _skip_perm = True break if not _skip_perm: - gf_requests["WFS"] = { - "TRANSACTION": False, - "LOCKFEATURE": False, - "GETFEATUREWITHLOCK": False - } + gf_requests["WFS"] = {"TRANSACTION": False, "LOCKFEATURE": False, "GETFEATUREWITHLOCK": False} _user = None _group = None - _group, _user, _disable_cache, users_geolimits, groups_geolimits, anonymous_geolimits = get_user_geolimits(dataset, user, group) + _group, _user, _disable_cache, users_geolimits, groups_geolimits, anonymous_geolimits = get_user_geolimits( + dataset, user, group + ) if _disable_cache: - gf_services_limits_first = {"*": gf_services.pop('*')} + gf_services_limits_first = {"*": gf_services.pop("*")} gf_services_limits_first.update(gf_services) gf_services = gf_services_limits_first - batch = Batch(f'Sync {workspace_name}:{layer_name}') + batch = Batch(f"Sync {workspace_name}:{layer_name}") priority = get_highest_priority() + 1 pri = itertools.count(priority) @@ -315,20 +316,37 @@ def resolve_geolimits(geolimits): wkt = resolve_geolimits(users_geolimits) if wkt: logger.debug(f"Adding GeoFence USER GeoLimit rule: U:{_user} L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - user=_user, - geo_limit=wkt)) + batch.add_insert_rule( + Rule( + pri.__next__(), + workspace_name, + layer_name, + Rule.LIMIT, + catalog_mode=Rule.CM_MIXED, + user=_user, + geo_limit=wkt, + ) + ) wkt = resolve_geolimits(anonymous_geolimits) if wkt: logger.debug(f"Adding GeoFence ANON GeoLimit rule: L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - geo_limit=wkt)) + batch.add_insert_rule( + Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, geo_limit=wkt) + ) wkt = resolve_geolimits(groups_geolimits) if wkt: logger.debug(f"Adding GeoFence GROUP GeoLimit rule: G:{_group} L:{dataset} ") - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, - group=_group, - geo_limit=wkt)) + batch.add_insert_rule( + Rule( + pri.__next__(), + workspace_name, + layer_name, + Rule.LIMIT, + catalog_mode=Rule.CM_MIXED, + group=_group, + geo_limit=wkt, + ) + ) # Set services rules for service, allowed in gf_services.items(): if dataset and layer_name and allowed: @@ -336,34 +354,54 @@ def resolve_geolimits(geolimits): logger.debug(f"Adding GeoFence USER rules: U:{_user} S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, - service=service, request=request, user=_user)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service, user=_user)) + batch.add_insert_rule( + Rule( + pri.__next__(), + workspace_name, + layer_name, + enabled, + service=service, + request=request, + user=_user, + ) + ) + batch.add_insert_rule( + Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, service=service, user=_user) + ) elif not _group: logger.debug(f"Adding GeoFence ANON rules: S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, - service=service, request=request)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service)) + batch.add_insert_rule( + Rule(pri.__next__(), workspace_name, layer_name, enabled, service=service, request=request) + ) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, service=service)) if _group: logger.debug(f"Adding GeoFence GROUP rules: G:{_group} S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, - service=service, request=request, group=_group)) - batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, - service=service, group=_group)) + batch.add_insert_rule( + Rule( + pri.__next__(), + workspace_name, + layer_name, + enabled, + service=service, + request=request, + group=_group, + ) + ) + batch.add_insert_rule( + Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, service=service, group=_group) + ) gf_client.run_batch(batch) - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): set_geofence_invalidate_cache() else: dataset.set_dirty_state() @@ -383,15 +421,15 @@ def sync_resources_with_guardian(resource=None): if dirty_resources and dirty_resources.exists(): logger.debug(" --------------------------- synching with guardian!") for r in dirty_resources: - if r.polymorphic_ctype.name == 'dataset': + if r.polymorphic_ctype.name == "dataset": layer = None try: purge_geofence_dataset_rules(r) layer = Dataset.objects.get(id=r.id) perm_spec = layer.get_all_level_info() # All the other users - if 'users' in perm_spec: - for user, perms in perm_spec['users'].items(): + if "users" in perm_spec: + for user, perms in perm_spec["users"].items(): user = get_user_model().objects.get(username=user) # Set the GeoFence User Rules geofence_user = str(user) @@ -399,8 +437,8 @@ def sync_resources_with_guardian(resource=None): geofence_user = None sync_geofence_with_guardian(layer, perms, user=geofence_user) # All the other groups - if 'groups' in perm_spec: - for group, perms in perm_spec['groups'].items(): + if "groups" in perm_spec: + for group, perms in perm_spec["groups"].items(): group = Group.objects.get(name=group) # Set the GeoFence Group Rules sync_geofence_with_guardian(layer, perms, group=group) @@ -436,33 +474,28 @@ def get_user_geolimits(layer, user, group): def _get_gf_services(layer, perms): gf_services = {} - gf_services["WMS"] = 'view_resourcebase' in perms or 'change_dataset_style' in perms - gf_services["GWC"] = 'view_resourcebase' in perms or 'change_dataset_style' in perms - gf_services["WFS"] = ('download_resourcebase' in perms or 'change_dataset_data' in perms) \ - and layer.is_vector() - gf_services["WCS"] = ('download_resourcebase' in perms or 'change_dataset_data' in perms) \ - and not layer.is_vector() - gf_services["WPS"] = 'download_resourcebase' in perms or 'change_dataset_data' in perms - gf_services["*"] = 'download_resourcebase' in perms and \ - ('view_resourcebase' in perms or 'change_dataset_style' in perms) + gf_services["WMS"] = "view_resourcebase" in perms or "change_dataset_style" in perms + gf_services["GWC"] = "view_resourcebase" in perms or "change_dataset_style" in perms + gf_services["WFS"] = ("download_resourcebase" in perms or "change_dataset_data" in perms) and layer.is_vector() + gf_services["WCS"] = ("download_resourcebase" in perms or "change_dataset_data" in perms) and not layer.is_vector() + gf_services["WPS"] = "download_resourcebase" in perms or "change_dataset_data" in perms + gf_services["*"] = "download_resourcebase" in perms and ( + "view_resourcebase" in perms or "change_dataset_style" in perms + ) return gf_services def _get_gwc_filters_and_formats(disable_cache: list = []) -> typing.Tuple[list, list]: - filters = [{ - "styleParameterFilter": { - "STYLES": "" - } - }] + filters = [{"styleParameterFilter": {"STYLES": ""}}] formats = [ - 'application/json;type=utfgrid', - 'image/gif', - 'image/jpeg', - 'image/png', - 'image/png8', - 'image/vnd.jpeg-png', - 'image/vnd.jpeg-png8' + "application/json;type=utfgrid", + "image/gif", + "image/jpeg", + "image/png", + "image/png8", + "image/vnd.jpeg-png", + "image/vnd.jpeg-png8", ] if disable_cache and any(disable_cache): filters = None diff --git a/geonode/geoserver/signals.py b/geonode/geoserver/signals.py index 08eacb70e34..f73641629a5 100644 --- a/geonode/geoserver/signals.py +++ b/geonode/geoserver/signals.py @@ -28,9 +28,7 @@ # use different name to avoid module clash from geonode.utils import is_monochromatic_image from geonode.decorators import on_ogc_backend -from geonode.geoserver.helpers import ( - gs_catalog, - ogc_server_settings) +from geonode.geoserver.helpers import gs_catalog, ogc_server_settings from geonode.geoserver.tasks import geoserver_create_thumbnail from geonode.layers.models import Dataset from geonode.services.enumerations import CASCADED @@ -40,9 +38,9 @@ logger = logging.getLogger("geonode.geoserver.signals") -geoserver_automatic_default_style_set = Signal(providing_args=['instance']) +geoserver_automatic_default_style_set = Signal(providing_args=["instance"]) -post_set_permissions = Signal(providing_args=['instance']) +geofence_rule_assign = Signal(providing_args=["instance"]) def geoserver_delete(typename): @@ -54,12 +52,15 @@ def geoserver_delete(typename): @on_ogc_backend(BACKEND_PACKAGE) def geoserver_pre_delete(instance, sender, **kwargs): - """Removes the layer from GeoServer - """ + """Removes the layer from GeoServer""" # cascading_delete should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): - if not hasattr(instance, 'remote_service') or instance.remote_service is None or instance.remote_service.method == CASCADED: + if ( + not hasattr(instance, "remote_service") + or instance.remote_service is None + or instance.remote_service.method == CASCADED + ): if instance.alternate: geoserver_cascading_delete.apply_async((instance.alternate,)) @@ -68,34 +69,30 @@ def geoserver_pre_delete(instance, sender, **kwargs): def geoserver_post_save_local(instance, *args, **kwargs): """Send information to geoserver. - The attributes sent include: + The attributes sent include: - * Title - * Abstract - * Name - * Keywords - * Metadata Links, - * Point of Contact name and url + * Title + * Abstract + * Name + * Keywords + * Metadata Links, + * Point of Contact name and url """ - geoserver_post_save_datasets.apply_async( - (instance.id, args, kwargs)) + geoserver_post_save_datasets.apply_async((instance.id, args, kwargs)) @on_ogc_backend(BACKEND_PACKAGE) def geoserver_pre_save_maplayer(instance, sender, **kwargs): # If this object was saved via fixtures, # do not do post processing. - if kwargs.get('raw', False): + if kwargs.get("raw", False): return try: - instance.local = isinstance( - gs_catalog.get_layer( - instance.name), - GsLayer) + instance.local = isinstance(gs_catalog.get_layer(instance.name), GsLayer) except OSError as e: if e.errno == errno.ECONNREFUSED: - msg = f'Could not connect to catalog to verify if layer {instance.name} was local' + msg = f"Could not connect to catalog to verify if layer {instance.name} was local" logger.warn(msg) else: raise e @@ -113,16 +110,22 @@ def geoserver_pre_save_maplayer(instance, sender, **kwargs): pass -@deprecated(version='3.2.1', reason="Use direct calls to the ReourceManager.") +@deprecated(version="3.2.1", reason="Use direct calls to the ReourceManager.") def geoserver_post_save_map(instance, sender, created, **kwargs): instance.set_missing_info() if not created: if not instance.thumbnail_url: logger.debug(f"... Creating Thumbnail for Map [{instance.title}]") - geoserver_create_thumbnail.apply_async((instance.id, False, True, )) + geoserver_create_thumbnail.apply_async( + ( + instance.id, + False, + True, + ) + ) -@deprecated(version='3.2.1', reason="Use direct calls to the ReourceManager.") +@deprecated(version="3.2.1", reason="Use direct calls to the ReourceManager.") def geoserver_set_thumbnail(instance, **kwargs): # Creating Dataset Thumbnail # some thumbnail generators will update thumbnail_url. If so, don't @@ -130,14 +133,22 @@ def geoserver_set_thumbnail(instance, **kwargs): try: logger.debug(f"... Creating Thumbnail for Dataset {instance.title}") _recreate_thumbnail = False - if 'update_fields' in kwargs and kwargs['update_fields'] is not None and \ - 'thumbnail_url' in kwargs['update_fields']: + if ( + "update_fields" in kwargs + and kwargs["update_fields"] is not None + and "thumbnail_url" in kwargs["update_fields"] + ): _recreate_thumbnail = True - if not instance.thumbnail_url or \ - is_monochromatic_image(instance.thumbnail_url): + if not instance.thumbnail_url or is_monochromatic_image(instance.thumbnail_url): _recreate_thumbnail = True if _recreate_thumbnail: - geoserver_create_thumbnail.apply_async((instance.id, False, True, )) + geoserver_create_thumbnail.apply_async( + ( + instance.id, + False, + True, + ) + ) else: logger.debug(f"... Thumbnail for Dataset {instance.title} already exists: {instance.thumbnail_url}") except Exception as e: diff --git a/geonode/geoserver/tasks.py b/geonode/geoserver/tasks.py index f109002a04d..5d29fde8d4c 100644 --- a/geonode/geoserver/tasks.py +++ b/geonode/geoserver/tasks.py @@ -26,9 +26,7 @@ from celery.utils.log import get_task_logger from geonode.celery_app import app -from geonode.tasks.tasks import ( - AcquireLock, - FaultTolerantTask) +from geonode.tasks.tasks import AcquireLock, FaultTolerantTask from geonode.base.models import Link from geonode.base import enumerations from geonode.layers.models import Dataset @@ -43,7 +41,8 @@ set_dataset_style, cascading_delete, create_gs_thumbnail, - sync_instance_with_geoserver) + sync_instance_with_geoserver, +) logger = get_task_logger(__name__) @@ -53,21 +52,22 @@ @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_update_datasets', - queue='geoserver.catalog', + name="geonode.geoserver.tasks.geoserver_update_datasets", + queue="geoserver.catalog", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def geoserver_update_datasets(self, *args, **kwargs): """ Runs update layers. """ - lock_id = f'{self.request.id}' + lock_id = f"{self.request.id}" log_lock.debug(f"geoserver_update_datasets: Creating lock {lock_id}") with AcquireLock(lock_id) as lock: log_lock.debug(f"geoserver_update_datasets: Acquiring lock {lock_id}") @@ -83,20 +83,18 @@ def geoserver_update_datasets(self, *args, **kwargs): @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_set_style', - queue='geoserver.catalog', + name="geonode.geoserver.tasks.geoserver_set_style", + queue="geoserver.catalog", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) -def geoserver_set_style( - self, - instance_id, - base_file): + retry_jitter=False, +) +def geoserver_set_style(self, instance_id, base_file): """ Sets styles from SLD file. """ @@ -107,7 +105,7 @@ def geoserver_set_style( logger.debug(f"Dataset id {instance_id} does not exist yet!") raise - lock_id = f'{self.request.id}' if self.request.id else instance.name + lock_id = f"{self.request.id}" if self.request.id else instance.name log_lock.debug(f"geoserver_set_style: Creating lock {lock_id} for {instance.name}") with AcquireLock(lock_id) as lock: log_lock.debug(f"geoserver_set_style: Acquiring lock {lock_id} for {instance.name}") @@ -121,11 +119,7 @@ def geoserver_set_style( sld = base_file base_file = None - set_dataset_style( - instance, - instance.alternate, - sld, - base_file=base_file) + set_dataset_style(instance, instance.alternate, sld, base_file=base_file) except Exception as e: logger.exception(e) @@ -137,26 +131,23 @@ def geoserver_set_style( @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_create_style', - queue='geoserver.catalog', + name="geonode.geoserver.tasks.geoserver_create_style", + queue="geoserver.catalog", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) -def geoserver_create_style( - self, - instance_id, - name, - sld_file, - tempdir): + retry_jitter=False, +) +def geoserver_create_style(self, instance_id, name, sld_file, tempdir): """ Sets or create styles from Upload Session. """ from geonode.geoserver.signals import geoserver_automatic_default_style_set + instance = None try: instance = Dataset.objects.get(id=instance_id) @@ -164,7 +155,7 @@ def geoserver_create_style( logger.debug(f"Dataset id {instance_id} does not exist yet!") raise - lock_id = f'{self.request.id}' if self.request.id else instance.name + lock_id = f"{self.request.id}" if self.request.id else instance.name log_lock.debug(f"geoserver_create_style: Creating lock {lock_id} for {instance.name}") with AcquireLock(lock_id) as lock: log_lock.debug(f"geoserver_create_style: Acquiring lock {lock_id} for {instance.name}") @@ -188,11 +179,7 @@ def geoserver_create_style( sld = f.read() f.close() if not gs_catalog.get_style(name=name, workspace=settings.DEFAULT_WORKSPACE): - style = gs_catalog.create_style( - name, - sld, - raw=True, - workspace=settings.DEFAULT_WORKSPACE) + style = gs_catalog.create_style(name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) gs_dataset = gs_catalog.get_layer(name) _default_style = gs_dataset.default_style gs_dataset.default_style = style @@ -202,8 +189,9 @@ def geoserver_create_style( gs_catalog.delete(_default_style) Link.objects.filter( resource=instance.resourcebase_ptr, - name='Legend', - url__contains=f'STYLE={_default_style.name}').delete() + name="Legend", + url__contains=f"STYLE={_default_style.name}", + ).delete() except Exception as e: logger.exception(e) else: @@ -221,20 +209,18 @@ def geoserver_create_style( @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_post_save_datasets', - queue='geoserver.catalog', + name="geonode.geoserver.tasks.geoserver_post_save_datasets", + queue="geoserver.catalog", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) -def geoserver_post_save_datasets( - self, - instance_id, - *args, **kwargs): + retry_jitter=False, +) +def geoserver_post_save_datasets(self, instance_id, *args, **kwargs): """ Runs update layers. """ @@ -245,7 +231,7 @@ def geoserver_post_save_datasets( logger.debug(f"Dataset id {instance_id} does not exist yet!") raise - lock_id = f'{self.request.id}' if self.request.id else instance.name + lock_id = f"{self.request.id}" if self.request.id else instance.name log_lock.debug(f"geoserver_post_save_datasets: Creating lock {lock_id} for {instance_id}") with AcquireLock(lock_id) as lock: log_lock.debug(f"geoserver_post_save_datasets: Acquiring lock {lock_id} for {instance_id}") @@ -256,7 +242,7 @@ def geoserver_post_save_datasets( # Updating HAYSTACK Indexes if needed if settings.HAYSTACK_SEARCH: - call_command('update_index') + call_command("update_index") finally: lock.release() log_lock.debug(f"geoserver_post_save_datasets: Releasing lock {lock_id} for {instance_id}") @@ -265,16 +251,17 @@ def geoserver_post_save_datasets( @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_create_thumbnail', - queue='geoserver.events', + name="geonode.geoserver.tasks.geoserver_create_thumbnail", + queue="geoserver.events", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def geoserver_create_thumbnail(self, instance_id, overwrite=True, check_bbox=True): """ Runs create_gs_thumbnail. @@ -286,7 +273,7 @@ def geoserver_create_thumbnail(self, instance_id, overwrite=True, check_bbox=Tru logger.error(f"Resource id {instance_id} does not exist yet!") raise - lock_id = f'{self.request.id}' if self.request.id else instance.name + lock_id = f"{self.request.id}" if self.request.id else instance.name log_lock.debug(f"geoserver_create_thumbnail: Creating lock {lock_id} for {instance.name}") with AcquireLock(lock_id) as lock: log_lock.debug(f"geoserver_create_thumbnail: Acquiring lock {lock_id} for {instance.name}") @@ -310,21 +297,22 @@ def geoserver_create_thumbnail(self, instance_id, overwrite=True, check_bbox=Tru @app.task( bind=True, base=FaultTolerantTask, - name='geonode.geoserver.tasks.geoserver_cascading_delete', - queue='cleanup', + name="geonode.geoserver.tasks.geoserver_cascading_delete", + queue="cleanup", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def geoserver_cascading_delete(self, *args, **kwargs): """ Runs cascading_delete. """ - lock_id = f'{self.request.id}' + lock_id = f"{self.request.id}" with AcquireLock(lock_id) as lock: if lock.acquire() is True: try: @@ -335,22 +323,24 @@ def geoserver_cascading_delete(self, *args, **kwargs): @app.task( bind=True, - name='geonode.geoserver.tasks.geoserver_delete_map', - queue='cleanup', + name="geonode.geoserver.tasks.geoserver_delete_map", + queue="cleanup", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def geoserver_delete_map(self, object_id): """ Deletes a map and the associated map layers. """ from geonode.maps.models import Map - lock_id = f'{self.request.id}' + + lock_id = f"{self.request.id}" with AcquireLock(lock_id) as lock: if lock.acquire() is True: try: @@ -367,19 +357,20 @@ def geoserver_delete_map(self, object_id): @shared_task( bind=True, - name='geonode.security.tasks.synch_guardian', - queue='security', + name="geonode.security.tasks.synch_guardian", + queue="security", expires=600, time_limit=600, acks_late=False, - autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5}, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, retry_backoff=3, retry_backoff_max=30, - retry_jitter=False) + retry_jitter=False, +) def synch_guardian(): """ Sync resources with Guardian and clear their dirty state """ - if getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if getattr(settings, "DELAYED_SECURITY_SIGNALS", False): sync_resources_with_guardian() diff --git a/geonode/geoserver/tests/integration.py b/geonode/geoserver/tests/integration.py index 8f5c6063ed5..7fd155210ca 100644 --- a/geonode/geoserver/tests/integration.py +++ b/geonode/geoserver/tests/integration.py @@ -54,42 +54,36 @@ def _log(msg, *args): logger.debug(msg, *args) -@override_settings(SITEURL='http://localhost:8001/') +@override_settings(SITEURL="http://localhost:8001/") class GeoNodeGeoServerSync(GeoNodeLiveTestSupport): """ Tests GeoNode/GeoServer syncronization """ + port = 8001 def setUp(self): super(GeoNodeLiveTestSupport, self).setUp() - settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED'] = True + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True @on_ogc_backend(geoserver.BACKEND_PACKAGE) @timeout_decorator.timeout(LOCAL_TIMEOUT) def test_set_attributes_from_geoserver(self): - """Test attributes syncronization - """ + """Test attributes syncronization""" layer = Dataset.objects.all().first() create_dataset_data(layer.resourcebase_ptr_id) try: # set attributes for resource for attribute in layer.attribute_set.all(): - attribute.attribute_label = f'{attribute.attribute}_label' - attribute.description = f'{attribute.attribute}_description' + attribute.attribute_label = f"{attribute.attribute}_label" + attribute.description = f"{attribute.attribute}_description" attribute.save() # tests if everything is synced properly for attribute in layer.attribute_set.all(): - self.assertEqual( - attribute.attribute_label, - f'{attribute.attribute}_label' - ) - self.assertEqual( - attribute.description, - f'{attribute.attribute}_description' - ) + self.assertEqual(attribute.attribute_label, f"{attribute.attribute}_label") + self.assertEqual(attribute.description, f"{attribute.attribute}_description") # sync the attributes with GeoServer # since on geoserver are empty, we expect that now the layer @@ -100,7 +94,7 @@ def test_set_attributes_from_geoserver(self): self.assertIsNotNone(links) self.assertTrue(len(links) >= 7) - original_data_links = [ll for ll in links if 'original' == ll.link_type] + original_data_links = [ll for ll in links if "original" == ll.link_type] self.assertEqual(len(original_data_links), 0) finally: @@ -108,23 +102,23 @@ def test_set_attributes_from_geoserver(self): layer.delete() -@override_settings(SITEURL='http://localhost:8002/') +@override_settings(SITEURL="http://localhost:8002/") class GeoNodeGeoServerCapabilities(GeoNodeLiveTestSupport): """ Tests GeoNode/GeoServer GetCapabilities per layer, user, category and map """ + port = 8002 def setUp(self): super(GeoNodeLiveTestSupport, self).setUp() - settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED'] = True + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True @on_ogc_backend(geoserver.BACKEND_PACKAGE) @timeout_decorator.timeout(LOCAL_TIMEOUT) def test_capabilities(self): - """Test capabilities - """ + """Test capabilities""" # a category category = TopicCategory.objects.first() @@ -135,50 +129,40 @@ def test_capabilities(self): # create 3 layers, 2 with norman as an owner an 2 with category as a category layer1 = create_dataset( - name='layer1', - title="san_andres_y_providencia_poi", - owner_name=norman, - geometry_type="Point" - ) - layer2 = create_dataset( - name='layer2', - title="single_point", - owner_name=norman, - geometry_type="Point" + name="layer1", title="san_andres_y_providencia_poi", owner_name=norman, geometry_type="Point" ) + layer2 = create_dataset(name="layer2", title="single_point", owner_name=norman, geometry_type="Point") layer2.category = category layer2.save() layer3 = create_dataset( - name='layer3', - title="san_andres_y_providencia_administrative", - owner_name=admin, - geometry_type="Point" + name="layer3", title="san_andres_y_providencia_administrative", owner_name=admin, geometry_type="Point" ) layer3.category = category layer3.save() try: - namespaces = {'wms': 'http://www.opengis.net/wms', - 'xlink': 'http://www.w3.org/1999/xlink', - 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} + namespaces = { + "wms": "http://www.opengis.net/wms", + "xlink": "http://www.w3.org/1999/xlink", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } # 0. test capabilities_dataset - url = reverse('capabilities_dataset', args=[layer1.id]) + url = reverse("capabilities_dataset", args=[layer1.id]) resp = self.client.get(url) layercap = dlxml.fromstring(resp.content) rootdoc = etree.ElementTree(layercap) - layernodes = rootdoc.findall('./[wms:Name]', namespaces) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) layernode = layernodes[0] self.assertEqual(1, len(layernodes)) - self.assertEqual(layernode.find('wms:Name', namespaces).text, - layer1.name) + self.assertEqual(layernode.find("wms:Name", namespaces).text, layer1.name) # 1. test capabilities_user - url = reverse('capabilities_user', args=[norman.username]) + url = reverse("capabilities_user", args=[norman.username]) resp = self.client.get(url) layercap = dlxml.fromstring(resp.content) rootdoc = etree.ElementTree(layercap) - layernodes = rootdoc.findall('./[wms:Name]', namespaces) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) # norman has 2 layers self.assertEqual(1, len(layernodes)) @@ -186,18 +170,18 @@ def test_capabilities(self): # the norman two layers are named layer1 and layer2 count = 0 for layernode in layernodes: - if layernode.find('wms:Name', namespaces).text == layer1.name: + if layernode.find("wms:Name", namespaces).text == layer1.name: count += 1 - elif layernode.find('wms:Name', namespaces).text == layer2.name: + elif layernode.find("wms:Name", namespaces).text == layer2.name: count += 1 self.assertEqual(1, count) # 2. test capabilities_category - url = reverse('capabilities_category', args=[category.identifier]) + url = reverse("capabilities_category", args=[category.identifier]) resp = self.client.get(url) layercap = dlxml.fromstring(resp.content) rootdoc = etree.ElementTree(layercap) - layernodes = rootdoc.findall('./[wms:Name]', namespaces) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) # category is in two layers self.assertEqual(1, len(layernodes)) @@ -205,9 +189,9 @@ def test_capabilities(self): # the layers for category are named layer1 and layer3 count = 0 for layernode in layernodes: - if layernode.find('wms:Name', namespaces).text == layer1.name: + if layernode.find("wms:Name", namespaces).text == layer1.name: count += 1 - elif layernode.find('wms:Name', namespaces).text == layer3.name: + elif layernode.find("wms:Name", namespaces).text == layer3.name: count += 1 self.assertEqual(1, count) @@ -220,22 +204,22 @@ def test_capabilities(self): layer3.delete() -@override_settings(SITEURL='http://localhost:8003/') +@override_settings(SITEURL="http://localhost:8003/") class GeoNodePermissionsTest(GeoNodeLiveTestSupport): """ Tests GeoNode permissions and its integration with GeoServer """ + port = 8003 def setUp(self): super(GeoNodeLiveTestSupport, self).setUp() - settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED'] = True + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True @on_ogc_backend(geoserver.BACKEND_PACKAGE) @timeout_decorator.timeout(LOCAL_TIMEOUT) def test_unpublished(self): - """Test permissions on an unpublished layer - """ + """Test permissions on an unpublished layer""" layer = Dataset.objects.first() layer.set_default_permissions() check_dataset(layer) @@ -247,11 +231,10 @@ def test_unpublished(self): # request getCapabilities: layer must be there as it is published and # advertised: we need to check if in response there is # geonode:san_andres_y_providencia_water - geoserver_base_url = settings.OGC_SERVER['default']['LOCATION'] - get_capabilities_url = 'ows?' \ - 'service=wms&version=1.3.0&request=GetCapabilities' + geoserver_base_url = settings.OGC_SERVER["default"]["LOCATION"] + get_capabilities_url = "ows?" "service=wms&version=1.3.0&request=GetCapabilities" url = urljoin(geoserver_base_url, get_capabilities_url) - str_to_check = f'geonode:{layer.name}' + str_to_check = f"geonode:{layer.name}" request = Request(url) response = urlopen(request) @@ -302,71 +285,67 @@ def test_unpublished(self): def test_default_anonymous_permissions(self): anonymous = get_user_model().objects.get(username="AnonymousUser") norman = get_user_model().objects.get(username="norman") - with override_settings(RESOURCE_PUBLISHING=False, - ADMIN_MODERATE_UPLOADS=False, - DEFAULT_ANONYMOUS_VIEW_PERMISSION=True, - DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False): - self.client.login(username='norman', password='norman') + with override_settings( + RESOURCE_PUBLISHING=False, + ADMIN_MODERATE_UPLOADS=False, + DEFAULT_ANONYMOUS_VIEW_PERMISSION=True, + DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False, + ): + self.client.login(username="norman", password="norman") saved_dataset = create_dataset( - name='san_andres_y_providencia_poi_by_norman', - title='san_andres_y_providencia_poi', + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", owner_name=norman, - geometry_type='Point' + geometry_type="Point", ) try: - namespaces = {'wms': 'http://www.opengis.net/wms', - 'xlink': 'http://www.w3.org/1999/xlink', - 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} - url = urljoin(settings.SITEURL, reverse('capabilities_dataset', args=[saved_dataset.id])) + namespaces = { + "wms": "http://www.opengis.net/wms", + "xlink": "http://www.w3.org/1999/xlink", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } + url = urljoin(settings.SITEURL, reverse("capabilities_dataset", args=[saved_dataset.id])) resp = self.client.get(url) content = resp.content self.assertTrue(content) layercap = dlxml.fromstring(content) rootdoc = etree.ElementTree(layercap) - layernodes = rootdoc.findall('./[wms:Name]', namespaces) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) layernode = layernodes[0] self.assertEqual(1, len(layernodes)) - self.assertEqual(layernode.find('wms:Name', namespaces).text, - saved_dataset.name) + self.assertEqual(layernode.find("wms:Name", namespaces).text, saved_dataset.name) self.client.logout() resp = self.client.get(url) layercap = dlxml.fromstring(resp.content) self.assertIsNotNone(layercap) finally: # annonymous can not download created resource - self.assertEqual(['view_resourcebase'], self.get_user_resource_perms(saved_dataset, anonymous)) + self.assertEqual(["view_resourcebase"], self.get_user_resource_perms(saved_dataset, anonymous)) # Cleanup saved_dataset.delete() - with override_settings( - DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, - DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True - ): + with override_settings(DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True): saved_dataset = create_dataset( - name='san_andres_y_providencia_poi_by_norman', - title='san_andres_y_providencia_poi', + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", owner_name=norman, - geometry_type='Point' + geometry_type="Point", ) # annonymous can view/download created resource self.assertEqual( - ['download_resourcebase', 'view_resourcebase'], - self.get_user_resource_perms(saved_dataset, anonymous) + ["download_resourcebase", "view_resourcebase"], self.get_user_resource_perms(saved_dataset, anonymous) ) # Cleanup saved_dataset.delete() - with override_settings( - DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, - DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False - ): + with override_settings(DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False): saved_dataset = create_dataset( - name='san_andres_y_providencia_poi_by_norman', - title='san_andres_y_providencia_poi', + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", owner_name=norman, - geometry_type='Point' + geometry_type="Point", ) # annonymous can not view/download created resource self.assertEqual([], self.get_user_resource_perms(saved_dataset, anonymous)) @@ -374,6 +353,4 @@ def test_default_anonymous_permissions(self): saved_dataset.delete() def get_user_resource_perms(self, instance, user): - return list( - instance.get_user_perms(user).union(instance.get_self_resource().get_user_perms(user)) - ) + return list(instance.get_user_perms(user).union(instance.get_self_resource().get_user_perms(user))) diff --git a/geonode/geoserver/tests/test_helpers.py b/geonode/geoserver/tests/test_helpers.py index d13b8cf568e..1a871d11eaf 100644 --- a/geonode/geoserver/tests/test_helpers.py +++ b/geonode/geoserver/tests/test_helpers.py @@ -32,35 +32,21 @@ from geonode.decorators import on_ogc_backend from geonode.tests.base import GeoNodeBaseTestSupport from geonode.geoserver.views import _response_callback -from geonode.geoserver.helpers import ( - gs_catalog, - ows_endpoint_in_path, - get_dataset_storetype, - extract_name_from_sld) +from geonode.geoserver.helpers import gs_catalog, ows_endpoint_in_path, get_dataset_storetype, extract_name_from_sld from geonode.layers.populate_datasets_data import create_dataset_data -from geonode.geoserver.ows import ( - _wcs_link, - _wfs_link, - _wms_link) +from geonode.geoserver.ows import _wcs_link, _wfs_link, _wms_link -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models) +from geonode.base.populate_test_data import all_public, create_models, remove_models logger = logging.getLogger(__name__) class HelperTest(GeoNodeBaseTestSupport): - type = 'dataset' + type = "dataset" - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] @classmethod def setUpClass(cls): @@ -75,8 +61,8 @@ def tearDownClass(cls): def setUp(self): super().setUp() - self.user = 'admin' - self.passwd = 'admin' + self.user = "admin" + self.passwd = "admin" create_dataset_data() @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -96,30 +82,29 @@ def test_extract_name_from_sld(self): def test_safe_path_leaf(self): base_path = settings.MEDIA_ROOT - malformed_paths = [ - 'c:/etc/passwd', - 'c:\\etc\\passwd', - '\0_a*b:ce%f/(g)h+i_0.txt' - ] + malformed_paths = ["c:/etc/passwd", "c:\\etc\\passwd", "\0_a*b:ce%f/(g)h+i_0.txt"] for _path in malformed_paths: with self.assertRaises(ValidationError): safe_path_leaf(_path) unsafe_paths = [ - '/root/', - '~/.ssh', - '$HOME/.ssh', - '/etc/passwd', - '.../style.sld', + "/root/", + "~/.ssh", + "$HOME/.ssh", + "/etc/passwd", + ".../style.sld", 'fi:l*e/p"a?t>h|.t """ - kwargs = { - 'content': content, - 'status': 200, - 'content_type': 'application/xml' - } + kwargs = {"content": content, "status": 200, "content_type": "application/xml"} _content = _response_callback(**kwargs).content self.assertTrue(re.findall(f'{urljoin(settings.SITEURL, "/gs/")}ows', str(_content))) - kwargs = { - 'content': content, - 'status': 200, - 'content_type': 'text/xml; charset=UTF-8' - } + kwargs = {"content": content, "status": 200, "content_type": "text/xml; charset=UTF-8"} _content = _response_callback(**kwargs).content self.assertTrue(re.findall(f'{urljoin(settings.SITEURL, "/gs/")}ows', str(_content))) def test_return_element_if_not_exists_in_the_subtypes(self): - el = get_dataset_storetype('not-existing-type') - self.assertEqual('not-existing-type', el) + el = get_dataset_storetype("not-existing-type") + self.assertEqual("not-existing-type", el) def test_datastore_should_return_vector(self): - el = get_dataset_storetype('dataStore') - self.assertEqual('vector', el) + el = get_dataset_storetype("dataStore") + self.assertEqual("vector", el) def test_coverageStore_should_return_raster(self): - el = get_dataset_storetype('coverageStore') - self.assertEqual('raster', el) + el = get_dataset_storetype("coverageStore") + self.assertEqual("raster", el) def test_remoteStore_should_return_remote(self): - el = get_dataset_storetype('remoteStore') - self.assertEqual('remote', el) + el = get_dataset_storetype("remoteStore") + self.assertEqual("remote", el) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_geoserver_proxy_strip_paths(self): - response = self.client.get(f"{reverse('gs_layers')}?service=WFS&version=1.1.0&request=DescribeFeatureType&typeName=geonode:tipi_forestali&outputFormat=application/json&access_token=something") + response = self.client.get( + f"{reverse('gs_layers')}?service=WFS&version=1.1.0&request=DescribeFeatureType&typeName=geonode:tipi_forestali&outputFormat=application/json&access_token=something" + ) self.assertEqual(response.status_code, 200) - response = self.client.get(f"{reverse('ows_endpoint')}?service=WFS&version=1.1.0&request=DescribeFeatureType&typeName=geonode:tipi_forestali&outputFormat=image/png&access_token=something") + response = self.client.get( + f"{reverse('ows_endpoint')}?service=WFS&version=1.1.0&request=DescribeFeatureType&typeName=geonode:tipi_forestali&outputFormat=image/png&access_token=something" + ) self.assertEqual(response.status_code, 200) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_ows_links(self): - ows_url = 'http://foo.org/ows' - identifier = 'foo:fake_alternate' + ows_url = "http://foo.org/ows" + identifier = "foo:fake_alternate" min_x, min_y, max_x, max_y = -1, -1, 1, 1 - expected_url = f'{ows_url}?service=WCS&request=GetCoverage&coverageid=foo__fake_alternate&format=image%2Ftiff&version=2.0.1&compression=DEFLATE&tileWidth=512&tileHeight=512&outputCrs=4326' + expected_url = f"{ows_url}?service=WCS&request=GetCoverage&coverageid=foo__fake_alternate&format=image%2Ftiff&version=2.0.1&compression=DEFLATE&tileWidth=512&tileHeight=512&outputCrs=4326" download_url = _wcs_link( ows_url, identifier, "image/tiff", - srid='4326', + srid="4326", bbox=[min_x, min_y, max_x, max_y], compression="DEFLATE", - tile_size=512) + tile_size=512, + ) self.assertEqual(download_url, expected_url, download_url) - expected_url = f'{ows_url}?service=WFS&version=1.0.0&request=GetFeature&typename=foo%3Afake_alternate&outputFormat=application%2Fzip&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D' + expected_url = f"{ows_url}?service=WFS&version=1.0.0&request=GetFeature&typename=foo%3Afake_alternate&outputFormat=application%2Fzip&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D" download_url = _wfs_link( - ows_url, - identifier, - "application/zip", - {}, - srid='4326', - bbox=[min_x, min_y, max_x, max_y]) + ows_url, identifier, "application/zip", {}, srid="4326", bbox=[min_x, min_y, max_x, max_y] + ) self.assertEqual(download_url, expected_url, download_url) - expected_url = f'{ows_url}?service=WMS&request=GetMap&layers=foo%3Afake_alternate&format=image%2Fpng&height=512&width=512&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D' + expected_url = f"{ows_url}?service=WMS&request=GetMap&layers=foo%3Afake_alternate&format=image%2Fpng&height=512&width=512&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D" download_url = _wms_link( - ows_url, - identifier, - "image/png", - 512, 512, - srid='4326', - bbox=[min_x, min_y, max_x, max_y]) + ows_url, identifier, "image/png", 512, 512, srid="4326", bbox=[min_x, min_y, max_x, max_y] + ) self.assertEqual(download_url, expected_url, download_url) @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -307,7 +281,7 @@ def test_ows_endpoint_in_path(self): end_time_1 = time.time() - start_time start_time = time.time() - re.match(r'.*/(rest)/.*$', path, re.IGNORECASE) + re.match(r".*/(rest)/.*$", path, re.IGNORECASE) end_time_2 = time.time() - start_time self.assertLess(end_time_1, end_time_2) diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py index b78242cf930..85672984b00 100644 --- a/geonode/geoserver/tests/test_manager.py +++ b/geonode/geoserver/tests/test_manager.py @@ -37,14 +37,13 @@ class TestGeoServerResourceManager(GeoNodeBaseTestSupport): - def setUp(self): self.files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") self.files_as_dict, self.tmpdir = get_files(self.files) self.cat = gs_catalog self.user = get_user_model().objects.get(username="admin") self.sut = create_single_dataset("san_andres_y_providencia_water.shp") - self.sut.name = 'san_andres_y_providencia_water' + self.sut.name = "san_andres_y_providencia_water" self.sut.save() self.geoserver_url = settings.GEOSERVER_LOCATION self.geoserver_manager = GeoServerResourceManager() @@ -57,27 +56,35 @@ def tearDown(self) -> None: @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self): layer = Dataset.objects.get(name=self.sut.name) - _gs_import_session_info = self.geoserver_manager._execute_resource_import(layer, list(self.files_as_dict.values()), self.user, action_type="append") - basic_auth = base64.b64encode(b'admin:geoserver') + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="append" + ) + basic_auth = base64.b64encode(b"admin:geoserver") result = requests.get( - f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}', - headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}) + f"{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}", + headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}, + ) self.assertEqual(result.status_code, 200) - self.assertEqual(result.json().get('import').get('state'), enumerations.STATE_COMPLETE) + self.assertEqual(result.json().get("import").get("state"), enumerations.STATE_COMPLETE) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_revise_resource_value_in_replace_should_add_expected_rows_in_the_catalog(self): layer = Dataset.objects.get(name=self.sut.name) - _gs_import_session_info = self.geoserver_manager._execute_resource_import(layer, list(self.files_as_dict.values()), self.user, action_type="replace") - basic_auth = base64.b64encode(b'admin:geoserver') + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="replace" + ) + basic_auth = base64.b64encode(b"admin:geoserver") result = requests.get( - f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}', - headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}) + f"{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}", + headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}, + ) self.assertEqual(result.status_code, 200) - self.assertEqual(result.json().get('import').get('state'), enumerations.STATE_COMPLETE) + self.assertEqual(result.json().get("import").get("state"), enumerations.STATE_COMPLETE) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_revise_resource_value_in_replace_should_return_none_for_not_existing_dataset(self): - layer = create_single_dataset('fake_dataset') - _gs_import_session_info = self.geoserver_manager._execute_resource_import(layer, list(self.files_as_dict.values()), self.user, action_type="replace") + layer = create_single_dataset("fake_dataset") + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="replace" + ) self.assertEqual(_gs_import_session_info.import_session.state, enumerations.STATE_COMPLETE) diff --git a/geonode/geoserver/tests/test_server.py b/geonode/geoserver/tests/test_server.py index 44e64dea59f..fd5f5782c27 100644 --- a/geonode/geoserver/tests/test_server.py +++ b/geonode/geoserver/tests/test_server.py @@ -46,15 +46,8 @@ from geonode.utils import mkdtemp, OGC_Servers_Handler from geonode.layers.models import Dataset, Style from geonode.layers.populate_datasets_data import create_dataset_data -from geonode.base.populate_test_data import ( - all_public, - create_models, - remove_models, - create_single_dataset) -from geonode.geoserver.helpers import ( - gs_catalog, - get_sld_for, - extract_name_from_sld) +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset +from geonode.geoserver.helpers import gs_catalog, get_sld_for, extract_name_from_sld import logging @@ -543,23 +536,19 @@ """ SLDS = { - 'san_andres_y_providencia': san_andres_y_providencia_sld, - 'lac': lac_sld, - 'freshgwabs2': freshgwabs2_sld, - 'raster': raster_sld, - 'line': line_sld + "san_andres_y_providencia": san_andres_y_providencia_sld, + "lac": lac_sld, + "freshgwabs2": freshgwabs2_sld, + "raster": raster_sld, + "line": line_sld, } class LayerTests(GeoNodeBaseTestSupport): - type = 'dataset' + type = "dataset" - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] @classmethod def setUpClass(cls): @@ -574,38 +563,34 @@ def tearDownClass(cls): def setUp(self): super().setUp() - self.user = 'admin' - self.passwd = 'admin' + self.user = "admin" + self.passwd = "admin" create_dataset_data() self.config = Configuration.load() self.OGC_DEFAULT_SETTINGS = { - 'default': { - 'BACKEND': 'geonode.geoserver', - 'LOCATION': 'http://localhost:8080/geoserver/', - 'USER': 'admin', - 'PASSWORD': 'geoserver', - 'MAPFISH_PRINT_ENABLED': True, - 'PRINT_NG_ENABLED': True, - 'GEONODE_SECURITY_ENABLED': True, - 'GEOFENCE_SECURITY_ENABLED': True, - 'WMST_ENABLED': False, - 'BACKEND_WRITE_ENABLED': True, - 'WPS_ENABLED': False, - 'DATASTORE': '', + "default": { + "BACKEND": "geonode.geoserver", + "LOCATION": "http://localhost:8080/geoserver/", + "USER": "admin", + "PASSWORD": "geoserver", + "MAPFISH_PRINT_ENABLED": True, + "PRINT_NG_ENABLED": True, + "GEONODE_SECURITY_ENABLED": True, + "GEOFENCE_SECURITY_ENABLED": True, + "WMST_ENABLED": False, + "BACKEND_WRITE_ENABLED": True, + "WPS_ENABLED": False, + "DATASTORE": "", } } self.UPLOADER_DEFAULT_SETTINGS = { - 'BACKEND': 'geonode.importer', - 'OPTIONS': { - 'TIME_ENABLED': False, - 'MOSAIC_ENABLED': False}} + "BACKEND": "geonode.importer", + "OPTIONS": {"TIME_ENABLED": False, "MOSAIC_ENABLED": False}, + } - self.DATABASE_DEFAULT_SETTINGS = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'development.db'}} + self.DATABASE_DEFAULT_SETTINGS = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "development.db"}} @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_style_manager(self): @@ -614,21 +599,15 @@ def test_style_manager(self): """ layer = Dataset.objects.first() - bob = get_user_model().objects.get(username='bobby') - assign_perm('change_dataset_style', bob, layer) + bob = get_user_model().objects.get(username="bobby") + assign_perm("change_dataset_style", bob, layer) - self.assertTrue(self.client.login(username='bobby', password='bob')) - response = self.client.get( - reverse( - 'dataset_style_manage', args=( - layer.alternate,))) + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(reverse("dataset_style_manage", args=(layer.alternate,))) self.assertEqual(response.status_code, 200) - form_data = {'default_style': 'polygon'} - response = self.client.post( - reverse( - 'dataset_style_manage', args=( - layer.alternate,)), data=form_data) + form_data = {"default_style": "polygon"} + response = self.client.post(reverse("dataset_style_manage", args=(layer.alternate,)), data=form_data) self.assertEqual(response.status_code, 302) @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -648,54 +627,44 @@ def test_style_validity_and_name(self): for f in files: path = os.path.join(d, f) - with open(path, 'w') as f: + with open(path, "w") as f: name, ext = splitext(basename(path)) f.write(SLDS[name]) # Test 'san_andres_y_providencia.sld' - san_andres_y_providencia_sld_file = os.path.join( - d, "san_andres_y_providencia.sld") + san_andres_y_providencia_sld_file = os.path.join(d, "san_andres_y_providencia.sld") with open(san_andres_y_providencia_sld_file) as san_andres_y_providencia_sld_xml_file: san_andres_y_providencia_sld_xml = san_andres_y_providencia_sld_xml_file.read() - san_andres_y_providencia_sld_name = extract_name_from_sld( - None, san_andres_y_providencia_sld_xml) - self.assertEqual( - san_andres_y_providencia_sld_name, - 'san_andres_y_providencia_administrative') + san_andres_y_providencia_sld_name = extract_name_from_sld(None, san_andres_y_providencia_sld_xml) + self.assertEqual(san_andres_y_providencia_sld_name, "san_andres_y_providencia_administrative") # Test 'lac.sld' lac_sld_file = os.path.join(d, "lac.sld") with open(lac_sld_file) as lac_sld_xml_file: lac_sld_xml = lac_sld_xml_file.read() - lac_sld_name = extract_name_from_sld( - None, lac_sld_xml, sld_file=lac_sld_file) - self.assertEqual(lac_sld_name, 'LAC NonIndigenous Access to Sanitation') + lac_sld_name = extract_name_from_sld(None, lac_sld_xml, sld_file=lac_sld_file) + self.assertEqual(lac_sld_name, "LAC NonIndigenous Access to Sanitation") # Test 'freshgwabs2.sld' freshgwabs2_sld_file = os.path.join(d, "freshgwabs2.sld") with open(freshgwabs2_sld_file) as freshgwabs2_sld_xml_file: freshgwabs2_sld_xml = freshgwabs2_sld_xml_file.read() - freshgwabs2_sld_name = extract_name_from_sld( - None, freshgwabs2_sld_xml, sld_file=freshgwabs2_sld_file) - self.assertEqual(freshgwabs2_sld_name, 'freshgwabs2') + freshgwabs2_sld_name = extract_name_from_sld(None, freshgwabs2_sld_xml, sld_file=freshgwabs2_sld_file) + self.assertEqual(freshgwabs2_sld_name, "freshgwabs2") # Test 'raster.sld' raster_sld_file = os.path.join(d, "raster.sld") with open(raster_sld_file) as raster_sld_xml_file: raster_sld_xml = raster_sld_xml_file.read() - raster_sld_name = extract_name_from_sld( - None, raster_sld_xml, sld_file=raster_sld_file) - self.assertEqual( - raster_sld_name, - 'geonode-geonode_gwpollriskafriotest') + raster_sld_name = extract_name_from_sld(None, raster_sld_xml, sld_file=raster_sld_file) + self.assertEqual(raster_sld_name, "geonode-geonode_gwpollriskafriotest") # Test 'line.sld' line_sld_file = os.path.join(d, "line.sld") with open(line_sld_file) as line_sld_xml_file: line_sld_xml = line_sld_xml_file.read() - line_sld_name = extract_name_from_sld( - None, line_sld_xml, sld_file=line_sld_file) - self.assertEqual(line_sld_name, 'line 3') + line_sld_name = extract_name_from_sld(None, line_sld_xml, sld_file=line_sld_file) + self.assertEqual(line_sld_name, "line 3") finally: if d is not None: shutil.rmtree(d, ignore_errors=True) @@ -707,26 +676,26 @@ def test_style_change_on_basic_auth(self): """ layer = Dataset.objects.filter(default_style__isnull=False).first() - bob = get_user_model().objects.get(username='bobby') - assign_perm('change_dataset_style', bob, layer) + bob = get_user_model().objects.get(username="bobby") + assign_perm("change_dataset_style", bob, layer) - self.assertTrue(bob.has_perm('change_dataset_style', obj=layer)) + self.assertTrue(bob.has_perm("change_dataset_style", obj=layer)) # Test that HTTP_AUTHORIZATION in request.META is working properly valid_uname_pw = b"bobby:bob" invalid_uname_pw = b"n0t:v@l1d" valid_auth_headers = { - 'HTTP_AUTHORIZATION': f"BASIC {base64.b64encode(valid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"BASIC {base64.b64encode(valid_uname_pw).decode()}", } invalid_auth_headers = { - 'HTTP_AUTHORIZATION': f"BASIC {base64.b64encode(invalid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"BASIC {base64.b64encode(invalid_uname_pw).decode()}", } change_style_url = urljoin( - settings.SITEURL, - f"/gs/rest/workspaces/{settings.DEFAULT_WORKSPACE}/styles/{layer.name}?raw=true") + settings.SITEURL, f"/gs/rest/workspaces/{settings.DEFAULT_WORKSPACE}/styles/{layer.name}?raw=true" + ) logger.debug(f"{change_style_url}") rf = RequestFactory() @@ -735,50 +704,65 @@ def test_style_change_on_basic_auth(self): post_request = rf.post( change_style_url, data=san_andres_y_providencia_sld, - content_type='application/vnd.ogc.sld+xml', - **valid_auth_headers + content_type="application/vnd.ogc.sld+xml", + **valid_auth_headers, ) post_request.user = AnonymousUser() raw_url, headers, access_token, downstream_path = check_geoserver_access( post_request, - '/gs/rest/workspaces', - 'rest/workspaces', - workspace='geonode', + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", layername=layer.name, - allowed_hosts=[urlsplit(ogc_server_settings.public_url).hostname, ]) + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) self.assertIsNotNone(raw_url) self.assertIsNotNone(headers) self.assertIsNotNone(access_token) - authorized = style_change_check(post_request, downstream_path, style_name='styles', access_token=access_token) + authorized = style_change_check(post_request, downstream_path, style_name="styles", access_token=access_token) self.assertTrue(authorized) - authorized = style_change_check(post_request, 'rest/styles', style_name=f'{layer.name}', access_token=access_token) + authorized = style_change_check( + post_request, "rest/styles", style_name=f"{layer.name}", access_token=access_token + ) self.assertTrue(authorized) - authorized = style_change_check(post_request, f'rest/workspaces/{layer.workspace}/styles/{layer.name}', style_name=f'{layer.name}', access_token=access_token) + authorized = style_change_check( + post_request, + f"rest/workspaces/{layer.workspace}/styles/{layer.name}", + style_name=f"{layer.name}", + access_token=access_token, + ) self.assertTrue(authorized) - authorized = style_change_check(post_request, f'rest/layers/{layer.name}', access_token=access_token) + authorized = style_change_check(post_request, f"rest/layers/{layer.name}", access_token=access_token) self.assertTrue(authorized) - authorized = style_change_check(post_request, f'rest/workspaces/{layer.workspace}/layers/{layer.name}', access_token=access_token) + authorized = style_change_check( + post_request, f"rest/workspaces/{layer.workspace}/layers/{layer.name}", access_token=access_token + ) self.assertTrue(authorized) put_request = rf.put( change_style_url, data=san_andres_y_providencia_sld, - content_type='application/vnd.ogc.sld+xml', - **valid_auth_headers + content_type="application/vnd.ogc.sld+xml", + **valid_auth_headers, ) put_request.user = AnonymousUser() raw_url, headers, access_token, downstream_path = check_geoserver_access( put_request, - '/gs/rest/workspaces', - 'rest/workspaces', - workspace='geonode', + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", layername=layer.name, - allowed_hosts=[urlsplit(ogc_server_settings.public_url).hostname, ]) + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) self.assertIsNotNone(raw_url) self.assertIsNotNone(headers) self.assertIsNotNone(access_token) @@ -789,36 +773,48 @@ def test_style_change_on_basic_auth(self): # [Regression] "style_change_check" always fails in the case the style does not exist on GeoNode too, preventing a user editing temporary generated styles Style.objects.filter(name=layer.name).delete() - authorized = style_change_check(put_request, downstream_path, style_name='styles', access_token=access_token) + authorized = style_change_check(put_request, downstream_path, style_name="styles", access_token=access_token) self.assertTrue(authorized) - authorized = style_change_check(put_request, 'rest/styles', style_name=f'{layer.name}', access_token=access_token) + authorized = style_change_check( + put_request, "rest/styles", style_name=f"{layer.name}", access_token=access_token + ) self.assertTrue(authorized) - authorized = style_change_check(put_request, f'rest/workspaces/{layer.workspace}/styles/{layer.name}', style_name=f'{layer.name}', access_token=access_token) + authorized = style_change_check( + put_request, + f"rest/workspaces/{layer.workspace}/styles/{layer.name}", + style_name=f"{layer.name}", + access_token=access_token, + ) self.assertTrue(authorized) - authorized = style_change_check(put_request, f'rest/layers/{layer.name}', access_token=access_token) + authorized = style_change_check(put_request, f"rest/layers/{layer.name}", access_token=access_token) self.assertTrue(authorized) - authorized = style_change_check(put_request, f'rest/workspaces/{layer.workspace}/layers/{layer.name}', access_token=access_token) + authorized = style_change_check( + put_request, f"rest/workspaces/{layer.workspace}/layers/{layer.name}", access_token=access_token + ) self.assertTrue(authorized) # Check is NOT 'authorized' post_request = rf.post( change_style_url, data=san_andres_y_providencia_sld, - content_type='application/vnd.ogc.sld+xml', - **invalid_auth_headers + content_type="application/vnd.ogc.sld+xml", + **invalid_auth_headers, ) post_request.user = AnonymousUser() raw_url, headers, access_token, downstream_path = check_geoserver_access( post_request, - '/gs/rest/workspaces', - 'rest/workspaces', - workspace='geonode', + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", layername=layer.name, - allowed_hosts=[urlsplit(ogc_server_settings.public_url).hostname, ]) + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) self.assertIsNotNone(raw_url) self.assertIsNotNone(headers) self.assertIsNone(access_token) @@ -828,122 +824,113 @@ def test_style_change_on_basic_auth(self): @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_acls(self): - """ Verify that the dataset_acls view is behaving as expected - """ + """Verify that the dataset_acls view is behaving as expected""" # Test that HTTP_AUTHORIZATION in request.META is working properly valid_uname_pw = b"bobby:bob" invalid_uname_pw = b"n0t:v@l1d" valid_auth_headers = { - 'HTTP_AUTHORIZATION': f"basic {base64.b64encode(valid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(valid_uname_pw).decode()}", } invalid_auth_headers = { - 'HTTP_AUTHORIZATION': f"basic {base64.b64encode(invalid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(invalid_uname_pw).decode()}", } - bob = get_user_model().objects.get(username='bobby') - dataset_ca = Dataset.objects.get(alternate='geonode:CA') - assign_perm('change_dataset_data', bob, dataset_ca) + bob = get_user_model().objects.get(username="bobby") + dataset_ca = Dataset.objects.get(alternate="geonode:CA") + assign_perm("change_dataset_data", bob, dataset_ca) # Test that requesting when supplying the geoserver credentials returns # the expected json expected_result = { - 'email': 'bobby@bob.com', - 'fullname': 'bobby', - 'is_anonymous': False, - 'is_superuser': False, - 'name': 'bobby', - 'ro': [ - 'geonode:layer2', - 'geonode:mylayer', - 'geonode:foo', - 'geonode:whatever', - 'geonode:fooey', - 'geonode:quux', - 'geonode:fleem' + "email": "bobby@bob.com", + "fullname": "bobby", + "is_anonymous": False, + "is_superuser": False, + "name": "bobby", + "ro": [ + "geonode:layer2", + "geonode:mylayer", + "geonode:foo", + "geonode:whatever", + "geonode:fooey", + "geonode:quux", + "geonode:fleem", ], - 'rw': ['geonode:CA'] + "rw": ["geonode:CA"], } - response = self.client.get(reverse('dataset_acls'), **valid_auth_headers) + response = self.client.get(reverse("dataset_acls"), **valid_auth_headers) content = response.content if isinstance(content, bytes): - content = content.decode('UTF-8') + content = content.decode("UTF-8") response_json = json.loads(content) # 'ro' and 'rw' are unsorted collections self.assertEqual(sorted(expected_result), sorted(response_json)) # Test that requesting when supplying invalid credentials returns the # appropriate error code - response = self.client.get( - reverse('dataset_acls'), - **invalid_auth_headers) + response = self.client.get(reverse("dataset_acls"), **invalid_auth_headers) self.assertEqual(response.status_code, 401) # Test logging in using Djangos normal auth system - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") # Basic check that the returned content is at least valid json - response = self.client.get(reverse('dataset_acls')) + response = self.client.get(reverse("dataset_acls")) content = response.content if isinstance(content, bytes): - content = content.decode('UTF-8') + content = content.decode("UTF-8") response_json = json.loads(content) - self.assertEqual('admin', response_json['fullname']) - self.assertEqual('ad@m.in', response_json['email']) + self.assertEqual("admin", response_json["fullname"]) + self.assertEqual("ad@m.in", response_json["email"]) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_resolve_user(self): - """Verify that the resolve_user view is behaving as expected - """ + """Verify that the resolve_user view is behaving as expected""" # Test that HTTP_AUTHORIZATION in request.META is working properly valid_uname_pw = b"admin:admin" invalid_uname_pw = b"n0t:v@l1d" valid_auth_headers = { - 'HTTP_AUTHORIZATION': f"basic {base64.b64encode(valid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(valid_uname_pw).decode()}", } invalid_auth_headers = { - 'HTTP_AUTHORIZATION': f"basic {base64.b64encode(invalid_uname_pw).decode()}", + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(invalid_uname_pw).decode()}", } - response = self.client.get( - reverse('dataset_resolve_user'), - **valid_auth_headers) + response = self.client.get(reverse("dataset_resolve_user"), **valid_auth_headers) content = response.content if isinstance(content, bytes): - content = content.decode('UTF-8') + content = content.decode("UTF-8") response_json = json.loads(content) - self.assertEqual({'geoserver': False, - 'superuser': True, - 'user': 'admin', - 'fullname': 'admin', - 'email': 'ad@m.in'}, response_json) + self.assertEqual( + {"geoserver": False, "superuser": True, "user": "admin", "fullname": "admin", "email": "ad@m.in"}, + response_json, + ) # Test that requesting when supplying invalid credentials returns the # appropriate error code - response = self.client.get( - reverse('dataset_acls'), - **invalid_auth_headers) + response = self.client.get(reverse("dataset_acls"), **invalid_auth_headers) self.assertEqual(response.status_code, 401) # Test logging in using Djangos normal auth system - self.client.login(username='admin', password='admin') + self.client.login(username="admin", password="admin") # Basic check that the returned content is at least valid json - response = self.client.get(reverse('dataset_resolve_user')) + response = self.client.get(reverse("dataset_resolve_user")) content = response.content if isinstance(content, bytes): - content = content.decode('UTF-8') + content = content.decode("UTF-8") response_json = json.loads(content) - self.assertEqual('admin', response_json['user']) - self.assertEqual('admin', response_json['fullname']) - self.assertEqual('ad@m.in', response_json['email']) + self.assertEqual("admin", response_json["user"]) + self.assertEqual("admin", response_json["fullname"]) + self.assertEqual("ad@m.in", response_json["email"]) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_ogc_server_settings(self): @@ -953,22 +940,19 @@ def test_ogc_server_settings(self): with override_settings(OGC_SERVER=self.OGC_DEFAULT_SETTINGS, UPLOADER=self.UPLOADER_DEFAULT_SETTINGS): OGC_SERVER = self.OGC_DEFAULT_SETTINGS.copy() - OGC_SERVER.update( - {'PUBLIC_LOCATION': 'http://geoserver:8080/geoserver/'}) + OGC_SERVER.update({"PUBLIC_LOCATION": "http://geoserver:8080/geoserver/"}) - ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] + ogc_settings = OGC_Servers_Handler(OGC_SERVER)["default"] - default = OGC_SERVER.get('default') + default = OGC_SERVER.get("default") self.assertEqual(ogc_settings.server, default) - self.assertEqual(ogc_settings.BACKEND, default.get('BACKEND')) - self.assertEqual(ogc_settings.LOCATION, default.get('LOCATION')) - self.assertEqual( - ogc_settings.PUBLIC_LOCATION, - default.get('PUBLIC_LOCATION')) - self.assertEqual(ogc_settings.USER, default.get('USER')) - self.assertEqual(ogc_settings.PASSWORD, default.get('PASSWORD')) - self.assertEqual(ogc_settings.DATASTORE, '') - self.assertEqual(ogc_settings.credentials, ('admin', 'geoserver')) + self.assertEqual(ogc_settings.BACKEND, default.get("BACKEND")) + self.assertEqual(ogc_settings.LOCATION, default.get("LOCATION")) + self.assertEqual(ogc_settings.PUBLIC_LOCATION, default.get("PUBLIC_LOCATION")) + self.assertEqual(ogc_settings.USER, default.get("USER")) + self.assertEqual(ogc_settings.PASSWORD, default.get("PASSWORD")) + self.assertEqual(ogc_settings.DATASTORE, "") + self.assertEqual(ogc_settings.credentials, ("admin", "geoserver")) self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED) self.assertTrue(ogc_settings.PRINT_NG_ENABLED) self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED) @@ -984,10 +968,10 @@ def test_ogc_server_defaults(self): from django.urls import reverse, resolve from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities - OGC_SERVER = {'default': dict()} + OGC_SERVER = {"default": dict()} - defaults = self.OGC_DEFAULT_SETTINGS.get('default') - ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] + defaults = self.OGC_DEFAULT_SETTINGS.get("default") + ogc_settings = OGC_Servers_Handler(OGC_SERVER)["default"] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest") self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows") @@ -996,33 +980,33 @@ def test_ogc_server_defaults(self): self.assertIsNone(ogc_settings.SFDSDFDSF) # Testing REST endpoints - route = resolve('/gs/rest/layers').route - self.assertEqual(route, '^gs/rest/layers') - - route = resolve('/gs/rest/imports').route - self.assertEqual(route, '^gs/rest/imports') - - route = resolve('/gs/rest/sldservice').route - self.assertEqual(route, '^gs/rest/sldservice') - - store_resolver = resolve('/gs/rest/stores/geonode_data/') - self.assertEqual(store_resolver.url_name, 'gs_stores') - self.assertEqual(store_resolver.kwargs['store_type'], 'geonode_data') - self.assertEqual(store_resolver.route, '^gs/rest/stores/(?P\\w+)/$') - - sld_resolver = resolve('/gs/rest/styles') - self.assertEqual(sld_resolver.url_name, 'gs_styles') - self.assertTrue('workspace' not in sld_resolver.kwargs) - self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/styles') - self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/styles') - self.assertEqual(sld_resolver.route, '^gs/rest/styles') - - sld_resolver = resolve('/gs/rest/workspaces/geonode/styles') - self.assertEqual(sld_resolver.url_name, 'gs_workspaces') - self.assertEqual(sld_resolver.kwargs['workspace'], 'geonode') - self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/workspaces') - self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/workspaces') - self.assertEqual(sld_resolver.route, '^gs/rest/workspaces/(?P\\w+)') + route = resolve("/gs/rest/layers").route + self.assertEqual(route, "^gs/rest/layers") + + route = resolve("/gs/rest/imports").route + self.assertEqual(route, "^gs/rest/imports") + + route = resolve("/gs/rest/sldservice").route + self.assertEqual(route, "^gs/rest/sldservice") + + store_resolver = resolve("/gs/rest/stores/geonode_data/") + self.assertEqual(store_resolver.url_name, "gs_stores") + self.assertEqual(store_resolver.kwargs["store_type"], "geonode_data") + self.assertEqual(store_resolver.route, "^gs/rest/stores/(?P\\w+)/$") + + sld_resolver = resolve("/gs/rest/styles") + self.assertEqual(sld_resolver.url_name, "gs_styles") + self.assertTrue("workspace" not in sld_resolver.kwargs) + self.assertEqual(sld_resolver.kwargs["proxy_path"], "/gs/rest/styles") + self.assertEqual(sld_resolver.kwargs["downstream_path"], "rest/styles") + self.assertEqual(sld_resolver.route, "^gs/rest/styles") + + sld_resolver = resolve("/gs/rest/workspaces/geonode/styles") + self.assertEqual(sld_resolver.url_name, "gs_workspaces") + self.assertEqual(sld_resolver.kwargs["workspace"], "geonode") + self.assertEqual(sld_resolver.kwargs["proxy_path"], "/gs/rest/workspaces") + self.assertEqual(sld_resolver.kwargs["downstream_path"], "rest/workspaces") + self.assertEqual(sld_resolver.route, "^gs/rest/workspaces/(?P\\w+)") # Testing OWS endpoints wcs = _wcs_get_capabilities() @@ -1030,9 +1014,9 @@ def test_ogc_server_defaults(self): self.assertIsNotNone(wcs) try: - wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") self.assertTrue(wcs.startswith(wcs_url)) self.assertIn("service=WCS", wcs) @@ -1044,9 +1028,9 @@ def test_ogc_server_defaults(self): self.assertIsNotNone(wfs) try: - wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wfs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") self.assertTrue(wfs.startswith(wfs_url)) self.assertIn("service=WFS", wfs) self.assertIn("request=GetCapabilities", wfs) @@ -1057,9 +1041,9 @@ def test_ogc_server_defaults(self): self.assertIsNotNone(wms) try: - wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) + wms_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) except Exception: - wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") self.assertTrue(wms.startswith(wms_url)) self.assertIn("service=WMS", wms) self.assertIn("request=GetCapabilities", wms) @@ -1067,8 +1051,9 @@ def test_ogc_server_defaults(self): # Test OWS Download Links from geonode.geoserver.ows import wcs_links, wfs_links, wms_links + instance = create_single_dataset("san_andres_y_providencia_water") - instance.name = 'san_andres_y_providencia_water' + instance.name = "san_andres_y_providencia_water" instance.save() bbox = instance.bbox srid = instance.srid @@ -1081,7 +1066,7 @@ def test_ogc_server_defaults(self): style = gs_catalog.get_style(instance.name, workspace=instance.workspace) self.assertIsNotNone(style) self.assertFalse(isinstance(style, str)) - style_name = 'point' + style_name = "point" style_body = """ """ instance.default_style, _ = Style.objects.get_or_create( - name=style_name, - defaults=dict( - sld_title=style_name, - sld_body=style_body - ) + name=style_name, defaults=dict(sld_title=style_name, sld_body=style_body) ) self.assertIsNotNone(instance.default_style) self.assertIsNotNone(instance.default_style.name) # WMS Links - wms_links = wms_links(f"{ogc_settings.public_url}wms?", - instance.alternate, - bbox, - srid, - height, - width) + wms_links = wms_links(f"{ogc_settings.public_url}wms?", instance.alternate, bbox, srid, height, width) self.assertIsNotNone(wms_links) self.assertEqual(len(wms_links), 3) - wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms') - identifier = urlencode({'layers': instance.alternate}) + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wms") + identifier = urlencode({"layers": instance.alternate}) for _link in wms_links: - logger.debug(f'{wms_url} --> {_link[3]}') + logger.debug(f"{wms_url} --> {_link[3]}") self.assertTrue(wms_url in _link[3]) - logger.debug(f'{identifier} --> {_link[3]}') + logger.debug(f"{identifier} --> {_link[3]}") self.assertTrue(identifier in _link[3]) # WFS Links - wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?", - instance.alternate, - bbox, - srid) + wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?", instance.alternate, bbox, srid) self.assertIsNotNone(wfs_links) self.assertEqual(len(wfs_links), 6) - wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs') - identifier = urlencode({'typename': instance.alternate}) + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wfs") + identifier = urlencode({"typename": instance.alternate}) for _link in wfs_links: - logger.debug(f'{wfs_url} --> {_link[3]}') + logger.debug(f"{wfs_url} --> {_link[3]}") self.assertTrue(wfs_url in _link[3]) - logger.debug(f'{identifier} --> {_link[3]}') + logger.debug(f"{identifier} --> {_link[3]}") self.assertTrue(identifier in _link[3]) # WCS Links - wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?", - instance.alternate) + wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?", instance.alternate) self.assertIsNotNone(wcs_links) self.assertEqual(len(wcs_links), 1) - wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs') - identifier = urlencode({'coverageid': instance.alternate.replace(':', '__', 1)}) + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wcs") + identifier = urlencode({"coverageid": instance.alternate.replace(":", "__", 1)}) for _link in wcs_links: - logger.debug(f'{wcs_url} --> {_link[3]}') + logger.debug(f"{wcs_url} --> {_link[3]}") self.assertTrue(wcs_url in _link[3]) - logger.debug(f'{identifier} --> {_link[3]}') + logger.debug(f"{identifier} --> {_link[3]}") self.assertTrue(identifier in _link[3]) if srid: - self.assertFalse('outputCrs' in _link[3]) + self.assertFalse("outputCrs" in _link[3]) if bbox: - self.assertFalse('subset=Long' in _link[3]) - self.assertFalse('subset=Lat' in _link[3]) + self.assertFalse("subset=Long" in _link[3]) + self.assertFalse("subset=Lat" in _link[3]) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_importer_configuration(self): @@ -1187,30 +1159,28 @@ def test_importer_configuration(self): ogc_server_settings = self.OGC_DEFAULT_SETTINGS.copy() uploader_settings = self.UPLOADER_DEFAULT_SETTINGS.copy() - uploader_settings['BACKEND'] = 'geonode.importer' - self.assertTrue(['geonode_imports' not in database_settings.keys()]) + uploader_settings["BACKEND"] = "geonode.importer" + self.assertTrue(["geonode_imports" not in database_settings.keys()]) # Test the importer backend without specifying a datastore or # corresponding database. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): - OGC_Servers_Handler(ogc_server_settings)['default'] + OGC_Servers_Handler(ogc_server_settings)["default"] - ogc_server_settings['default']['DATASTORE'] = 'geonode_imports' + ogc_server_settings["default"]["DATASTORE"] = "geonode_imports" # Test the importer backend with a datastore but no corresponding # database. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): - OGC_Servers_Handler(ogc_server_settings)['default'] + OGC_Servers_Handler(ogc_server_settings)["default"] - database_settings['geonode_imports'] = database_settings[ - 'default'].copy() - database_settings['geonode_imports'].update( - {'NAME': 'geonode_imports'}) + database_settings["geonode_imports"] = database_settings["default"].copy() + database_settings["geonode_imports"].update({"NAME": "geonode_imports"}) # Test the importer backend with a datastore and a corresponding # database, no exceptions should be thrown. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): - OGC_Servers_Handler(ogc_server_settings)['default'] + OGC_Servers_Handler(ogc_server_settings)["default"] @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_set_resources_links(self): @@ -1221,100 +1191,89 @@ def test_set_resources_links(self): with self.settings(UPDATE_RESOURCE_LINKS_AT_MIGRATE=True, ASYNC_SIGNALS=False): # Links - _def_link_types = ['original', 'metadata'] + _def_link_types = ["original", "metadata"] _links = Link.objects.filter(link_type__in=_def_link_types) # Check 'original' and 'metadata' links exist - self.assertIsNotNone( - _links, - "No 'original' and 'metadata' links have been found" - ) + self.assertIsNotNone(_links, "No 'original' and 'metadata' links have been found") # Delete all 'original' and 'metadata' links _links.delete() self.assertFalse(_links.exists(), "No links have been deleted") # Delete resources metadata _datasets = Dataset.objects.exclude( - Q(metadata_xml__isnull=True) | - Q(metadata_xml__exact='') | - Q(csw_anytext__isnull=True) | - Q(csw_anytext__exact='') + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") ) count = _datasets.count() if count: _datasets.update(metadata_xml=None) _updated_datasets = Dataset.objects.exclude( - Q(metadata_xml__isnull=True) | - Q(metadata_xml__exact='') | - Q(csw_anytext__isnull=True) | - Q(csw_anytext__exact='') + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") ) updated_count = _updated_datasets.count() - self.assertTrue( - updated_count == 0, - "Metadata have not been updated (deleted) correctly" - ) + self.assertTrue(updated_count == 0, "Metadata have not been updated (deleted) correctly") # Call migrate call_command("migrate", verbosity=0) # Check links _post_migrate_links = Link.objects.filter(link_type__in=_def_link_types) - self.assertTrue( - _post_migrate_links.exists(), - "No links have been restored" - ) + self.assertTrue(_post_migrate_links.exists(), "No links have been restored") # Check layers _post_migrate_datasets = Dataset.objects.exclude( - Q(metadata_xml__isnull=True) | - Q(metadata_xml__exact='') | - Q(csw_anytext__isnull=True) | - Q(csw_anytext__exact='') + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") ) for _lyr in _post_migrate_datasets: # Check original links in csw_anytext _post_migrate_links_orig = Link.objects.filter( - resource=_lyr.resourcebase_ptr, - resource_id=_lyr.resourcebase_ptr.id, - link_type='original' + resource=_lyr.resourcebase_ptr, resource_id=_lyr.resourcebase_ptr.id, link_type="original" ) self.assertTrue( _post_migrate_links_orig.exists(), - f"No 'original' links has been found for the layer '{_lyr.alternate}'" + f"No 'original' links has been found for the layer '{_lyr.alternate}'", ) for _link_orig in _post_migrate_links_orig: self.assertIn( _link_orig.url, _lyr.csw_anytext, - f"The link URL {_link_orig.url} is not present in the 'csw_anytext' attribute of the layer '{_lyr.alternate}'" + f"The link URL {_link_orig.url} is not present in the 'csw_anytext' attribute of the layer '{_lyr.alternate}'", ) # Check catalogue catalogue = get_catalogue() record = catalogue.get_record(_lyr.uuid) self.assertIsNotNone(record) self.assertTrue( - hasattr(record, 'links'), - f"No records have been found in the catalogue for the resource '{_lyr.alternate}'" + hasattr(record, "links"), + f"No records have been found in the catalogue for the resource '{_lyr.alternate}'", ) # Check 'metadata' links for each record - for mime, name, metadata_url in record.links['metadata']: + for mime, name, metadata_url in record.links["metadata"]: try: _post_migrate_link_meta = Link.objects.get( resource=_lyr.resourcebase_ptr, url=metadata_url, name=name, - extension='xml', + extension="xml", mime=mime, - link_type='metadata' + link_type="metadata", ) except Link.DoesNotExist: _post_migrate_link_meta = None self.assertIsNotNone( _post_migrate_link_meta, - f"No '{name}' links have been found in the catalogue for the resource '{_lyr.alternate}'" + f"No '{name}' links have been found in the catalogue for the resource '{_lyr.alternate}'", ) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_gs_proxy_never_caches(self): - url = reverse('gs_styles') + url = reverse("gs_styles") response = self.client.get(url) - self.assertFalse(response.has_header('Cache-Control')) + self.assertFalse(response.has_header("Cache-Control")) diff --git a/geonode/geoserver/tests/test_tasks.py b/geonode/geoserver/tests/test_tasks.py index a091a541328..ea2fd44430c 100644 --- a/geonode/geoserver/tests/test_tasks.py +++ b/geonode/geoserver/tests/test_tasks.py @@ -10,13 +10,9 @@ class TasksTest(GeoNodeBaseTestSupport): - type = 'dataset' + type = "dataset" - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] @classmethod def setUpClass(cls): @@ -42,7 +38,7 @@ def test_geoserver_style_visual_mode_automatically_with_sld_file(self): handler = create_autospec(self.mock_signal_callback) geoserver_automatic_default_style_set.connect(handler) - with patch('geoserver.catalog.Catalog.get_style') as style_mck: + with patch("geoserver.catalog.Catalog.get_style") as style_mck: style_mck.return_value = True geoserver_create_style(dataset.id, dataset.name, sld_file=sld_file, tempdir=None) self.assertEqual(handler.call_count, 0) @@ -59,10 +55,7 @@ def test_geoserver_style_visual_mode_automatically_without_sld_file(self): def test_geoserver_set_style_with_real_file(self, mocked_set_dataset_style): dataset = Dataset.objects.first() sld_file = "geonode/base/fixtures/test_sld.sld" - geoserver_set_style( - instance_id=dataset.id, - base_file=sld_file - ) + geoserver_set_style(instance_id=dataset.id, base_file=sld_file) mocked_set_dataset_style.assert_called_once() args_list = mocked_set_dataset_style.call_args_list[0].args @@ -77,11 +70,8 @@ def test_geoserver_set_style_with_real_file(self, mocked_set_dataset_style): def test_geoserver_set_style_with_xml(self, mocked_set_dataset_style): dataset = Dataset.objects.first() - with open("geonode/base/fixtures/test_sld.sld", 'r+') as _file: - geoserver_set_style( - instance_id=dataset.id, - base_file=_file.read() - ) + with open("geonode/base/fixtures/test_sld.sld", "r+") as _file: + geoserver_set_style(instance_id=dataset.id, base_file=_file.read()) mocked_set_dataset_style.assert_called_once() args_list = mocked_set_dataset_style.call_args_list[0].args diff --git a/geonode/geoserver/upload.py b/geonode/geoserver/upload.py index 580826fe432..2c50bb1222c 100644 --- a/geonode/geoserver/upload.py +++ b/geonode/geoserver/upload.py @@ -35,7 +35,8 @@ ogc_server_settings, _create_db_featurestore, _create_featurestore, - _create_coveragestore) + _create_coveragestore, +) logger = logging.getLogger(__name__) @@ -46,21 +47,24 @@ def geoserver_dataset_type(filename): def geoserver_upload( - dataset, - base_file, - user, - name, - overwrite=True, - title=None, - abstract=None, - permissions=None, - keywords=(), - charset='UTF-8'): + dataset, + base_file, + user, + name, + overwrite=True, + title=None, + abstract=None, + permissions=None, + keywords=(), + charset="UTF-8", +): # Step 2. Check that it is uploading to the same resource type as # the existing resource - logger.debug('>>> Step 2. Make sure we are not trying to overwrite a ' - 'existing resource named [%s] with the wrong type', name) + logger.debug( + ">>> Step 2. Make sure we are not trying to overwrite a " "existing resource named [%s] with the wrong type", + name, + ) the_dataset_type = geoserver_dataset_type(base_file) # Get a short handle to the gsconfig geoserver catalog @@ -83,22 +87,23 @@ def geoserver_upload( # to have the right resource type for resource in resources: if resource.name == name: - msg = 'Name already in use and overwrite is False' + msg = "Name already in use and overwrite is False" assert overwrite, msg existing_type = resource.resource_type if existing_type != the_dataset_type: - msg = (f'Type of uploaded file {name} ({the_dataset_type}) ' - 'does not match type of existing ' - f'resource type {existing_type}') + msg = ( + f"Type of uploaded file {name} ({the_dataset_type}) " + "does not match type of existing " + f"resource type {existing_type}" + ) logger.debug(msg) raise GeoNodeException(msg) # Step 3. Identify whether it is vector or raster and which extra files # are needed. - logger.debug('>>> Step 3. Identifying if [%s] is vector or raster and ' - 'gathering extra files', name) + logger.debug(">>> Step 3. Identifying if [%s] is vector or raster and " "gathering extra files", name) if the_dataset_type == FeatureType.resource_type: - logger.debug('Uploading vector layer: [%s]', base_file) + logger.debug("Uploading vector layer: [%s]", base_file) if ogc_server_settings.DATASTORE: create_store_and_resource = _create_db_featurestore else: @@ -107,38 +112,38 @@ def geoserver_upload( logger.debug("Uploading raster layer: [%s]", base_file) create_store_and_resource = _create_coveragestore else: - msg = (f'The layer type for name {name} is {the_dataset_type}. It should be ' - f'{FeatureType.resource_type} or {Coverage.resource_type},') + msg = ( + f"The layer type for name {name} is {the_dataset_type}. It should be " + f"{FeatureType.resource_type} or {Coverage.resource_type}," + ) logger.warn(msg) raise GeoNodeException(msg) # Step 4. Create the store in GeoServer - logger.debug('>>> Step 4. Starting upload of [%s] to GeoServer...', name) + logger.debug(">>> Step 4. Starting upload of [%s] to GeoServer...", name) # Get the helper files if they exist files, _tmpdir = get_files(base_file) data = files - if 'shp' not in files: + if "shp" not in files: data = base_file try: store, gs_resource = create_store_and_resource( - name, - data, - charset=charset, - overwrite=overwrite, - workspace=workspace) + name, data, charset=charset, overwrite=overwrite, workspace=workspace + ) except UploadError as e: - msg = (f'Could not save the layer {name}, there was an upload ' - f'error: {e}') + msg = f"Could not save the layer {name}, there was an upload " f"error: {e}" logger.warn(msg) e.args = (msg,) raise except ConflictingDataError as e: # A datastore of this name already exists - msg = (f'GeoServer reported a conflict creating a store with name {name}: ' - f'"{e}". This should never happen because a brand new name ' - 'should have been generated. But since it happened, ' - 'try renaming the file or deleting the store in GeoServer.') + msg = ( + f"GeoServer reported a conflict creating a store with name {name}: " + f'"{e}". This should never happen because a brand new name ' + "should have been generated. But since it happened, " + "try renaming the file or deleting the store in GeoServer." + ) logger.warn(msg) e.args = (msg,) raise @@ -146,55 +151,56 @@ def geoserver_upload( logger.error("Error during the creation of the resource in GeoServer", exc_info=e) raise e - logger.debug(f'The File {name} has been sent to GeoServer without errors.') + logger.debug(f"The File {name} has been sent to GeoServer without errors.") # Step 5. Create the resource in GeoServer - logger.debug(f'>>> Step 5. Generating the metadata for {name} after successful import to GeoSever') + logger.debug(f">>> Step 5. Generating the metadata for {name} after successful import to GeoSever") # Verify the resource was created if not gs_resource: - gs_resource = gs_catalog.get_resource( - name=name, - workspace=workspace) + gs_resource = gs_catalog.get_resource(name=name, workspace=workspace) if not gs_resource: - msg = f'GeoNode encountered problems when creating layer {name}.It cannot find the Dataset that matches this Workspace.try renaming your files.' + msg = f"GeoNode encountered problems when creating layer {name}.It cannot find the Dataset that matches this Workspace.try renaming your files." logger.warn(msg) raise GeoNodeException(msg) assert gs_resource.name == name # Step 6. Make sure our data always has a valid projection - logger.debug(f'>>> Step 6. Making sure [{name}] has a valid projection') + logger.debug(f">>> Step 6. Making sure [{name}] has a valid projection") _native_bbox = None try: _native_bbox = gs_resource.native_bbox except Exception: pass - if _native_bbox and len(_native_bbox) >= 5 and _native_bbox[4:5][0] == 'EPSG:4326': + if _native_bbox and len(_native_bbox) >= 5 and _native_bbox[4:5][0] == "EPSG:4326": box = _native_bbox[:4] minx, maxx, miny, maxy = [float(a) for a in box] - if -180 <= round(minx, 5) <= 180 and -180 <= round(maxx, 5) <= 180 and \ - -90 <= round(miny, 5) <= 90 and -90 <= round(maxy, 5) <= 90: + if ( + -180 <= round(minx, 5) <= 180 + and -180 <= round(maxx, 5) <= 180 + and -90 <= round(miny, 5) <= 90 + and -90 <= round(maxy, 5) <= 90 + ): gs_resource.latlon_bbox = _native_bbox gs_resource.projection = "EPSG:4326" else: - logger.warning('BBOX coordinates outside normal EPSG:4326 values for layer ' - '[%s].', name) + logger.warning("BBOX coordinates outside normal EPSG:4326 values for layer " "[%s].", name) _native_bbox = [-180, -90, 180, 90, "EPSG:4326"] gs_resource.latlon_bbox = _native_bbox gs_resource.projection = "EPSG:4326" - logger.debug('BBOX coordinates forced to [-180, -90, 180, 90] for layer [%s].', name) + logger.debug("BBOX coordinates forced to [-180, -90, 180, 90] for layer [%s].", name) # Step 7. Create the style and assign it to the created resource - logger.debug(f'>>> Step 7. Creating style for [{name}]') + logger.debug(f">>> Step 7. Creating style for [{name}]") cat.save(gs_resource) publishing = cat.get_layer(name) or gs_resource sld = None try: - if 'sld' in files: - with open(files['sld'], 'rb') as f: + if "sld" in files: + with open(files["sld"], "rb") as f: sld = f.read() else: sld = get_sld_for(cat, dataset) @@ -213,13 +219,11 @@ def geoserver_upload( cat.create_style(name, sld, overwrite=overwrite, raw=True, workspace=workspace) cat.reset() except geoserver.catalog.ConflictingDataError as e: - msg = (f'There was already a style named {name}_dataset in GeoServer, ' - f'try to use: "{e}"') + msg = f"There was already a style named {name}_dataset in GeoServer, " f'try to use: "{e}"' logger.warn(msg) e.args = (msg,) except geoserver.catalog.UploadError as e: - msg = (f'Error while trying to upload style named {name}_dataset in GeoServer, ' - f'try to use: "{e}"') + msg = f"Error while trying to upload style named {name}_dataset in GeoServer, " f'try to use: "{e}"' e.args = (msg,) logger.exception(e) @@ -227,33 +231,34 @@ def geoserver_upload( try: style = cat.get_style(name, workspace=workspace) or cat.get_style(name) except Exception as e: - style = cat.get_style('point') + style = cat.get_style("point") msg = f'Could not find any suitable style in GeoServer for Dataset: "{name}"' e.args = (msg,) logger.exception(e) if style: publishing.default_style = style - logger.debug('default style set to %s', name) + logger.debug("default style set to %s", name) try: cat.save(publishing) except geoserver.catalog.FailedRequestError as e: - msg = (f'Error while trying to save resource named {publishing} in GeoServer, ' - f'try to use: "{e}"') + msg = f"Error while trying to save resource named {publishing} in GeoServer, " f'try to use: "{e}"' e.args = (msg,) logger.exception(e) # Step 8. Create the Django record for the layer - logger.debug('>>> Step 8. Creating Django record for [%s]', name) + logger.debug(">>> Step 8. Creating Django record for [%s]", name) alternate = f"{workspace.name}:{gs_resource.name}" dataset_uuid = str(uuid.uuid4()) - defaults = dict(store=gs_resource.store.name, - subtype=gs_resource.store.resource_type, - alternate=alternate, - title=title or gs_resource.title, - uuid=dataset_uuid, - abstract=abstract or gs_resource.abstract or '', - owner=user) + defaults = dict( + store=gs_resource.store.name, + subtype=gs_resource.store.resource_type, + alternate=alternate, + title=title or gs_resource.title, + uuid=dataset_uuid, + abstract=abstract or gs_resource.abstract or "", + owner=user, + ) return name, workspace.name, defaults, gs_resource diff --git a/geonode/geoserver/urls.py b/geonode/geoserver/urls.py index 709315d4ac3..963854b881c 100644 --- a/geonode/geoserver/urls.py +++ b/geonode/geoserver/urls.py @@ -23,56 +23,70 @@ urlpatterns = [ # 'geonode.geoserver.views', # REST Endpoints - url(r'^rest/stores/(?P\w+)/$', views.stores, name="gs_stores"), - url(r'^rest/styles', views.geoserver_proxy, dict(proxy_path='/gs/rest/styles', - downstream_path='rest/styles'), name="gs_styles"), - url(r'^rest/workspaces/(?P\w+)', views.geoserver_proxy, dict(proxy_path='/gs/rest/workspaces', - downstream_path='rest/workspaces'), name="gs_workspaces"), - url(r'^rest/layers', views.geoserver_proxy, dict(proxy_path='/gs/rest/layers', - downstream_path='rest/layers'), name="gs_layers"), - url(r'^rest/imports', views.geoserver_proxy, dict(proxy_path='/gs/rest/imports', - downstream_path='rest/imports'), name="gs_imports"), - url(r'^rest/sldservice', views.geoserver_proxy, dict(proxy_path='/gs/rest/sldservice', - downstream_path='rest/sldservice'), name="gs_sldservice"), - + url(r"^rest/stores/(?P\w+)/$", views.stores, name="gs_stores"), + url( + r"^rest/styles", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/styles", downstream_path="rest/styles"), + name="gs_styles", + ), + url( + r"^rest/workspaces/(?P\w+)", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/workspaces", downstream_path="rest/workspaces"), + name="gs_workspaces", + ), + url( + r"^rest/layers", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/layers", downstream_path="rest/layers"), + name="gs_layers", + ), + url( + r"^rest/imports", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/imports", downstream_path="rest/imports"), + name="gs_imports", + ), + url( + r"^rest/sldservice", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/sldservice", downstream_path="rest/sldservice"), + name="gs_sldservice", + ), # OWS Endpoints - url(r'^ows', views.geoserver_proxy, dict(proxy_path='/gs/ows', - downstream_path='ows'), name='ows_endpoint'), - url(r'^gwc', views.geoserver_proxy, dict(proxy_path='/gs/gwc', - downstream_path='gwc'), name='gwc_endpoint'), - url(r'^wms', views.geoserver_proxy, dict(proxy_path='/gs/wms', - downstream_path='wms'), name='wms_endpoint'), - url(r'^wfs', views.geoserver_proxy, dict(proxy_path='/gs/wfs', - downstream_path='wfs'), name='wfs_endpoint'), - url(r'^wcs', views.geoserver_proxy, dict(proxy_path='/gs/wcs', - downstream_path='wcs'), name='wcs_endpoint'), - url(r'^wps', views.geoserver_proxy, dict(proxy_path='/gs/wps', - downstream_path='wps'), name='wps_endpoint'), - url(r'^pdf', views.geoserver_proxy, dict(proxy_path='/gs/pdf', - downstream_path='pdf'), name='pdf_endpoint'), - url(r'^(?P[^/]*)/(?P[^/]*)/ows', + url(r"^ows", views.geoserver_proxy, dict(proxy_path="/gs/ows", downstream_path="ows"), name="ows_endpoint"), + url(r"^gwc", views.geoserver_proxy, dict(proxy_path="/gs/gwc", downstream_path="gwc"), name="gwc_endpoint"), + url(r"^wms", views.geoserver_proxy, dict(proxy_path="/gs/wms", downstream_path="wms"), name="wms_endpoint"), + url(r"^wfs", views.geoserver_proxy, dict(proxy_path="/gs/wfs", downstream_path="wfs"), name="wfs_endpoint"), + url(r"^wcs", views.geoserver_proxy, dict(proxy_path="/gs/wcs", downstream_path="wcs"), name="wcs_endpoint"), + url(r"^wps", views.geoserver_proxy, dict(proxy_path="/gs/wps", downstream_path="wps"), name="wps_endpoint"), + url(r"^pdf", views.geoserver_proxy, dict(proxy_path="/gs/pdf", downstream_path="pdf"), name="pdf_endpoint"), + url( + r"^(?P[^/]*)/(?P[^/]*)/ows", views.geoserver_proxy, - dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='ows')), - url(r'^(?P[^/]*)/(?P[^/]*)/wms', + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="ows"), + ), + url( + r"^(?P[^/]*)/(?P[^/]*)/wms", views.geoserver_proxy, - dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wms')), - url(r'^(?P[^/]*)/(?P[^/]*)/wfs', + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wms"), + ), + url( + r"^(?P[^/]*)/(?P[^/]*)/wfs", views.geoserver_proxy, - dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wfs')), - url(r'^(?P[^/]*)/(?P[^/]*)/wcs', + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wfs"), + ), + url( + r"^(?P[^/]*)/(?P[^/]*)/wcs", views.geoserver_proxy, - dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wcs')), - - url(r'^updatelayers/$', - views.updatelayers, name="updatelayers"), - url(r'^(?P[^/]*)/style$', - views.dataset_style, name="dataset_style"), - url(r'^(?P[^/]*)/style/upload$', - views.dataset_style_upload, name='dataset_style_upload'), - url(r'^(?P[^/]*)/style/manage$', - views.dataset_style_manage, name='dataset_style_manage'), - url(r'^acls/?$', views.dataset_acls, name='dataset_acls'), - url(r'^resolve_user/?$', views.resolve_user, - name='dataset_resolve_user'), - url(r'^online/?$', views.server_online, name='server_online'), + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wcs"), + ), + url(r"^updatelayers/$", views.updatelayers, name="updatelayers"), + url(r"^(?P[^/]*)/style$", views.dataset_style, name="dataset_style"), + url(r"^(?P[^/]*)/style/upload$", views.dataset_style_upload, name="dataset_style_upload"), + url(r"^(?P[^/]*)/style/manage$", views.dataset_style_manage, name="dataset_style_manage"), + url(r"^acls/?$", views.dataset_acls, name="dataset_acls"), + url(r"^resolve_user/?$", views.resolve_user, name="dataset_resolve_user"), + url(r"^online/?$", views.server_online, name="server_online"), ] diff --git a/geonode/geoserver/views.py b/geonode/geoserver/views.py index ca8d94f86ec..5642dcc75ec 100644 --- a/geonode/geoserver/views.py +++ b/geonode/geoserver/views.py @@ -26,11 +26,7 @@ from owslib.etree import etree as dlxml from os.path import isfile -from urllib.parse import ( - urlsplit, - urljoin, - unquote, - parse_qsl) +from urllib.parse import urlsplit, urljoin, unquote, parse_qsl from django.contrib.auth import authenticate from django.http import HttpResponse, HttpResponseRedirect @@ -55,9 +51,7 @@ from geonode.layers.models import Dataset, Style from geonode.layers.views import _resolve_dataset, _PERMISSION_MSG_MODIFY from geonode.maps.models import Map -from geonode.proxy.views import ( - proxy, - fetch_response_headers) +from geonode.proxy.views import proxy, fetch_response_headers from .tasks import geoserver_update_datasets from geonode.utils import ( json_response, @@ -65,11 +59,10 @@ http_client, safe_path_leaf, get_headers, - get_dataset_workspace) + get_dataset_workspace, +) from geoserver.catalog import FailedRequestError -from geonode.geoserver.signals import ( - gs_catalog, - geoserver_post_save_local) +from geonode.geoserver.signals import gs_catalog, geoserver_post_save_local from .helpers import ( get_stores, ogc_server_settings, @@ -80,7 +73,8 @@ ows_endpoint_in_path, temp_style_name_regex, _stylefilterparams_geowebcache_dataset, - _invalidate_geowebcache_dataset) + _invalidate_geowebcache_dataset, +) from django.views.decorators.csrf import csrf_exempt @@ -98,15 +92,14 @@ def updatelayers(request): params = request.GET # Get the owner specified in the request if any, otherwise used the logged # user - owner = params.get('owner', None) - owner = get_user_model().objects.get( - username=owner) if owner is not None else request.user - workspace = params.get('workspace', None) - store = params.get('store', None) - filter = params.get('filter', None) + owner = params.get("owner", None) + owner = get_user_model().objects.get(username=owner) if owner is not None else request.user + workspace = params.get("workspace", None) + store = params.get("store", None) + filter = params.get("filter", None) result = geoserver_update_datasets.delay( - ignore_errors=False, owner=owner, workspace=workspace, - store=store, filter=filter) + ignore_errors=False, owner=owner, workspace=workspace, store=store, filter=filter + ) # Attempt to run task synchronously result.get() @@ -116,13 +109,9 @@ def updatelayers(request): @login_required @require_POST def dataset_style(request, layername): - layer = _resolve_dataset( - request, - layername, - 'base.change_resourcebase', - _PERMISSION_MSG_MODIFY) + layer = _resolve_dataset(request, layername, "base.change_resourcebase", _PERMISSION_MSG_MODIFY) - style_name = request.POST.get('defaultStyle') + style_name = request.POST.get("defaultStyle") # would be nice to implement # better handling of default style switching @@ -130,8 +119,7 @@ def dataset_style(request, layername): old_default = layer.default_style if old_default.name == style_name: - return HttpResponse( - f"Default style for {layer.name} remains {style_name}", status=200) + return HttpResponse(f"Default style for {layer.name} remains {style_name}", status=200) # This code assumes without checking # that the new default style name is included @@ -141,8 +129,7 @@ def dataset_style(request, layername): # Does this change this in geoserver?? layer.default_style = new_style - layer.styles = [ - s for s in layer.styles if s.name != style_name] + [old_default] + layer.styles = [s for s in layer.styles if s.name != style_name] + [old_default] layer.save(notify=True) # Invalidate GeoWebCache for the updated resource @@ -152,32 +139,28 @@ def dataset_style(request, layername): except Exception: pass - return HttpResponse( - f"Default style for {layer.name} changed to {style_name}", status=200) + return HttpResponse(f"Default style for {layer.name} changed to {style_name}", status=200) @login_required def dataset_style_upload(request, layername): def respond(*args, **kw): - kw['content_type'] = 'text/html' + kw["content_type"] = "text/html" return json_response(*args, **kw) + form = LayerStyleUploadForm(request.POST, request.FILES) if not form.is_valid(): return respond(errors="Please provide an SLD file.") data = form.cleaned_data - layer = _resolve_dataset( - request, - layername, - 'base.change_resourcebase', - _PERMISSION_MSG_MODIFY) + layer = _resolve_dataset(request, layername, "base.change_resourcebase", _PERMISSION_MSG_MODIFY) - sld = request.FILES['sld'].read() + sld = request.FILES["sld"].read() sld_name = None try: # Check SLD is valid try: - _allowed_sld_extensions = ['.sld', '.xml', '.css', '.txt', '.yml'] + _allowed_sld_extensions = [".sld", ".xml", ".css", ".txt", ".yml"] if sld and os.path.splitext(safe_path_leaf(sld))[1].lower() in _allowed_sld_extensions: if isfile(sld): with open(sld) as sld_file: @@ -185,34 +168,24 @@ def respond(*args, **kw): etree.XML(sld, parser=etree.XMLParser(resolve_entities=False)) except Exception: logger.exception("The uploaded SLD file is not valid XML") - raise Exception( - "The uploaded SLD file is not valid XML") + raise Exception("The uploaded SLD file is not valid XML") - sld_name = extract_name_from_sld( - gs_catalog, sld, sld_file=request.FILES['sld']) + sld_name = extract_name_from_sld(gs_catalog, sld, sld_file=request.FILES["sld"]) except Exception as e: respond(errors=f"The uploaded SLD file is not valid XML: {e}") - name = data.get('name') or sld_name + name = data.get("name") or sld_name - set_dataset_style(layer, data.get('title') or name, sld) + set_dataset_style(layer, data.get("title") or name, sld) - return respond( - body={ - 'success': True, - 'style': data.get('title') or name, - 'updated': data['update']}) + return respond(body={"success": True, "style": data.get("title") or name, "updated": data["update"]}) @login_required def dataset_style_manage(request, layername): - layer = _resolve_dataset( - request, - layername, - 'layers.change_dataset_style', - _PERMISSION_MSG_MODIFY) + layer = _resolve_dataset(request, layername, "layers.change_dataset_style", _PERMISSION_MSG_MODIFY) - if request.method == 'GET': + if request.method == "GET": try: cat = gs_catalog @@ -220,14 +193,13 @@ def dataset_style_manage(request, layername): try: set_styles(layer, cat) except AttributeError: - logger.warn( - 'Unable to set the default style. Ensure Geoserver is running and that this layer exists.') + logger.warn("Unable to set the default style. Ensure Geoserver is running and that this layer exists.") gs_styles = [] # Temporary Hack to remove GeoServer temp styles from the list - Style.objects.filter(name__iregex=r'\w{8}-\w{4}-\w{4}-\w{4}-\w{12}_(ms)_\d{13}').delete() - for style in Style.objects.values('name', 'sld_title'): - gs_styles.append((style['name'], style['sld_title'])) + Style.objects.filter(name__iregex=r"\w{8}-\w{4}-\w{4}-\w{4}-\w{12}_(ms)_\d{13}").delete() + for style in Style.objects.values("name", "sld_title"): + gs_styles.append((style["name"], style["sld_title"])) current_dataset_styles = layer.styles.all() dataset_styles = [] for style in current_dataset_styles: @@ -257,35 +229,30 @@ def dataset_style_manage(request, layername): return render( request, - 'datasets/dataset_style_manage.html', + "datasets/dataset_style_manage.html", context={ "layer": layer, "gs_styles": gs_styles, "dataset_styles": dataset_styles, "dataset_style_names": [s[0] for s in dataset_styles], - "default_style": default_style - } + "default_style": default_style, + }, ) except (FailedRequestError, OSError): tb = traceback.format_exc() logger.debug(tb) - msg = (f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' - f'to manage style information for layer "{layer.name}"') + msg = ( + f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' + f'to manage style information for layer "{layer.name}"' + ) logger.debug(msg) # If geoserver is not online, return an error - return render( - request, - 'datasets/dataset_style_manage.html', - context={ - "layer": layer, - "error": msg - } - ) - elif request.method in ('POST', 'PUT', 'DELETE'): + return render(request, "datasets/dataset_style_manage.html", context={"layer": layer, "error": msg}) + elif request.method in ("POST", "PUT", "DELETE"): try: workspace = get_dataset_workspace(layer) or settings.DEFAULT_WORKSPACE - selected_styles = request.POST.getlist('style-select') - default_style = request.POST['default_style'] + selected_styles = request.POST.getlist("style-select") + default_style = request.POST["default_style"] # Save to GeoServer cat = gs_catalog @@ -298,8 +265,7 @@ def dataset_style_manage(request, layername): gs_dataset = cat.get_layer(layer.alternate) if gs_dataset: - _default_style = cat.get_style(default_style) or \ - cat.get_style(default_style, workspace=workspace) + _default_style = cat.get_style(default_style) or cat.get_style(default_style, workspace=workspace) if _default_style: gs_dataset.default_style = _default_style elif cat.get_style(default_style, workspace=settings.DEFAULT_WORKSPACE): @@ -330,16 +296,9 @@ def dataset_style_manage(request, layername): except (FailedRequestError, OSError, MultiValueDictKeyError): tb = traceback.format_exc() logger.debug(tb) - msg = (f'Error Saving Styles for Dataset "{layer.name}"') + msg = f'Error Saving Styles for Dataset "{layer.name}"' logger.warn(msg) - return render( - request, - 'datasets/dataset_style_manage.html', - context={ - "layer": layer, - "error": msg - } - ) + return render(request, "datasets/dataset_style_manage.html", context={"layer": layer, "error": msg}) def style_change_check(request, path, style_name=None, access_token=None): @@ -357,16 +316,16 @@ def style_change_check(request, path, style_name=None, access_token=None): # we will suppose that a user can create a new style only if he is an # authenticated (we need to discuss about it) authorized = True - if request.method in ('PUT', 'POST'): + if request.method in ("PUT", "POST"): if not request.user.is_authenticated and not access_token: authorized = False - elif re.match(r'^.*(? 0: + raw_url = urljoin(ogc_server_settings.LOCATION, posixpath.join(workspace, layername, downstream_path, path)) + + if downstream_path in ("rest/styles") and len(request.body) > 0: if ws: # Lets try # http://localhost:8080/geoserver/rest/workspaces//styles/