From 49818ebac11e0028388d81e7b3ae6179b145cd3a Mon Sep 17 00:00:00 2001 From: Emanuele Tajariol Date: Mon, 23 Jan 2023 11:20:12 +0100 Subject: [PATCH] [Fixes #10537] Improve rules creation using GeoFence batch (#10538) * [Fixes #10537] Improve rules creation using GeoFence batch * - code improvements accordingly to the PR comments * - code improvements accordingly to the PR comments * - Test fixes Co-authored-by: afabiani (cherry picked from commit d101ead2e2552f3d4f9b6cd355ce1eced3dbaecf) --- geonode/base/api/tests.py | 6 +- geonode/br/management/commands/restore.py | 6 +- geonode/geoserver/geofence.py | 384 ++++++++++++++++++++++ geonode/geoserver/helpers.py | 4 +- geonode/geoserver/security.py | 365 +++++--------------- geonode/layers/tests.py | 10 +- geonode/security/tests.py | 49 +-- geonode/utils.py | 2 +- 8 files changed, 505 insertions(+), 321 deletions(-) create mode 100644 geonode/geoserver/geofence.py diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 5e7459a06d7..19eb884a0ca 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -484,7 +484,7 @@ def test_base_resources(self): self.assertEqual(len(response.data), 5) self.assertEqual(response.data['total'], 28) - url = "{base_url}?{params}".format(base_url=reverse('base-resources-list'), params="filter{metadata_only}=false") + url = f"{reverse('base-resources-list')}?filter{{metadata_only}}=false" # Anonymous response = self.client.get(url, format='json') self.assertEqual(response.status_code, 200) @@ -558,7 +558,7 @@ def test_base_resources(self): resource = ResourceBase.objects.filter(owner__username='bobby').first() self.assertEqual(resource.owner.username, 'bobby') # Admin - url_with_id = "{base_url}/{res_id}?{params}".format(base_url=reverse('base-resources-list'), res_id=resource.id, params="filter{metadata_only}=false") + url_with_id = f"{reverse('base-resources-list')}/{resource.id}?filter{{metadata_only}}=false" response = self.client.get(f"{url_with_id}", format='json') self.assertEqual(response.data['resource']['state'], enumerations.STATE_PROCESSED) @@ -1718,7 +1718,7 @@ def test_embed_urls(self): resources = ResourceBase.objects.all() for resource in resources: - url = "{base_url}?{params}".format(base_url=reverse('base-resources-detail', kwargs={'pk': resource.pk}), params="filter{metadata_only}=false") + url = f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}?filter{{metadata_only}}=false" response = self.client.get(url, format='json') if resource.title.endswith('metadata true'): self.assertEqual(response.status_code, 404) diff --git a/geonode/br/management/commands/restore.py b/geonode/br/management/commands/restore.py index f146a2f0d9a..2f710423409 100755 --- a/geonode/br/management/commands/restore.py +++ b/geonode/br/management/commands/restore.py @@ -275,17 +275,17 @@ def execute_restore(self, **options): chmod_tree(static_root) for static_files_folder in static_folders: if getattr(settings, 'PROJECT_ROOT', None) and \ - static_files_folder.startswith(settings.PROJECT_ROOT): + static_files_folder.startswith(settings.PROJECT_ROOT): print(f"[Sanity Check] Full Write Access to '{static_files_folder}' ...") chmod_tree(static_files_folder) for template_files_folder in template_folders: if getattr(settings, 'PROJECT_ROOT', None) and \ - template_files_folder.startswith(settings.PROJECT_ROOT): + template_files_folder.startswith(settings.PROJECT_ROOT): print(f"[Sanity Check] Full Write Access to '{template_files_folder}' ...") chmod_tree(template_files_folder) for locale_files_folder in locale_folders: if getattr(settings, 'PROJECT_ROOT', None) and \ - locale_files_folder.startswith(settings.PROJECT_ROOT): + locale_files_folder.startswith(settings.PROJECT_ROOT): print(f"[Sanity Check] Full Write Access to '{locale_files_folder}' ...") chmod_tree(locale_files_folder) except Exception as exception: diff --git a/geonode/geoserver/geofence.py b/geonode/geoserver/geofence.py new file mode 100644 index 00000000000..d8c385c64e4 --- /dev/null +++ b/geonode/geoserver/geofence.py @@ -0,0 +1,384 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import urllib +import requests + +from django.conf import settings +from requests.auth import HTTPBasicAuth + +logger = logging.getLogger(__name__) + +ogc_server_settings = settings.OGC_SERVER['default'] + + +class GeofenceException(Exception): + pass + + +class Rule: + """_summary_ + JSON representation of a GeoFence Rule + + e.g.: + {"Rule": + { + "priority": 0, + "userName": "admin", + "service": "WMS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + } + + Returns: + _type_: Rule + """ + + ALLOW = "ALLOW" + DENY = "DENY" + LIMIT = "LIMIT" + CM_MIXED = "MIXED" + + def __init__(self, priority, workspace, layer, access: (str, bool), + user=None, group=None, + service=None, request=None, subfield=None, + geo_limit=None, catalog_mode=None) -> None: + self.fields = {} + + # access may be either a boolean or ALLOW/DENY/LIMIT + if access is True: + access = Rule.ALLOW + elif access is False: + access = Rule.DENY + + for field, value in ( + ('priority', priority), + + ('userName', user), + ('roleName', group), + + ('service', service), + ('request', request), + ('subfield', subfield), + + ('workspace', workspace), + ('layer', layer), + + ('access', access), + ): + if value is not None and value != '*': + self.fields[field] = value + + limits = {} + for field, value in ( + ('allowedArea', geo_limit), + ('catalogMode', catalog_mode), + ): + if value is not None: + limits[field] = value + + if limits: + self.fields['limits'] = limits + + def get_object(self): + logger.debug(f"Creating Rule object: {self.fields}") + return {'Rule': self.fields} + + +class Batch: + """_summary_ + Returns a list of Operations that GeoFence can execute in a batch + + e.g.: + { + "Batch": { + "operations": [ + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 0, + "userName": "admin", + "service": "WMS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 1, + "userName": "admin", + "service": "GWC", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 2, + "userName": "admin", + "service": "WFS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 3, + "userName": "admin", + "service": "WPS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 4, + "userName": "admin", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + } + ] + } + } + + Returns: + _type_: Batch + """ + + def __init__(self, log_name=None) -> None: + self.operations = [] + self.log_name = f'"{log_name}"' if log_name else '' + + def __str__(self) -> str: + return super().__str__() + + def add_delete_rule(self, rule_id: int): + self.operations.append({ + '@service': 'rules', + '@type': 'delete', + '@id': rule_id + }) + + def add_insert_rule(self, rule: Rule): + operation = { + '@service': 'rules', + '@type': 'insert', + } + operation.update(rule.get_object()) + self.operations.append(operation) + + def get_batch_length(self): + return len(self.operations) + + def get_object(self): + logger.debug(f"Creating Batch object {self.log_name} with {len(self.operations)} operations") + return { + 'Batch': { + 'operations': self.operations + } + } + + +class GeofenceClient: + """_summary_ + Instance of a simple GeoFence REST client allowing to interact with the GeoServer APIs. + Exposes few utility methods to insert or purge the rules and run batches of operations. + + Returns: + _type_: Rule + """ + + def __init__(self, baseurl: str, username: str, pw: str) -> None: + self.baseurl = baseurl + self.username = username + self.pw = pw + + def invalidate_cache(self): + r = requests.put( + f'{self.baseurl.rstrip("/")}/geofence/ruleCache/invalidate', + auth=HTTPBasicAuth(self.username, self.pw)) + + if r.status_code != 200: + logger.debug("Could not invalidate cache") + raise GeofenceException("Could not invalidate cache") + + def get_rules(self, page=None, entries=None, + workspace=None, workspace_any=None, + layer=None, layer_any=None): + if (page is None and entries is not None) or (page is not None and entries is None): + raise GeofenceException(f"Bad page/entries combination {page}/{entries}") + + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules.json?page={page}&entries={entries} + """ + params = {} + + if entries: + params.update({'page': page, 'entries': entries}) + + for param, value in ( + ('workspace', workspace), + ('workspaceAny', workspace_any), + ('layer', layer), + ('layerAny', layer_any), + ): + if value is not None: + params[param] = value + + url = f'{self.baseurl.rstrip("/")}/geofence/rules.json?{urllib.parse.urlencode(params)}' + + r = requests.get( + url, + headers={'Content-type': 'application/json'}, + auth=HTTPBasicAuth(self.username, self.pw), + timeout=ogc_server_settings.get('TIMEOUT', 10), + verify=False) + + if r.status_code != 200: + logger.debug(f"Could not retrieve GeoFence Rules from {url} -- code:{r.status_code} - {r.text}") + raise GeofenceException(f"Could not retrieve GeoFence Rules: [{r.status_code}]") + + return r.json() + except Exception as e: + logger.debug("Error while retrieving GeoFence rules", exc_info=e) + raise GeofenceException(f"Error while retrieving GeoFence rules: {e}") + + def get_rules_count(self): + """Get the number of available GeoFence Rules""" + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules/count.json + """ + r = requests.get( + f'{self.baseurl.rstrip("/")}/geofence/rules/count.json', + headers={'Content-type': 'application/json'}, + auth=HTTPBasicAuth(self.username, self.pw), + timeout=ogc_server_settings.get('TIMEOUT', 10), + verify=False) + + if r.status_code != 200: + logger.debug(f"Could not retrieve GeoFence Rules count: [{r.status_code}] - {r.text}") + raise GeofenceException(f"Could not retrieve GeoFence Rules count: [{r.status_code}]") + + response = r.json() + return response['count'] + + except Exception as e: + logger.debug("Error while retrieving GeoFence rules count", exc_info=e) + raise GeofenceException(f"Error while retrieving GeoFence rules count: {e}") + + def insert_rule(self, rule: Rule): + try: + """ + curl -X POST -u admin:geoserver -H "Content-Type: text/xml" -d \ + "geonode{layer}ALLOW" \ + http://:/geoserver/rest/geofence/rules + """ + r = requests.post( + f'{self.baseurl.rstrip("/")}/geofence/rules', + # headers={'Content-type': 'application/json'}, + json=rule.get_object(), + auth=HTTPBasicAuth(self.username, self.pw), + timeout=ogc_server_settings.get('TIMEOUT', 60), + verify=False) + + if r.status_code not in (200, 201): + logger.debug(f"Could not insert rule: [{r.status_code}] - {r.content}") + raise GeofenceException(f"Could not insert rule: [{r.status_code}]") + + except Exception as e: + logger.debug("Error while inserting rule", exc_info=e) + raise GeofenceException(f"Error while inserting rule: {e}") + + def run_batch(self, batch: Batch): + if batch.get_batch_length() == 0: + logger.debug(f'Skipping batch execution {batch.log_name}') + return + + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules/count.json + """ + r = requests.post( + f'{self.baseurl.rstrip("/")}/geofence/batch/exec', + json=batch.get_object(), + auth=HTTPBasicAuth(self.username, self.pw), + timeout=ogc_server_settings.get('TIMEOUT', 60), + verify=False) + + if r.status_code != 200: + logger.debug(f"Error while running batch {batch.log_name}: [{r.status_code}] - {r.content}") + raise GeofenceException(f"Error while running batch {batch.log_name}: [{r.status_code}]") + + return + + except Exception as e: + logger.debug(f"Error while requesting batch execution {batch.log_name}", exc_info=e) + raise GeofenceException(f"Error while requesting batch execution {batch.log_name}: {e}") + + def purge_all_rules(self): + """purge all existing GeoFence Cache Rules""" + rules_objs = self.get_rules() + rules = rules_objs['rules'] + + batch = Batch('Purge All') + for rule in rules: + batch.add_delete_rule(rule['id']) + + logger.debug(f"Going to remove all {len(rules)} rules in geofence") + self.run_batch(batch) + + def purge_layer_rules(self, layer_name: str, workspace: str = None): + """purge existing GeoFence Cache Rules related to a specific Layer""" + gs_rules = self.get_rules( + workspace=workspace, workspace_any=False, + layer=layer_name, layer_any=False) + + batch = Batch(f'Purge {workspace}:{layer_name}') + + if gs_rules and gs_rules['rules']: + logger.debug(f"Going to remove {len(gs_rules['rules'])} rules for layer '{layer_name}'") + for r in gs_rules['rules']: + if r['layer'] and r['layer'] == layer_name: + batch.add_delete_rule(r['id']) + else: + logger.debug(f"Bad rule retrieved for dataset '{layer_name}': {r}") + self.run_batch(batch) diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index 7b492522668..2500e8ea3f4 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -76,7 +76,7 @@ is_monochromatic_image, set_resource_default_links) -from .security import set_geowebcache_invalidate_cache +from .geofence import GeofenceClient logger = logging.getLogger(__name__) @@ -1245,6 +1245,7 @@ def set_styles(layer, gs_catalog: Catalog): logger.debug(f" -- Resource Links[Legend link]...error: {e}") try: + from .security import set_geowebcache_invalidate_cache set_geowebcache_invalidate_cache(layer.alternate or layer.typename, cat=gs_catalog) except Exception: tb = traceback.format_exc() @@ -1918,6 +1919,7 @@ def get_time_info(layer): retries=ogc_server_settings.MAX_RETRIES, backoff_factor=ogc_server_settings.BACKOFF_FACTOR) gs_uploader = Client(url, _user, _password) +gf_client = GeofenceClient(url, _user, _password) _punc = re.compile(r"[\.:]") # regex for punctuation that confuses restconfig _foregrounds = [ diff --git a/geonode/geoserver/security.py b/geonode/geoserver/security.py index 1b37bc2c4ba..18da209bfa8 100644 --- a/geonode/geoserver/security.py +++ b/geonode/geoserver/security.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # ######################################################################### -import json +import itertools import logging import typing import requests @@ -32,136 +32,19 @@ from django.contrib.auth.models import Group from django.contrib.auth import get_user_model -from geonode.utils import get_dataset_workspace from geonode.groups.models import GroupProfile +from geonode.utils import get_dataset_workspace +from geonode.geoserver.helpers import gf_client +from geonode.geoserver.geofence import Batch, Rule logger = logging.getLogger(__name__) -def _get_geofence_payload(layer, dataset_name, workspace, access, user=None, group=None, - service=None, request=None, geo_limit=None): - highest_priority = get_highest_priority() - root_el = etree.Element("Rule") - username_el = etree.SubElement(root_el, "userName") - if user is not None: - username_el.text = user - else: - username_el.text = '' - priority_el = etree.SubElement(root_el, "priority") - priority_el.text = str(highest_priority if highest_priority >= 0 else 0) - if group is not None: - role_el = etree.SubElement(root_el, "roleName") - role_el.text = f"ROLE_{group.upper()}" - workspace_el = etree.SubElement(root_el, "workspace") - workspace_el.text = workspace - dataset_el = etree.SubElement(root_el, "layer") - dataset_el.text = dataset_name - if service is not None and service != "*": - service_el = etree.SubElement(root_el, "service") - service_el.text = service - if request is not None and request != "*": - service_el = etree.SubElement(root_el, "request") - service_el.text = request - if service and service == "*" and geo_limit is not None and geo_limit != "": - access_el = etree.SubElement(root_el, "access") - access_el.text = "LIMIT" - limits = etree.SubElement(root_el, "limits") - catalog_mode = etree.SubElement(limits, "catalogMode") - catalog_mode.text = "MIXED" - allowed_area = etree.SubElement(limits, "allowedArea") - allowed_area.text = geo_limit - else: - access_el = etree.SubElement(root_el, "access") - access_el.text = access - return etree.tostring(root_el) - - -def _update_geofence_rule(layer, dataset_name, workspace, - service, request=None, - user=None, group=None, - geo_limit=None, allow=True): - payload = _get_geofence_payload( - layer=layer, - dataset_name=dataset_name, - workspace=workspace, - access="ALLOW" if allow else "DENY", - user=user, - group=group, - service=service, - request=request, - geo_limit=geo_limit - ) - logger.debug(f"request data: {payload}") - response = requests.post( - f"{settings.OGC_SERVER['default']['LOCATION']}rest/geofence/rules", - data=payload, - headers={ - 'Content-type': 'application/xml' - }, - auth=HTTPBasicAuth( - username=settings.OGC_SERVER['default']['USER'], - password=settings.OGC_SERVER['default']['PASSWORD'] - ) - ) - logger.debug(f"response status_code: {response.status_code}") - if response.status_code not in (200, 201): - msg = (f"Could not ADD GeoServer User {user} Rule for " - f"Dataset {layer}: '{response.text}'") - if 'Duplicate Rule' in response.text: - logger.debug(msg) - else: - raise RuntimeError(msg) - - -def get_geofence_rules(page=0, entries=1, count=False): - """Get the number of available GeoFence Cache Rules""" - try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] - - _url = '' - _headers = {'Content-type': 'application/json'} - if count: - """ - curl -X GET -u admin:geoserver \ - http://:/geoserver/rest/geofence/rules/count.json - """ - _url = f"{url}rest/geofence/rules/count.json" - elif page or entries: - """ - curl -X GET -u admin:geoserver \ - http://:/geoserver/rest/geofence/rules.json?page={page}&entries={entries} - """ - _url = f'{url}rest/geofence/rules.json?page={page}&entries={entries}' - r = requests.get(_url, - headers=_headers, - auth=HTTPBasicAuth(user, passwd), - timeout=10, - verify=False) - if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not retrieve GeoFence Rules count.") - - rules_objs = json.loads(r.text) - return rules_objs - except Exception: - tb = traceback.format_exc() - logger.debug(tb) - return {'count': -1} - - -def get_geofence_rules_count(): - """Get the number of available GeoFence Cache Rules""" - rules_objs = get_geofence_rules(count=True) - rules_count = rules_objs['count'] - return rules_count - - def get_highest_priority(): """Get the highest Rules priority""" try: - rules_count = get_geofence_rules_count() - rules_objs = get_geofence_rules(rules_count - 1) + rules_count = gf_client.get_rules_count() + rules_objs = gf_client.get_rules(page=rules_count - 1, entries=1) if len(rules_objs['rules']) > 0: highest_priority = rules_objs['rules'][0]['priority'] else: @@ -176,44 +59,7 @@ def get_highest_priority(): def purge_geofence_all(): """purge all existing GeoFence Cache Rules""" if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: - try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] - """ - curl -X GET -u admin:geoserver -H "Content-Type: application/json" \ - http://:/geoserver/rest/geofence/rules.json - """ - headers = {'Content-type': 'application/json'} - r = requests.get(f"{url}rest/geofence/rules.json", - headers=headers, - auth=HTTPBasicAuth(user, passwd), - timeout=10, - verify=False) - if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Retrieve GeoFence Rules") - else: - try: - rules_objs = json.loads(r.text) - rules_count = rules_objs['count'] - rules = rules_objs['rules'] - if rules_count > 0: - # Delete GeoFence Rules associated to the Dataset - # curl -X DELETE -u admin:geoserver http://:/geoserver/rest/geofence/rules/id/{r_id} - for rule in rules: - r = requests.delete(f"{url}rest/geofence/rules/id/{str(rule['id'])}", - headers=headers, - auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): - msg = f"Could not DELETE GeoServer Rule id[{rule['id']}]" - e = Exception(msg) - logger.debug(f"Response [{r.status_code}] : {r.text}") - raise e - except Exception: - logger.debug(f"Response [{r.status_code}] : {r.text}") - except Exception: - tb = traceback.format_exc() - logger.debug(tb) + gf_client.purge_all_rules() def purge_geofence_dataset_rules(resource): @@ -223,43 +69,13 @@ def purge_geofence_dataset_rules(resource): curl -u admin:geoserver http://:/geoserver/rest/geofence/rules.json?workspace=geonode&layer={layer} """ - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] - headers = {'Content-type': 'application/json'} workspace = get_dataset_workspace(resource.dataset) dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ else resource.dataset.alternate try: - r = requests.get( - f"{url}rest/geofence/rules.json?workspace={workspace}&layer={dataset_name}", - headers=headers, - auth=HTTPBasicAuth(user, passwd), - timeout=10, - verify=False - ) - if (r.status_code >= 200 and r.status_code < 300): - gs_rules = r.json() - r_ids = [] - if gs_rules and gs_rules['rules']: - for r in gs_rules['rules']: - if r['layer'] and r['layer'] == dataset_name: - r_ids.append(r['id']) - - # Delete GeoFence Rules associated to the Dataset - # curl -X DELETE -u admin:geoserver http://:/geoserver/rest/geofence/rules/id/{r_id} - for r_id in r_ids: - r = requests.delete( - f"{url}rest/geofence/rules/id/{str(r_id)}", - headers=headers, - auth=HTTPBasicAuth(user, passwd)) - if (r.status_code < 200 or r.status_code > 201): - msg = "Could not DELETE GeoServer Rule for Dataset " - msg = msg + str(dataset_name) - e = Exception(msg) - logger.debug(f"Response [{r.status_code}] : {r.text}") - logger.exception(e) - except Exception: + gf_client.purge_layer_rules(dataset_name, workspace=workspace) + except Exception as e: + logger.error(f"Error removing rules for {workspace}:{dataset_name}", exc_info=e) tb = traceback.format_exc() logger.debug(tb) @@ -268,19 +84,7 @@ def set_geofence_invalidate_cache(): """invalidate GeoFence Cache Rules""" if settings.OGC_SERVER['default']['GEOFENCE_SECURITY_ENABLED']: try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] - """ - curl -X GET -u admin:geoserver \ - http://:/geoserver/rest/ruleCache/invalidate - """ - r = requests.put(f"{url}rest/ruleCache/invalidate", - auth=HTTPBasicAuth(user, passwd)) - - if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Invalidate GeoFence Rules.") - return False + gf_client.invalidate_cache() return True except Exception: tb = traceback.format_exc() @@ -422,7 +226,7 @@ def set_geowebcache_invalidate_cache(dataset_alternate, cat=None): headers = {'Content-type': 'text/xml'} payload = f"{dataset_alternate}" r = requests.post( - f"{url}gwc/rest/masstruncate", + f"{url.rstrip('/')}/gwc/rest/masstruncate", headers=headers, data=payload, auth=HTTPBasicAuth(user, passwd)) @@ -444,58 +248,32 @@ def set_geofence_all(instance): geoserver """ - resource = instance.get_self_resource() logger.debug(f"Inside set_geofence_all for instance {instance}") workspace = get_dataset_workspace(resource.dataset) dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \ else resource.dataset.alternate logger.debug(f"going to work in workspace {workspace}") - try: - url = settings.OGC_SERVER['default']['LOCATION'] - user = settings.OGC_SERVER['default']['USER'] - passwd = settings.OGC_SERVER['default']['PASSWORD'] - - # Create GeoFence Rules for ANONYMOUS to the Dataset - """ - curl -X POST -u admin:geoserver -H "Content-Type: text/xml" -d \ - "geonode{layer}ALLOW" \ - http://:/geoserver/rest/geofence/rules - """ - headers = {'Content-type': 'application/xml'} - payload = _get_geofence_payload( - layer=resource.dataset, - dataset_name=dataset_name, - workspace=workspace, - access="ALLOW" - ) - response = requests.post( - f"{url}rest/geofence/rules", - headers=headers, - data=payload, - auth=HTTPBasicAuth(user, passwd) - ) - if response.status_code not in (200, 201): - logger.debug( - f"Response {response.status_code} : {response.text}") - raise RuntimeError("Could not ADD GeoServer ANONYMOUS Rule " - f"for Dataset {dataset_name}") - except Exception: - tb = traceback.format_exc() - logger.debug(tb) - finally: - if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): + try: + priority = get_highest_priority() + 1 + gf_client.insert_rule(Rule(priority, workspace, dataset_name, Rule.ALLOW)) + except Exception as e: + tb = traceback.format_exc() + logger.debug(tb) + raise RuntimeError(f"Could not ADD GeoServer ANONYMOUS Rule for Dataset {dataset_name}: {e}") + finally: set_geofence_invalidate_cache() - else: - resource.set_dirty_state() + else: + resource.set_dirty_state() def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_perms=None): """ Sync Guardian permissions to GeoFence. """ - _dataset_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate - _dataset_workspace = get_dataset_workspace(dataset) + layer_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate + workspace_name = get_dataset_workspace(dataset) # Create new rule-set gf_services = _get_gf_services(dataset, perms) @@ -518,56 +296,73 @@ def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_per } _user = None _group = None - users_geolimits = None - groups_geolimits = None - anonymous_geolimits = None - _group, _user, _disable_cache, users_geolimits, groups_geolimits, anonymous_geolimits = get_user_geolimits(dataset, user, group, gf_services) + + _group, _user, _disable_cache, users_geolimits, groups_geolimits, anonymous_geolimits = get_user_geolimits(dataset, user, group) if _disable_cache: gf_services_limits_first = {"*": gf_services.pop('*')} gf_services_limits_first.update(gf_services) gf_services = gf_services_limits_first + batch = Batch(f'Sync {workspace_name}:{layer_name}') + priority = get_highest_priority() + 1 + pri = itertools.count(priority) + + def resolve_geolimits(geolimits): + return geolimits.last().wkt if geolimits and geolimits.exists() else None + + # Set global geolimits + wkt = resolve_geolimits(users_geolimits) + if wkt: + logger.debug(f"Adding GeoFence USER GeoLimit rule: U:{_user} L:{dataset} ") + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, + user=_user, + geo_limit=wkt)) + wkt = resolve_geolimits(anonymous_geolimits) + if wkt: + logger.debug(f"Adding GeoFence ANON GeoLimit rule: L:{dataset} ") + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, + geo_limit=wkt)) + wkt = resolve_geolimits(groups_geolimits) + if wkt: + logger.debug(f"Adding GeoFence GROUP GeoLimit rule: G:{_group} L:{dataset} ") + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.LIMIT, catalog_mode=Rule.CM_MIXED, + group=_group, + geo_limit=wkt)) + # Set services rules for service, allowed in gf_services.items(): - if dataset and _dataset_name and allowed: + if dataset and layer_name and allowed: if _user: - logger.debug(f"Adding 'user' to geofence the rule: {dataset} {service} {_user}") - _wkt = None - if users_geolimits and users_geolimits.count(): - _wkt = users_geolimits.last().wkt + logger.debug(f"Adding GeoFence USER rules: U:{_user} S:{service} L:{dataset} ") if service in gf_requests: for request, enabled in gf_requests[service].items(): - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, - service, request=request, user=_user, allow=enabled) - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, service, user=_user, geo_limit=_wkt) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, + service=service, request=request, user=_user)) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, + service=service, user=_user)) + elif not _group: - logger.debug(f"Adding to geofence the rule: {dataset} {service} *") - _wkt = None - if anonymous_geolimits and anonymous_geolimits.count(): - _wkt = anonymous_geolimits.last().wkt - if service in gf_requests: - for request, enabled in gf_requests[service].items(): - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, - service, request=request, user=_user, allow=enabled) - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, service, geo_limit=_wkt) + logger.debug(f"Adding GeoFence ANON rules: S:{service} L:{dataset} ") + if service in gf_requests: for request, enabled in gf_requests[service].items(): - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, - service, request=request, user=_user, allow=enabled) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, + service=service, request=request)) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, + service=service)) + if _group: - logger.debug(f"Adding 'group' to geofence the rule: {dataset} {service} {_group}") - _wkt = None - if groups_geolimits and groups_geolimits.count(): - _wkt = groups_geolimits.last().wkt - if service in gf_requests: - for request, enabled in gf_requests[service].items(): - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, - service, request=request, group=_group, allow=enabled) - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, service, group=_group, geo_limit=_wkt) + logger.debug(f"Adding GeoFence GROUP rules: G:{_group} S:{service} L:{dataset} ") + if service in gf_requests: for request, enabled in gf_requests[service].items(): - _update_geofence_rule(dataset, _dataset_name, _dataset_workspace, - service, request=request, group=_group, allow=enabled) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, enabled, + service=service, request=request, group=_group)) + batch.add_insert_rule(Rule(pri.__next__(), workspace_name, layer_name, Rule.ALLOW, + service=service, group=_group)) + + gf_client.run_batch(batch) + if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False): set_geofence_invalidate_cache() else: @@ -615,7 +410,7 @@ def sync_resources_with_guardian(resource=None): logger.warn(f"!WARNING! - Failure Synching-up Security Rules for Resource [{r}]") -def get_user_geolimits(layer, user, group, gf_services): +def get_user_geolimits(layer, user, group): _user = None _group = None _disable_dataset_cache = None @@ -625,19 +420,16 @@ def get_user_geolimits(layer, user, group, gf_services): if user: _user = user if isinstance(user, str) else user.username users_geolimits = layer.users_geolimits.filter(user=get_user_model().objects.get(username=_user)) - gf_services["*"] = users_geolimits.exists() if not gf_services["*"] else gf_services["*"] _disable_dataset_cache = users_geolimits.exists() if group: _group = group if isinstance(group, str) else group.name if GroupProfile.objects.filter(group__name=_group).count() == 1: groups_geolimits = layer.groups_geolimits.filter(group=GroupProfile.objects.get(group__name=_group)) - gf_services["*"] = groups_geolimits.exists() if not gf_services["*"] else gf_services["*"] _disable_dataset_cache = groups_geolimits.exists() if not user and not group: anonymous_geolimits = layer.users_geolimits.filter(user=get_anonymous_user()) - gf_services["*"] = anonymous_geolimits.exists() if not gf_services["*"] else gf_services["*"] _disable_dataset_cache = anonymous_geolimits.exists() return _group, _user, _disable_dataset_cache, users_geolimits, groups_geolimits, anonymous_geolimits @@ -683,7 +475,6 @@ def sync_permissions_and_disable_cache(cache_rules, resource, perms, user, group sync_geofence_with_guardian(dataset=resource, perms=perms, user=user, group_perms=group_perms) else: sync_geofence_with_guardian(dataset=resource, perms=perms, user=user, group=group) - gf_services = _get_gf_services(layer=resource, perms=perms) - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer=resource, user=user, group=group, gf_services=gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer=resource, user=user, group=group) cache_rules.append(_disable_dataset_cache) return list(set(cache_rules)) diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 41f504617a8..1c8169978fa 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -2199,11 +2199,11 @@ def _create_arguments(self, perms_type, mode='set'): args = [] username = get_user_model().objects.exclude(username='admin').exclude(username='AnonymousUser').first().username opts = { - "permission": perms_type, - "users": [username], - "resources": str(dataset.id), - "delete": True if mode == 'unset' else False - } + "permission": perms_type, + "users": [username], + "resources": str(dataset.id), + "delete": True if mode == 'unset' else False + } return dataset, args, username, opts diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 842793747ba..b677d0217ca 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -72,14 +72,14 @@ from geonode.geoserver.security import ( _get_gf_services, get_user_geolimits, - get_geofence_rules, - get_geofence_rules_count, + # get_geofence_rules, + # get_geofence_rules_count, get_highest_priority, set_geofence_all, purge_geofence_all, sync_geofence_with_guardian, sync_resources_with_guardian, - _get_gwc_filters_and_formats + _get_gwc_filters_and_formats, ) from .utils import ( @@ -98,6 +98,16 @@ def _log(msg, *args): logger.debug(msg, *args) +def get_geofence_rules_count(): + from geonode.geoserver.helpers import gf_client + return gf_client.get_rules_count() + + +def get_geofence_rules(): + from geonode.geoserver.helpers import gf_client + return gf_client.get_rules() + + class StreamToLogger: """ Fake file-like stream object that redirects writes to a logger instance. @@ -507,8 +517,7 @@ def test_perm_specs_synchronization(self): # Reset GeoFence Rules purge_geofence_all() - geofence_rules_count = get_geofence_rules_count() - self.assertEqual(geofence_rules_count, 0) + self.assertEqual(get_geofence_rules_count(), 0) perm_spec = {'users': {'AnonymousUser': []}, 'groups': []} layer.set_permissions(perm_spec) @@ -545,7 +554,7 @@ def test_perm_specs_synchronization(self): geofence_rules_count = get_geofence_rules_count() self.assertEqual(geofence_rules_count, 10) - rules_objs = get_geofence_rules(entries=10) + rules_objs = get_geofence_rules() _deny_wfst_rule_exists = False for rule in rules_objs['rules']: if rule['service'] == "WFS" and \ @@ -570,7 +579,7 @@ def test_perm_specs_synchronization(self): geofence_rules_count = get_geofence_rules_count() self.assertEqual(geofence_rules_count, 13) - rules_objs = get_geofence_rules(entries=13) + rules_objs = get_geofence_rules() _deny_wfst_rule_exists = False _deny_wfst_rule_position = -1 _allow_wfs_rule_position = -1 @@ -600,7 +609,7 @@ def test_perm_specs_synchronization(self): geofence_rules_count = get_geofence_rules_count() self.assertEqual(geofence_rules_count, 7) - rules_objs = get_geofence_rules(entries=7) + rules_objs = get_geofence_rules() _deny_wfst_rule_exists = False for rule in rules_objs['rules']: if rule['service'] == "WFS" and \ @@ -630,8 +639,7 @@ def test_perm_specs_synchronization(self): layer = Dataset.objects.first() # grab bobby bobby = get_user_model().objects.get(username="bobby") - gf_services = _get_gf_services(layer, layer.get_all_level_info()) - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, None, None, gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, None, None) filters, formats = _get_gwc_filters_and_formats([_disable_dataset_cache]) self.assertListEqual(filters, [{ "styleParameterFilter": { @@ -658,8 +666,7 @@ def test_perm_specs_synchronization(self): geo_limit.save() layer.users_geolimits.add(geo_limit) self.assertEqual(layer.users_geolimits.all().count(), 1) - gf_services = _get_gf_services(layer, layer.get_all_level_info()) - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, bobby, None, gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(layer, bobby, None) filters, formats = _get_gwc_filters_and_formats([_disable_dataset_cache]) self.assertIsNone(filters) self.assertIsNone(formats) @@ -670,14 +677,14 @@ def test_perm_specs_synchronization(self): geofence_rules_count = get_geofence_rules_count() self.assertEqual(geofence_rules_count, 8) - rules_objs = get_geofence_rules(entries=8) + rules_objs = get_geofence_rules() self.assertEqual(len(rules_objs['rules']), 8) # Order is important _limit_rule_position = -1 for cnt, rule in enumerate(rules_objs['rules']): if rule['service'] is None and rule['userName'] == 'bobby': self.assertEqual(rule['userName'], 'bobby') - self.assertEqual(rule['workspace'], 'CA') + self.assertEqual(rule['workspace'], 'geonode') self.assertEqual(rule['layer'], 'CA') self.assertEqual(rule['access'], 'LIMIT') @@ -709,7 +716,7 @@ def test_perm_specs_synchronization(self): geofence_rules_count = get_geofence_rules_count() self.assertEqual(geofence_rules_count, 6) - rules_objs = get_geofence_rules(entries=6) + rules_objs = get_geofence_rules() self.assertEqual(len(rules_objs['rules']), 6) # Order is important _limit_rule_position = -1 @@ -717,7 +724,7 @@ def test_perm_specs_synchronization(self): if rule['roleName'] == 'ROLE_BAR': if rule['service'] is None: self.assertEqual(rule['userName'], None) - self.assertEqual(rule['workspace'], 'CA') + self.assertEqual(rule['workspace'], 'geonode') self.assertEqual(rule['layer'], 'CA') self.assertEqual(rule['access'], 'LIMIT') @@ -750,7 +757,7 @@ def test_perm_specs_synchronization(self): if rule['service'] is None: self.assertEqual(rule['service'], None) self.assertEqual(rule['userName'], None) - self.assertEqual(rule['workspace'], 'CA') + self.assertEqual(rule['workspace'], 'geonode') self.assertEqual(rule['layer'], 'CA') self.assertEqual(rule['access'], 'LIMIT') @@ -2185,7 +2192,7 @@ def setUp(self): self.gf_services = _get_gf_services(self.layer, self.perms) def test_should_not_disable_cache_for_user_without_geolimits(self): - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None, self.gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None) self.assertFalse(_disable_dataset_cache) def test_should_disable_cache_for_user_with_geolimits(self): @@ -2195,11 +2202,11 @@ def test_should_disable_cache_for_user_with_geolimits(self): ) self.layer.users_geolimits.set([geo_limit]) self.layer.refresh_from_db() - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None, self.gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, self.owner, None) self.assertTrue(_disable_dataset_cache) def test_should_not_disable_cache_for_anonymous_without_geolimits(self): - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None, self.gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None) self.assertFalse(_disable_dataset_cache) def test_should_disable_cache_for_anonymous_with_geolimits(self): @@ -2209,7 +2216,7 @@ def test_should_disable_cache_for_anonymous_with_geolimits(self): ) self.layer.users_geolimits.set([geo_limit]) self.layer.refresh_from_db() - _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None, self.gf_services) + _, _, _disable_dataset_cache, _, _, _ = get_user_geolimits(self.layer, None, None) self.assertTrue(_disable_dataset_cache) diff --git a/geonode/utils.py b/geonode/utils.py index 6c38fa6fc29..1316afd35da 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -357,7 +357,7 @@ def get_dataset_workspace(dataset): except Exception: workspace = None if not workspace and alternate and ':' in alternate: - workspace = alternate.split(":")[1] + workspace = alternate.split(":")[0] if not workspace: default_workspace = getattr(settings, "DEFAULT_WORKSPACE", "geonode") try: