From e553227fb8d8bf269ac88154cd46e035ff57f581 Mon Sep 17 00:00:00 2001 From: Mattia Date: Tue, 15 Oct 2024 11:51:55 +0200 Subject: [PATCH] [Fixes #12368] fix serializers for overwrite workflow --- geonode/upload/api/serializer.py | 15 +++++++++++++-- geonode/upload/api/views.py | 3 ++- geonode/upload/handlers/base.py | 1 + geonode/upload/handlers/common/vector.py | 6 ++++-- geonode/upload/handlers/shapefile/handler.py | 6 ++++-- geonode/upload/handlers/shapefile/serializer.py | 13 +++++++++++++ geonode/upload/orchestrator.py | 6 ++++-- geonode/upload/publisher.py | 3 +++ 8 files changed, 44 insertions(+), 9 deletions(-) diff --git a/geonode/upload/api/serializer.py b/geonode/upload/api/serializer.py index 13806685c40..2150455178f 100644 --- a/geonode/upload/api/serializer.py +++ b/geonode/upload/api/serializer.py @@ -33,7 +33,6 @@ class Meta: "xml_file", "sld_file", "store_spatial_files", - "overwrite_existing_layer", "skip_existing_layers", "source", ) @@ -42,11 +41,23 @@ class Meta: xml_file = serializers.FileField(required=False) sld_file = serializers.FileField(required=False) store_spatial_files = serializers.BooleanField(required=False, default=True) - overwrite_existing_layer = serializers.BooleanField(required=False, default=False) skip_existing_layers = serializers.BooleanField(required=False, default=False) source = serializers.CharField(required=False, default="upload") +class OverwriteImporterSerializer(ImporterSerializer): + class Meta: + ref_name = "OverwriteImporterSerializer" + model = ResourceBase + view_name = "importer_upload" + fields = ImporterSerializer.Meta.fields + ( + "overwrite_existing_layer", + "resource_pk", + ) + overwrite_existing_layer = serializers.BooleanField(required=True) + resource_pk = serializers.IntegerField(required=True) + + class UploadSizeLimitSerializer(BaseDynamicModelSerializer): class Meta: model = UploadSizeLimit diff --git a/geonode/upload/api/views.py b/geonode/upload/api/views.py index 53f86bab425..5c33fc58804 100644 --- a/geonode/upload/api/views.py +++ b/geonode/upload/api/views.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import ast import logging from urllib.parse import urljoin from django.conf import settings @@ -43,7 +44,7 @@ from geonode.upload.models import UploadParallelismLimit, UploadSizeLimit from geonode.upload.utils import UploadLimitValidator from geonode.upload.api.exceptions import HandlerException, ImportException -from geonode.upload.api.serializer import ImporterSerializer +from geonode.upload.api.serializer import ImporterSerializer, OverwriteImporterSerializer from geonode.upload.celery_tasks import import_orchestrator from geonode.upload.orchestrator import orchestrator from oauth2_provider.contrib.rest_framework import OAuth2Authentication diff --git a/geonode/upload/handlers/base.py b/geonode/upload/handlers/base.py index a70793c2ba8..5afb57bbe76 100644 --- a/geonode/upload/handlers/base.py +++ b/geonode/upload/handlers/base.py @@ -301,6 +301,7 @@ def overwrite_resourcehandlerinfo( def rollback(self, exec_id, rollback_from_step, action_to_rollback, *args, **kwargs): steps = self.ACTIONS.get(action_to_rollback) + if rollback_from_step not in steps: logger.info(f"Step not found {rollback_from_step}, skipping") return diff --git a/geonode/upload/handlers/common/vector.py b/geonode/upload/handlers/common/vector.py index e61667f0138..a33f6249dba 100644 --- a/geonode/upload/handlers/common/vector.py +++ b/geonode/upload/handlers/common/vector.py @@ -614,7 +614,7 @@ def overwrite_geonode_resource( ): _exec = self._get_execution_request_object(execution_id) - dataset = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user) + dataset = resource_type.objects.filter(pk=_exec.input_params.get("resource_pk"), owner=_exec.user) _overwrite = _exec.input_params.get("overwrite_existing_layer", False) # if the layer exists, we just update the information of the dataset by @@ -623,7 +623,9 @@ def overwrite_geonode_resource( dataset = dataset.first() delete_dataset_cache(dataset.alternate) - set_geowebcache_invalidate_cache(dataset.typename) + # recalculate featuretype info + DataPublisher(str(self)).cat.recalculate_featuretype(dataset) + set_geowebcache_invalidate_cache(dataset_alternate=dataset.alternate) dataset = resource_manager.update(dataset.uuid, instance=dataset, files=asset.location) diff --git a/geonode/upload/handlers/shapefile/handler.py b/geonode/upload/handlers/shapefile/handler.py index 87849ab0cd1..dfa9a31c983 100644 --- a/geonode/upload/handlers/shapefile/handler.py +++ b/geonode/upload/handlers/shapefile/handler.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import ast import json import logging import codecs @@ -27,7 +28,7 @@ from pathlib import Path from geonode.upload.handlers.shapefile.exceptions import InvalidShapeFileException -from geonode.upload.handlers.shapefile.serializer import ShapeFileSerializer +from geonode.upload.handlers.shapefile.serializer import OverwriteShapeFileSerializer, ShapeFileSerializer from geonode.upload.utils import ImporterRequestAction as ira logger = logging.getLogger("importer") @@ -88,7 +89,8 @@ def has_serializer(data) -> bool: if not _base: return False if _base.endswith("shp") if isinstance(_base, str) else _base.name.endswith("shp"): - return ShapeFileSerializer + is_overwrite_flow = ast.literal_eval(data.get("overwrite_existing_layer", "False")) + return OverwriteShapeFileSerializer if is_overwrite_flow else ShapeFileSerializer return False @staticmethod diff --git a/geonode/upload/handlers/shapefile/serializer.py b/geonode/upload/handlers/shapefile/serializer.py index ac309c05bf8..ae6616ec939 100644 --- a/geonode/upload/handlers/shapefile/serializer.py +++ b/geonode/upload/handlers/shapefile/serializer.py @@ -49,3 +49,16 @@ class Meta: overwrite_existing_layer = serializers.BooleanField(required=False, default=False) skip_existing_layers = serializers.BooleanField(required=False, default=False) source = serializers.CharField(required=False, default="upload") + + +class OverwriteShapeFileSerializer(ShapeFileSerializer): + class Meta: + ref_name = "ShapeFileSerializer" + model = ResourceBase + view_name = "importer_upload" + fields = ShapeFileSerializer.Meta.fields + ( + "overwrite_existing_layer", + "resource_pk", + ) + overwrite_existing_layer = serializers.BooleanField(required=True) + resource_pk = serializers.IntegerField(required=True) diff --git a/geonode/upload/orchestrator.py b/geonode/upload/orchestrator.py index 96e519df50f..368862e3383 100644 --- a/geonode/upload/orchestrator.py +++ b/geonode/upload/orchestrator.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import ast import logging from typing import Optional from uuid import UUID @@ -30,7 +31,7 @@ from rest_framework import serializers from geonode.upload.api.exceptions import ImportException -from geonode.upload.api.serializer import ImporterSerializer +from geonode.upload.api.serializer import ImporterSerializer, OverwriteImporterSerializer from geonode.upload.celery_app import importer_app from geonode.upload.handlers.base import BaseHandler from geonode.upload.utils import error_handler @@ -64,7 +65,8 @@ def get_serializer(self, _data) -> serializers.Serializer: if _serializer: return _serializer logger.info("specific serializer not found, fallback on the default one") - return ImporterSerializer + is_overwrite_flow = ast.literal_eval(_data.get("overwrite_existing_layer", "False")) + return (OverwriteImporterSerializer if is_overwrite_flow else ImporterSerializer) def load_handler(self, module_path): try: diff --git a/geonode/upload/publisher.py b/geonode/upload/publisher.py index b3e32a6ca02..2649210f681 100644 --- a/geonode/upload/publisher.py +++ b/geonode/upload/publisher.py @@ -85,6 +85,9 @@ def publish_resources(self, resources: List[str]): self.sanity_checks(resources) return result + def recalculate_geoserver_featuretype(self, dataset): + self.cat.recalculate_featuretype(dataset) + def overwrite_resources(self, resources: List[str]): """ We dont need to do anything for now. The data is replaced via ogr2ogr