From e51ee5d02b465e6a63147f1b571e32154e9414b4 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 11 Oct 2024 18:00:32 +0300 Subject: [PATCH 01/91] Initial commit for the metadata architecture refactoring --- geonode/base_schema.json | 55 +++++++++++++ geonode/metadata/__init__.py | 0 geonode/metadata/admin.py | 3 + geonode/metadata/apps.py | 27 +++++++ geonode/metadata/handlers.py | 102 ++++++++++++++++++++++++ geonode/metadata/manager.py | 66 +++++++++++++++ geonode/metadata/migrations/__init__.py | 0 geonode/metadata/models.py | 3 + geonode/metadata/settings.py | 3 + geonode/metadata/tests.py | 3 + geonode/metadata/urls.py | 7 ++ geonode/metadata/views.py | 36 +++++++++ geonode/model_schema.py | 14 ++++ geonode/settings.py | 1 + geonode/urls.py | 2 + manage.py | 2 +- 16 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 geonode/base_schema.json create mode 100644 geonode/metadata/__init__.py create mode 100644 geonode/metadata/admin.py create mode 100644 geonode/metadata/apps.py create mode 100644 geonode/metadata/handlers.py create mode 100644 geonode/metadata/manager.py create mode 100644 geonode/metadata/migrations/__init__.py create mode 100644 geonode/metadata/models.py create mode 100644 geonode/metadata/settings.py create mode 100644 geonode/metadata/tests.py create mode 100644 geonode/metadata/urls.py create mode 100644 geonode/metadata/views.py create mode 100644 geonode/model_schema.py diff --git a/geonode/base_schema.json b/geonode/base_schema.json new file mode 100644 index 00000000000..d3cc17db720 --- /dev/null +++ b/geonode/base_schema.json @@ -0,0 +1,55 @@ +{ +"uuid": { + "type": "string", + "title": "The UUID of the resource" + }, +"title": { + "type": "string", + "title": "The title of the resource" + }, +"abstract": { + "type": "string", + "title": "A description for the resource" + }, +"purpose": { + "type": "string", + "title": "The purpose of the resource" + }, +"alternate": { + "type": "string", + "title": "The alternate of the resource" + }, +"edition": { + "type": "string", + "title": "The edition of the resource" + }, +"attribution": { + "type": "string", + "title": "An attribution for the resource" + }, +"doi": { + "type": "string", + "title": "The DOI of the resource" + }, +"constraints other": { + "type": "string", + "title": "Other constraints of the resource" + }, +"language": { + "type": "string", + "description": "The language of the resource", + "default": "eng" + }, +"data quality statement": { + "type": "string", + "title": "Data quality statement of the resource" + }, +"srid": { + "type": "string", + "title": "The SRID of the resource", + "default": "EPSG:4326" + } + +} + + diff --git a/geonode/metadata/__init__.py b/geonode/metadata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/metadata/admin.py b/geonode/metadata/admin.py new file mode 100644 index 00000000000..8c38f3f3dad --- /dev/null +++ b/geonode/metadata/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/geonode/metadata/apps.py b/geonode/metadata/apps.py new file mode 100644 index 00000000000..0ea9ae913c7 --- /dev/null +++ b/geonode/metadata/apps.py @@ -0,0 +1,27 @@ +from django.apps import AppConfig + +class MetadataConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "geonode.metadata" + + def ready(self): + """Finalize setup""" + run_setup_hooks() + super(MetadataConfig, self).ready() + +def run_setup_hooks(*args, **kwargs): + from django.utils.module_loading import import_string + from geonode.metadata.settings import METADATA_HANDLERS + from geonode.metadata.manager import metadata_manager + import logging + + logger = logging.getLogger(__name__) + + _handlers = [ + import_string(module_path) for module_path in METADATA_HANDLERS + ] + for _handler in _handlers: + metadata_manager.add_handler(_handler) + #logger.info( + # f"The following handlers have been registered: {', '.join(_handlers)}" + #) \ No newline at end of file diff --git a/geonode/metadata/handlers.py b/geonode/metadata/handlers.py new file mode 100644 index 00000000000..7fe82dd3a44 --- /dev/null +++ b/geonode/metadata/handlers.py @@ -0,0 +1,102 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging +from abc import ABCMeta, abstractmethod +from geonode.base.models import ResourceBase +from geonode.model_schema import JSONSCHEMA_BASE + +logger = logging.getLogger(__name__) + +class Handler(metaclass=ABCMeta): + """ + Handlers take care of reading, storing, encoding, + decoding subschemas of the main Resource + """ + + @abstractmethod + def update_schema(self, jsonschema: dict = {}): + """ + It is called by the MetadataManager when creating the JSON Schema + It adds the subschema handled by the handler, and returns the + augmented instance of the JSON Schema. + """ + pass + + @abstractmethod + def get_jsonschema_instance(resource: ResourceBase, field_name: str): + """ + Called when reading metadata, returns the instance of the sub-schema + associated with the field field_name. + """ + pass + + @abstractmethod + def update_resource(resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + """ + Called when persisting data, updates the field field_name of the resource + with the content content, where json_instance is the full JSON Schema instance, + in case the handler needs some cross related data contained in the resource. + """ + pass + + @abstractmethod + def load_context(resource: ResourceBase, context: dict): + """ + Called before calls to update_resource in order to initialize info needed by the handler + """ + pass + + +class CoreHandler(Handler): + """ + The base handler builds a valid empty schema with the simple + fields of the ResourceBase model + """ + + def __init__(self): + self.json_base_schema = JSONSCHEMA_BASE + self.basic_schema = None + + def update_schema(self, jsonschema): + with open(self.json_base_schema) as f: + self.basic_schema = json.load(f) + # building the full base schema + for key, val in dict.items(self.basic_schema): + # add the base handler identity to the dictionary + val.update({"geonode:handler": "base"}) + jsonschema["properties"].update({key: val}) + + return jsonschema + + def get_jsonschema_instance(resource: ResourceBase, field_name: str): + + pass + + def update_resource(resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + + pass + + def load_context(resource: ResourceBase, context: dict): + + pass + + + diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py new file mode 100644 index 00000000000..9042949e935 --- /dev/null +++ b/geonode/metadata/manager.py @@ -0,0 +1,66 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +from abc import ABCMeta, abstractmethod +from geonode.metadata.handlers import CoreHandler +from geonode.model_schema import MODEL_SCHEMA + +logger = logging.getLogger(__name__) + +class MetadataManagerInterface(metaclass=ABCMeta): + + pass + +class MetadataManager(MetadataManagerInterface): + """ + The metadata manager is the bridge between the API and the geonode model. + The metadata manager will loop over all of the registered metadata handlers, + calling their update_schema(jsonschema) which will add the subschemas of the + fields handled by each handler. At the end of the loop, the schema will be ready + to be delivered to the caller. + """ + + def __init__(self): + self.jsonschema = MODEL_SCHEMA + self.schema = None + self.handlers = [] + + def add_handler(self, handler): + self.handlers.append(handler) + + def build_schema(self): + for handler in self.handlers: + handler_instance = handler() + #TODO I have to propely add the new additions instead of replacing them + self.schema = handler_instance.update_schema(self.jsonschema) + return self.schema + + def get_schema(self): + self.schema = self.build_schema() + return self.schema + + + #def load_context(self, jsonschema: dict = {}): + + # jsonschema = self.base_handler.update_schema(jsonschema) + + # return jsonschema + +metadata_manager = MetadataManager() \ No newline at end of file diff --git a/geonode/metadata/migrations/__init__.py b/geonode/metadata/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/metadata/models.py b/geonode/metadata/models.py new file mode 100644 index 00000000000..71a83623907 --- /dev/null +++ b/geonode/metadata/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py new file mode 100644 index 00000000000..8e50adb258b --- /dev/null +++ b/geonode/metadata/settings.py @@ -0,0 +1,3 @@ +METADATA_HANDLERS = [ + 'geonode.metadata.handlers.CoreHandler' +] \ No newline at end of file diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests.py new file mode 100644 index 00000000000..7ce503c2dd9 --- /dev/null +++ b/geonode/metadata/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py new file mode 100644 index 00000000000..a98150e91a3 --- /dev/null +++ b/geonode/metadata/urls.py @@ -0,0 +1,7 @@ +from django.urls import re_path +from geonode.metadata.views import get_schema + + +urlpatterns = [ + re_path(r"^metadata/schema/$", get_schema, name="get_schema"), +] \ No newline at end of file diff --git a/geonode/metadata/views.py b/geonode/metadata/views.py new file mode 100644 index 00000000000..c0e5e6e33f8 --- /dev/null +++ b/geonode/metadata/views.py @@ -0,0 +1,36 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.metadata.manager import metadata_manager +from django.views.decorators.csrf import csrf_exempt +from pathlib import Path +from django.http import HttpResponse, JsonResponse + +@csrf_exempt +def get_schema(request): + + schema = metadata_manager.get_schema() + if schema: + # response = HttpResponse(final_schema, content_type="application/json") + response = JsonResponse(schema, safe=False) + return response + + #else: + # return response + diff --git a/geonode/model_schema.py b/geonode/model_schema.py new file mode 100644 index 00000000000..5bdf920f3da --- /dev/null +++ b/geonode/model_schema.py @@ -0,0 +1,14 @@ +import os +from geonode.settings import PROJECT_ROOT + +MODEL_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "{GEONODE_SITE}/resource.json", + "title": "GeoNode resource", + "type": "object", + "properties": { + } + } + +# The base schema is defined as a file in order to be customizable from other GeoNode instances +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "base_schema.json") \ No newline at end of file diff --git a/geonode/settings.py b/geonode/settings.py index 119844bbe02..b79cc98f29e 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -408,6 +408,7 @@ "geonode.catalogue", "geonode.catalogue.metadataxsl", "geonode.harvesting", + "geonode.metadata", ) # GeoNode Apps diff --git a/geonode/urls.py b/geonode/urls.py index 99b1e5ebaf9..30a912adb8c 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -130,6 +130,8 @@ re_path(r"^api/v2/", include("geonode.facets.urls")), re_path(r"^api/v2/", include("geonode.assets.urls")), re_path(r"", include(api.urls)), + # metadata views + re_path(r"", include("geonode.metadata.urls")), ] # tinymce WYSIWYG HTML Editor diff --git a/manage.py b/manage.py index 00cf85e4071..7978c236ff5 100755 --- a/manage.py +++ b/manage.py @@ -22,7 +22,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.local_settings") from django.core.management import execute_from_command_line From afe2b797930371ec99a5a8c12bee33ba1474b365 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Sat, 12 Oct 2024 13:33:05 +0300 Subject: [PATCH 02/91] improving the code --- geonode/metadata/handlers.py | 5 +---- geonode/metadata/manager.py | 22 +++++++++++----------- geonode/metadata/settings.py | 17 ++++++++++++++++- geonode/metadata/views.py | 12 ++++++------ geonode/model_schema.py | 14 -------------- 5 files changed, 34 insertions(+), 36 deletions(-) delete mode 100644 geonode/model_schema.py diff --git a/geonode/metadata/handlers.py b/geonode/metadata/handlers.py index 7fe82dd3a44..f602da6c526 100644 --- a/geonode/metadata/handlers.py +++ b/geonode/metadata/handlers.py @@ -21,7 +21,7 @@ import logging from abc import ABCMeta, abstractmethod from geonode.base.models import ResourceBase -from geonode.model_schema import JSONSCHEMA_BASE +from geonode.metadata.settings import JSONSCHEMA_BASE logger = logging.getLogger(__name__) @@ -97,6 +97,3 @@ def update_resource(resource: ResourceBase, field_name: str, content: dict, json def load_context(resource: ResourceBase, context: dict): pass - - - diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 9042949e935..02afd7e6cd0 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -20,7 +20,7 @@ import logging from abc import ABCMeta, abstractmethod from geonode.metadata.handlers import CoreHandler -from geonode.model_schema import MODEL_SCHEMA +from geonode.metadata.settings import MODEL_SCHEMA logger = logging.getLogger(__name__) @@ -48,19 +48,19 @@ def add_handler(self, handler): def build_schema(self): for handler in self.handlers: handler_instance = handler() - #TODO I have to propely add the new additions instead of replacing them - self.schema = handler_instance.update_schema(self.jsonschema) + + if self.schema: + # Update the properties key of the schema with the properties of the new handler + self.schema["properties"].update(handler_instance.update_schema(self.jsonschema)["properties"]) + else: + self.schema = handler_instance.update_schema(self.jsonschema) + return self.schema def get_schema(self): - self.schema = self.build_schema() + if not self.schema: + self.build_schema() + return self.schema - - - #def load_context(self, jsonschema: dict = {}): - - # jsonschema = self.base_handler.update_schema(jsonschema) - - # return jsonschema metadata_manager = MetadataManager() \ No newline at end of file diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 8e50adb258b..8ead8aacead 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -1,3 +1,18 @@ +import os +from geonode.settings import PROJECT_ROOT + +MODEL_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "{GEONODE_SITE}/resource.json", + "title": "GeoNode resource", + "type": "object", + "properties": { + } + } + +# The base schema is defined as a file in order to be customizable from other GeoNode instances +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "base_schema.json") + METADATA_HANDLERS = [ - 'geonode.metadata.handlers.CoreHandler' + 'geonode.metadata.handlers.CoreHandler', ] \ No newline at end of file diff --git a/geonode/metadata/views.py b/geonode/metadata/views.py index c0e5e6e33f8..975ae37c8db 100644 --- a/geonode/metadata/views.py +++ b/geonode/metadata/views.py @@ -19,18 +19,18 @@ from geonode.metadata.manager import metadata_manager from django.views.decorators.csrf import csrf_exempt -from pathlib import Path from django.http import HttpResponse, JsonResponse +from django.core. exceptions import ObjectDoesNotExist @csrf_exempt def get_schema(request): schema = metadata_manager.get_schema() + if schema: - # response = HttpResponse(final_schema, content_type="application/json") - response = JsonResponse(schema, safe=False) - return response + return JsonResponse(schema) - #else: - # return response + else: + response = {"Message": "Schema not found"} + return JsonResponse(response) diff --git a/geonode/model_schema.py b/geonode/model_schema.py deleted file mode 100644 index 5bdf920f3da..00000000000 --- a/geonode/model_schema.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -from geonode.settings import PROJECT_ROOT - -MODEL_SCHEMA = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "{GEONODE_SITE}/resource.json", - "title": "GeoNode resource", - "type": "object", - "properties": { - } - } - -# The base schema is defined as a file in order to be customizable from other GeoNode instances -JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "base_schema.json") \ No newline at end of file From fbdb30a4bc1d30eac347ab8bd630205b3030252d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 14 Oct 2024 11:03:51 +0300 Subject: [PATCH 03/91] update the first handler --- geonode/base_schema.json | 57 ++++++++++++++++++++++--------------- geonode/metadata/apps.py | 10 +++---- geonode/metadata/manager.py | 12 ++++---- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/geonode/base_schema.json b/geonode/base_schema.json index d3cc17db720..c4bc4f7fb9f 100644 --- a/geonode/base_schema.json +++ b/geonode/base_schema.json @@ -1,52 +1,63 @@ { "uuid": { "type": "string", - "title": "The UUID of the resource" + "description": "The UUID of the resource", + "maxLength": 36 }, "title": { "type": "string", - "title": "The title of the resource" + "description": "Νame by which the cited resource is known", + "maxLength": 255 }, "abstract": { "type": "string", - "title": "A description for the resource" + "description": "Brief narrative summary of the content of the resource(s)", + "maxLength": 2000 }, "purpose": { "type": "string", - "title": "The purpose of the resource" + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500 }, "alternate": { "type": "string", - "title": "The alternate of the resource" + "description": "The alternate of the resource", + "maxLength": 255 }, + "edition": { "type": "string", - "title": "The edition of the resource" - }, -"attribution": { - "type": "string", - "title": "An attribution for the resource" + "description": "The edition of the resource" }, -"doi": { +"date type": { "type": "string", - "title": "The DOI of the resource" - }, -"constraints other": { - "type": "string", - "title": "Other constraints of the resource" - }, + "maxLength": 255, + "enum": ["Creation", "Publication", "Revision"] + }, "language": { "type": "string", - "description": "The language of the resource", + "description": "Language used within the dataset", + "maxLength": 255, + "oneOf": [ + { + "const": "abk", + "title": "Abkhazian" + }, + { + "const": "aar", + "title": "Afar" + }, + { + "const": "afr", + "title": "Africaans" + } + ], "default": "eng" }, -"data quality statement": { - "type": "string", - "title": "Data quality statement of the resource" - }, "srid": { "type": "string", - "title": "The SRID of the resource", + "description": "The SRID of the resource", + "maxLength": 255, "default": "EPSG:4326" } diff --git a/geonode/metadata/apps.py b/geonode/metadata/apps.py index 0ea9ae913c7..2675ae37ce1 100644 --- a/geonode/metadata/apps.py +++ b/geonode/metadata/apps.py @@ -1,4 +1,6 @@ from django.apps import AppConfig +from django.utils.module_loading import import_string +import logging class MetadataConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" @@ -10,10 +12,8 @@ def ready(self): super(MetadataConfig, self).ready() def run_setup_hooks(*args, **kwargs): - from django.utils.module_loading import import_string from geonode.metadata.settings import METADATA_HANDLERS from geonode.metadata.manager import metadata_manager - import logging logger = logging.getLogger(__name__) @@ -22,6 +22,6 @@ def run_setup_hooks(*args, **kwargs): ] for _handler in _handlers: metadata_manager.add_handler(_handler) - #logger.info( - # f"The following handlers have been registered: {', '.join(_handlers)}" - #) \ No newline at end of file + logger.info( + f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}" + ) \ No newline at end of file diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 02afd7e6cd0..87baae9fe60 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -43,17 +43,19 @@ def __init__(self): self.handlers = [] def add_handler(self, handler): - self.handlers.append(handler) + + # Handlers initialization + handler_obj = handler() + self.handlers.append(handler_obj) def build_schema(self): for handler in self.handlers: - handler_instance = handler() if self.schema: - # Update the properties key of the schema with the properties of the new handler - self.schema["properties"].update(handler_instance.update_schema(self.jsonschema)["properties"]) + # Update the properties key of the current schema with the properties of the new handler + self.schema["properties"].update(handler.update_schema(self.jsonschema)["properties"]) else: - self.schema = handler_instance.update_schema(self.jsonschema) + self.schema = handler.update_schema(self.jsonschema) return self.schema From 65f477f8cc1febd1d26ba13d961a2fb4468c4fc5 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 14 Oct 2024 11:30:33 +0300 Subject: [PATCH 04/91] rename the file of the main schema --- geonode/{base_schema.json => core_schema.json} | 0 geonode/metadata/settings.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename geonode/{base_schema.json => core_schema.json} (100%) diff --git a/geonode/base_schema.json b/geonode/core_schema.json similarity index 100% rename from geonode/base_schema.json rename to geonode/core_schema.json diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 8ead8aacead..34fd34a78a1 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -11,7 +11,7 @@ } # The base schema is defined as a file in order to be customizable from other GeoNode instances -JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "base_schema.json") +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "core_schema.json") METADATA_HANDLERS = [ 'geonode.metadata.handlers.CoreHandler', From bafa137d89cde870e3c5f73c17e73ea9c570144f Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 14 Oct 2024 11:34:22 +0300 Subject: [PATCH 05/91] fixing manage.py --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 7978c236ff5..00cf85e4071 100755 --- a/manage.py +++ b/manage.py @@ -22,7 +22,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.local_settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.settings") from django.core.management import execute_from_command_line From 47f1fa9a253949ec8ff0af34976f2d467a831538 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 14 Oct 2024 14:09:29 +0300 Subject: [PATCH 06/91] For testing a specific folder for json schemas examples was created --- .../jsonschema_examples}/core_schema.json | 0 .../full_schema_forUI.json | 74 +++++++++++++++++++ geonode/metadata/settings.py | 2 +- 3 files changed, 75 insertions(+), 1 deletion(-) rename geonode/{ => metadata/jsonschema_examples}/core_schema.json (100%) create mode 100644 geonode/metadata/jsonschema_examples/full_schema_forUI.json diff --git a/geonode/core_schema.json b/geonode/metadata/jsonschema_examples/core_schema.json similarity index 100% rename from geonode/core_schema.json rename to geonode/metadata/jsonschema_examples/core_schema.json diff --git a/geonode/metadata/jsonschema_examples/full_schema_forUI.json b/geonode/metadata/jsonschema_examples/full_schema_forUI.json new file mode 100644 index 00000000000..f43cd52ca1e --- /dev/null +++ b/geonode/metadata/jsonschema_examples/full_schema_forUI.json @@ -0,0 +1,74 @@ +{ + "$id": "{GEONODE_SITE}/resource.json", + "type": "object", + "properties": { + "uuid": { + "type": "string", + "description": "The UUID of the resource", + "maxLength": 36 + }, + "title": { + "type": "string", + "description": "Νame by which the cited resource is known", + "maxLength": 255 + }, + "abstract": { + "type": "string", + "description": "Brief narrative summary of the content of the resource(s)", + "maxLength": 2000 + }, + "purpose": { + "type": "string", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500 + }, + "alternate": { + "type": "string", + "description": "The alternate of the resource", + "maxLength": 255 + }, + "edition": { + "type": "string", + "description": "The edition of the resource" + }, + "date type": { + "type": "string", + "maxLength": 255, + "enum": [ + "Creation", + "Publication", + "Revision" + ], + "default": "Publication" + }, + "language": { + "type": "string", + "description": "Language used within the dataset", + "maxLength": 255, + "oneOf": [ + { + "const": "abk", + "title": "Abkhazian" + }, + { + "const": "aar", + "title": "Afar" + }, + { + "const": "afr", + "title": "Africaans" + } + ], + "default": "eng" + }, + "srid": { + "type": "string", + "description": "The SRID of the resource", + "maxLength": 255, + "default": "EPSG:4326" + } + }, + "required": [ + "title" + ] +} diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 34fd34a78a1..88b77fa28a7 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -11,7 +11,7 @@ } # The base schema is defined as a file in order to be customizable from other GeoNode instances -JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "core_schema.json") +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/core_schema.json") METADATA_HANDLERS = [ 'geonode.metadata.handlers.CoreHandler', From 32fab1d79c172896e777de5535f637eb44f5db08 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 14 Oct 2024 14:33:08 +0300 Subject: [PATCH 07/91] formatting the json schema files --- .../jsonschema_examples/core_schema.json | 136 +++++++++--------- .../full_schema_forUI.json | 2 +- 2 files changed, 72 insertions(+), 66 deletions(-) diff --git a/geonode/metadata/jsonschema_examples/core_schema.json b/geonode/metadata/jsonschema_examples/core_schema.json index c4bc4f7fb9f..dca662245a5 100644 --- a/geonode/metadata/jsonschema_examples/core_schema.json +++ b/geonode/metadata/jsonschema_examples/core_schema.json @@ -1,66 +1,72 @@ { -"uuid": { - "type": "string", - "description": "The UUID of the resource", - "maxLength": 36 - }, -"title": { - "type": "string", - "description": "Νame by which the cited resource is known", - "maxLength": 255 - }, -"abstract": { - "type": "string", - "description": "Brief narrative summary of the content of the resource(s)", - "maxLength": 2000 - }, -"purpose": { - "type": "string", - "description": "Summary of the intentions with which the resource(s) was developed", - "maxLength": 500 - }, -"alternate": { - "type": "string", - "description": "The alternate of the resource", - "maxLength": 255 - }, - -"edition": { - "type": "string", - "description": "The edition of the resource" - }, -"date type": { - "type": "string", - "maxLength": 255, - "enum": ["Creation", "Publication", "Revision"] - }, -"language": { - "type": "string", - "description": "Language used within the dataset", - "maxLength": 255, - "oneOf": [ - { - "const": "abk", - "title": "Abkhazian" - }, - { - "const": "aar", - "title": "Afar" - }, - { - "const": "afr", - "title": "Africaans" - } - ], - "default": "eng" - }, -"srid": { - "type": "string", - "description": "The SRID of the resource", - "maxLength": 255, - "default": "EPSG:4326" - } - -} - - + "uuid": { + "type": "string", + "description": "The UUID of the resource", + "maxLength": 36 + }, + "title": { + "type": "string", + "description": "Νame by which the cited resource is known", + "maxLength": 255 + }, + "abstract": { + "type": "string", + "description": "Brief narrative summary of the content of the resource(s)", + "maxLength": 2000 + }, + "purpose": { + "type": "string", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500 + }, + "alternate": { + "type": "string", + "description": "The alternate of the resource", + "maxLength": 255 + }, + "date type": { + "type": "string", + "maxLength": 255, + "enum": [ + "Creation", + "Publication", + "Revision" + ] + }, + "edition": { + "type": "string", + "description": "Version of the cited resource", + "maxLength": 255 + }, + "attribution": { + "type": "string", + "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", + "maxLength": 2048 + }, + "language": { + "type": "string", + "description": "Language used within the dataset", + "maxLength": 255, + "oneOf": [ + { + "const": "abk", + "title": "Abkhazian" + }, + { + "const": "aar", + "title": "Afar" + }, + { + "const": "afr", + "title": "Africaans" + } + ], + "default": "eng" + }, + "srid": { + "type": "string", + "description": "The SRID of the resource", + "maxLength": 255, + "default": "EPSG:4326" + } +} \ No newline at end of file diff --git a/geonode/metadata/jsonschema_examples/full_schema_forUI.json b/geonode/metadata/jsonschema_examples/full_schema_forUI.json index f43cd52ca1e..be6cb4a321d 100644 --- a/geonode/metadata/jsonschema_examples/full_schema_forUI.json +++ b/geonode/metadata/jsonschema_examples/full_schema_forUI.json @@ -71,4 +71,4 @@ "required": [ "title" ] -} +} \ No newline at end of file From 506dc4e24314b728725525dba932e0ebb258bb46 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 15 Oct 2024 14:20:31 +0300 Subject: [PATCH 08/91] update the json schema examples --- .../jsonschema_examples/core_schema.json | 892 +++++++++++++++- .../jsonschema_examples/full_schema_V1.json | 953 ++++++++++++++++++ .../full_schema_forUI.json | 74 -- 3 files changed, 1835 insertions(+), 84 deletions(-) create mode 100644 geonode/metadata/jsonschema_examples/full_schema_V1.json delete mode 100644 geonode/metadata/jsonschema_examples/full_schema_forUI.json diff --git a/geonode/metadata/jsonschema_examples/core_schema.json b/geonode/metadata/jsonschema_examples/core_schema.json index dca662245a5..034b44d99ff 100644 --- a/geonode/metadata/jsonschema_examples/core_schema.json +++ b/geonode/metadata/jsonschema_examples/core_schema.json @@ -2,49 +2,137 @@ "uuid": { "type": "string", "description": "The UUID of the resource", - "maxLength": 36 + "maxLength": 36, + "ui:widget": "hidden" }, "title": { "type": "string", + "title": "title", "description": "Νame by which the cited resource is known", "maxLength": 255 }, "abstract": { "type": "string", + "title": "abstract", "description": "Brief narrative summary of the content of the resource(s)", - "maxLength": 2000 + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } }, "purpose": { "type": "string", + "title": "purpose", "description": "Summary of the intentions with which the resource(s) was developed", - "maxLength": 500 + "maxLength": 500, + "ui:options": { + "widget": "textarea", + "rows": 5 + } }, "alternate": { "type": "string", - "description": "The alternate of the resource", - "maxLength": 255 + "maxLength": 255, + "ui:widget": "hidden" }, - "date type": { + "date_type": { "type": "string", + "title": "date type", "maxLength": 255, "enum": [ "Creation", "Publication", "Revision" - ] + ], + "default": "Publication" }, "edition": { "type": "string", + "title": "edition", "description": "Version of the cited resource", "maxLength": 255 }, "attribution": { "type": "string", + "title": "Attribution", "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", "maxLength": 2048 }, + "doi": { + "type": "string", + "title": "DOI", + "description": "a DOI will be added by Admin before publication.", + "maxLength": 255 + }, + "maintenance_frequency": { + "type": "string", + "title": "maintenance frequency", + "description": "frequency with which modifications and deletions are made to the data after it is first produced", + "maxLength": 255, + "oneOf": [ + { + "const": "unknown", + "title": "frequency of maintenance for the data is not known" + }, + { + "const": "continual", + "title": "data is repeatedly and frequently updated" + }, + { + "const": "notPlanned", + "title": "there are no plans to update the data" + }, + { + "const": "daily", + "title": "data is updated each day" + }, + { + "const": "annually", + "title": "data is updated every year" + }, + { + "const": "asNeeded", + "title": "data is updated as deemed necessary" + }, + { + "const": "monthly", + "title": "data is updated each month" + }, + { + "const": "fortnightly", + "title": "data is updated every two weeks" + }, + { + "const": "irregular", + "title": "data is updated in intervals that are uneven in duration" + }, + { + "const": "weekly", + "title": "data is updated on a weekly basis" + }, + { + "const": "biannually", + "title": "data is updated twice each year" + }, + { + "const": "quarterly", + "title": "data is updated every three months" + } + ] + }, + "other_constrains": { + "type": "string", + "title": "Other constrains", + "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, "language": { "type": "string", + "title": "language", "description": "Language used within the dataset", "maxLength": 255, "oneOf": [ @@ -58,15 +146,799 @@ }, { "const": "afr", - "title": "Africaans" + "title": "Afrikaans" + }, + { + "const": "amh", + "title": "Amharic" + }, + { + "const": "ara", + "title": "Arabic" + }, + { + "const": "asm", + "title": "Assamese" + }, + { + "const": "aym", + "title": "Aymara" + }, + { + "const": "aze", + "title": "Azerbaijani" + }, + { + "const": "bak", + "title": "Bashkir" + }, + { + "const": "ben", + "title": "Bengali" + }, + { + "const": "bih", + "title": "Bihari" + }, + { + "const": "bis", + "title": "Bislama" + }, + { + "const": "bre", + "title": "Breton" + }, + { + "const": "bul", + "title": "Bulgarian" + }, + { + "const": "bel", + "title": "Byelorussian" + }, + { + "const": "cat", + "title": "Catalan" + }, + { + "const": "chi", + "title": "Chinese" + }, + { + "const": "cos", + "title": "Corsican" + }, + { + "const": "dan", + "title": "Danish" + }, + { + "const": "dzo", + "title": "Dzongkha" + }, + { + "const": "eng", + "title": "English" + }, + { + "const": "fra", + "title": "French" + }, + { + "const": "epo", + "title": "Esperanto" + }, + { + "const": "est", + "title": "Estonian" + }, + { + "const": "fao", + "title": "Faroese" + }, + { + "const": "fij", + "title": "Fijian" + }, + { + "const": "fin", + "title": "Finnish" + }, + { + "const": "fry", + "title": "Frisian" + }, + { + "const": "glg", + "title": "Gallegan" + }, + { + "const": "ger", + "title": "German" + }, + { + "const": "gre", + "title": "Greek" + }, + { + "const": "kal", + "title": "Greenlandic" + }, + { + "const": "grn", + "title": "Guarani" + }, + { + "const": "guj", + "title": "Gujarati" + }, + { + "const": "hau", + "title": "Hausa" + }, + { + "const": "heb", + "title": "Hebrew" + }, + { + "const": "hin", + "title": "Hindi" + }, + { + "const": "hun", + "title": "Hungarian" + }, + { + "const": "ind", + "title": "Indonesian" + }, + { + "const": "ina", + "title": "Interlingua (International Auxiliary language Association)" + }, + { + "const": "iku", + "title": "Inuktitut" + }, + { + "const": "ipk", + "title": "Inupiak" + }, + { + "const": "ita", + "title": "Italian" + }, + { + "const": "jpn", + "title": "Japanese" + }, + { + "const": "kan", + "title": "Kannada" + }, + { + "const": "kas", + "title": "Kashmiri" + }, + { + "const": "kaz", + "title": "Kazakh" + }, + { + "const": "khm", + "title": "Khmer" + }, + { + "const": "kin", + "title": "Kinyarwanda" + }, + { + "const": "kir", + "title": "Kirghiz" + }, + { + "const": "kor", + "title": "Korean" + }, + { + "const": "kur", + "title": "Kurdish" + }, + { + "const": "oci", + "title": "Langue d 'Oc (post 1500)" + }, + { + "const": "lao", + "title": "Lao" + }, + { + "const": "lat", + "title": "Latin" + }, + { + "const": "lav", + "title": "Latvian" + }, + { + "const": "lin", + "title": "Lingala" + }, + { + "const": "lit", + "title": "Lithuanian" + }, + { + "const": "mlg", + "title": "Malagasy" + }, + { + "const": "mlt", + "title": "Maltese" + }, + { + "const": "mar", + "title": "Marathi" + }, + { + "const": "mol", + "title": "Moldavian" + }, + { + "const": "mon", + "title": "Mongolian" + }, + { + "const": "nau", + "title": "Nauru" + }, + { + "const": "nep", + "title": "Nepali" + }, + { + "const": "nor", + "title": "Norwegian" + }, + { + "const": "ori", + "title": "Oriya" + }, + { + "const": "orm", + "title": "Oromo" + }, + { + "const": "pan", + "title": "Panjabi" + }, + { + "const": "pol", + "title": "Polish" + }, + { + "const": "por", + "title": "Portuguese" + }, + { + "const": "pus", + "title": "Pushto" + }, + { + "const": "que", + "title": "Quechua" + }, + { + "const": "roh", + "title": "Rhaeto-Romance" + }, + { + "const": "run", + "title": "Rundi" + }, + { + "const": "rus", + "title": "Russian" + }, + { + "const": "smo", + "title": "Samoan" + }, + { + "const": "sag", + "title": "Sango" + }, + { + "const": "san", + "title": "Sanskrit" + }, + { + "const": "scr", + "title": "Serbo-Croatian" + }, + { + "const": "sna", + "title": "Shona" + }, + { + "const": "snd", + "title": "Sindhi" + }, + { + "const": "sin", + "title": "Singhalese" + }, + { + "const": "ssw", + "title": "Siswant" + }, + { + "const": "slv", + "title": "Slovenian" + }, + { + "const": "som", + "title": "Somali" + }, + { + "const": "sot", + "title": "Sotho" + }, + { + "const": "spa", + "title": "Spanish" + }, + { + "const": "sun", + "title": "Sudanese" + }, + { + "const": "swa", + "title": "Swahili" + }, + { + "const": "tgl", + "title": "Tagalog" + }, + { + "const": "tgk", + "title": "Tajik" + }, + { + "const": "tam", + "title": "Tamil" + }, + { + "const": "tat", + "title": "Tatar" + }, + { + "const": "tel", + "title": "Telugu" + }, + { + "const": "tha", + "title": "Thai" + }, + { + "const": "tir", + "title": "Tigrinya" + }, + { + "const": "tog", + "title": "Tonga (Nyasa)" + }, + { + "const": "tso", + "title": "Tsonga" + }, + { + "const": "tsn", + "title": "Tswana" + }, + { + "const": "tur", + "title": "Turkish" + }, + { + "const": "tuk", + "title": "Turkmen" + }, + { + "const": "twi", + "title": "Twi" + }, + { + "const": "uig", + "title": "Uighur" + }, + { + "const": "ukr", + "title": "Ukrainian" + }, + { + "const": "urd", + "title": "Urdu" + }, + { + "const": "uzb", + "title": "Uzbek" + }, + { + "const": "vie", + "title": "Vietnamese" + }, + { + "const": "vol", + "title": "Volapük" + }, + { + "const": "wol", + "title": "Wolof" + }, + { + "const": "xho", + "title": "Xhosa" + }, + { + "const": "yid", + "title": "Yiddish" + }, + { + "const": "yor", + "title": "Yoruba" + }, + { + "const": "zha", + "title": "Zhuang" + }, + { + "const": "zul", + "title": "Zulu" } ], "default": "eng" }, + "supplemental_information": { + "type": "string", + "title": "supplemental information", + "description": "any other descriptive information about the dataset", + "maxLength": 2000, + "defult": "No information provided", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "data_quality_statement": { + "type": "string", + "title": "data quality statement", + "description": "general explanation of the data producer's knowledge about the lineage of a dataset", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, "srid": { "type": "string", "description": "The SRID of the resource", - "maxLength": 255, - "default": "EPSG:4326" + "maxLength": 30, + "default": "EPSG:4326", + "ui:widget": "hidden" + }, + "csw_typename": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW typename", + "maxLength": 32, + "default": "gmd:MD_Metadata", + "ui:widget": "hidden" + }, + "csw_schema": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW schema", + "maxLength": 64, + "default": "http://www.isotc211.org/2005/gmd", + "ui:widget": "hidden" + }, + "csw_mdsource": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW source", + "maxLength": 256, + "default": "local", + "ui:widget": "hidden" + }, + "csw_type": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW type", + "maxLength": 32, + "default": "dataset", + "oneOf": [ + { + "const": "series", + "title": "series" + }, + { + "const": "software", + "title": "computer program or routine" + }, + { + "const": "featureType", + "title": "feature type" + }, + { + "const": "model", + "title": "copy or imitation of an existing or hypothetical object" + }, + { + "const": "collectionHardware", + "title": "collection hardware" + }, + { + "const": "collectionSession", + "title": "collection session" + }, + { + "const": "nonGeographicDataset", + "title": "non-geographic data" + }, + { + "const": "propertyType", + "title": "property type" + }, + { + "const": "fieldSession", + "title": "field session" + }, + { + "const": "dataset", + "title": "dataset" + }, + { + "const": "service", + "title": "service interfaces" + }, + { + "const": "attribute", + "title": "attribute class" + }, + { + "const": "attributeType", + "title": "characteristic of a feature" + }, + { + "const": "tile", + "title": "tile or spatial subset of geographic data" + }, + { + "const": "feature", + "title": "feature" + }, + { + "const": "dimensionGroup", + "title": "dimension group" + } + ], + "ui:widget": "hidden" + }, + "csw_anytext": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW anytext", + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "ui:widget": "hidden" + }, + "csw_wkt_geometry": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW WKT geometry", + "default": "POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "ui:widget": "hidden" + }, + "metadata uploaded": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "default": false, + "ui:widget": "hidden" + }, + "Metadata uploaded preserve": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "default": false + }, + "metadata xml": { + "$comment": "metadata XML specific fields", + "type": "string", + "default": "'", + "ui:widget": "hidden" + }, + "popular_count": { + "$comment": "metadata XML specific fields", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "share_count": { + "$comment": "metadata XML specific fields", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "featured": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Featured", + "description": "Should this resource be advertised in home page?", + "default": false + }, + "was_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was published", + "description": "Previous Published state.", + "default": true + }, + "is_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Is published", + "description": "Should this resource be published and searchable?", + "default": true + }, + "was_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was Approved", + "description": "Previous Approved state.", + "default": true + }, + "is_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Approved", + "description": "Is this resource validated from a publisher or editor?", + "default": true + }, + "advertised": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Advertised", + "description": "If False, will hide the resource from search results and catalog listings", + "default": true + }, + "thumbnail_url": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail url", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "thumbnail_path": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail path", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "rating": { + "$comment": "fields necessary for the apis", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "state": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "State", + "description": "Hold the resource processing state.", + "default": "READY", + "maxLength": 16, + "oneOf": [ + { + "const": "READY", + "title": "READY" + }, + { + "const": "RUNNING", + "title": "RUNNING" + }, + { + "const": "PENDING", + "title": "PENDING" + }, + { + "const": "WAITING", + "title": "WAITING" + }, + { + "const": "INCOMPLETE", + "title": "INCOMPLETE" + }, + { + "const": "COMPLETE", + "title": "COMPLETE" + }, + { + "const": "INVALID", + "title": "INVALID" + }, + { + "const": "PROCESSED", + "title": "PROCESSED" + } + ] + }, + "sourcetype": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Source Type", + "description": "The resource source type, which can be one of 'LOCAL', 'REMOTE' or 'COPYREMOTE'.", + "default": "LOCAL", + "maxLength": 16, + "oneOf": [ + { + "const": "LOCAL", + "title": "LOCAL" + }, + { + "const": "REMOTE", + "title": "REMOTE" + }, + { + "const": "COPYREMOTE", + "title": "COPYREMOTE" + } + ] + }, + "remote_typename": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Remote Service Typenamee", + "description": "Name of the Remote Service if any.", + "maxLength": 512, + "ui:widget": "hidden" + }, + "dirty_state": { + "$comment": "fields controlling security state", + "type": "boolean", + "title": "Dirty State", + "description": "Security Rules Are Not Synched with GeoServer!", + "default": false, + "ui:widget": "hidden" + }, + "resource_type": { + "$comment": "fields controlling security state", + "type": "string", + "title": "Resource Type", + "maxLength": 1024, + "ui:widget": "hidden" + }, + "metadata_only": { + "$comment": "fields controlling security state", + "type": "boolean", + "title": "Metadata", + "description": "If true, will be excluded from search", + "default": false + }, + "subtype": { + "$comment": "fields controlling security state", + "type": "string", + "maxLength": 128, + "ui:widget": "hidden" } } \ No newline at end of file diff --git a/geonode/metadata/jsonschema_examples/full_schema_V1.json b/geonode/metadata/jsonschema_examples/full_schema_V1.json new file mode 100644 index 00000000000..ac4027591e6 --- /dev/null +++ b/geonode/metadata/jsonschema_examples/full_schema_V1.json @@ -0,0 +1,953 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "{GEONODE_SITE}/resource.json", + "type": "object", + "title": "GeoNode resource", + "properties": { + "uuid": { + "type": "string", + "description": "The UUID of the resource", + "maxLength": 36, + "ui:widget": "hidden" + }, + "title": { + "type": "string", + "title": "title", + "description": "Νame by which the cited resource is known", + "maxLength": 255 + }, + "abstract": { + "type": "string", + "title": "abstract", + "description": "Brief narrative summary of the content of the resource(s)", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "purpose": { + "type": "string", + "title": "purpose", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "alternate": { + "type": "string", + "maxLength": 255, + "ui:widget": "hidden" + }, + "date_type": { + "type": "string", + "title": "date type", + "maxLength": 255, + "enum": [ + "Creation", + "Publication", + "Revision" + ], + "default": "Publication" + }, + "edition": { + "type": "string", + "title": "edition", + "description": "Version of the cited resource", + "maxLength": 255 + }, + "attribution": { + "type": "string", + "title": "Attribution", + "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", + "maxLength": 2048 + }, + "doi": { + "type": "string", + "title": "DOI", + "description": "a DOI will be added by Admin before publication.", + "maxLength": 255 + }, + "maintenance_frequency": { + "type": "string", + "title": "maintenance frequency", + "description": "frequency with which modifications and deletions are made to the data after it is first produced", + "maxLength": 255, + "oneOf": [ + { + "const": "unknown", + "title": "frequency of maintenance for the data is not known" + }, + { + "const": "continual", + "title": "data is repeatedly and frequently updated" + }, + { + "const": "notPlanned", + "title": "there are no plans to update the data" + }, + { + "const": "daily", + "title": "data is updated each day" + }, + { + "const": "annually", + "title": "data is updated every year" + }, + { + "const": "asNeeded", + "title": "data is updated as deemed necessary" + }, + { + "const": "monthly", + "title": "data is updated each month" + }, + { + "const": "fortnightly", + "title": "data is updated every two weeks" + }, + { + "const": "irregular", + "title": "data is updated in intervals that are uneven in duration" + }, + { + "const": "weekly", + "title": "data is updated on a weekly basis" + }, + { + "const": "biannually", + "title": "data is updated twice each year" + }, + { + "const": "quarterly", + "title": "data is updated every three months" + } + ] + }, + "other_constrains": { + "type": "string", + "title": "Other constrains", + "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "language": { + "type": "string", + "title": "language", + "description": "Language used within the dataset", + "maxLength": 255, + "oneOf": [ + { + "const": "abk", + "title": "Abkhazian" + }, + { + "const": "aar", + "title": "Afar" + }, + { + "const": "afr", + "title": "Afrikaans" + }, + { + "const": "amh", + "title": "Amharic" + }, + { + "const": "ara", + "title": "Arabic" + }, + { + "const": "asm", + "title": "Assamese" + }, + { + "const": "aym", + "title": "Aymara" + }, + { + "const": "aze", + "title": "Azerbaijani" + }, + { + "const": "bak", + "title": "Bashkir" + }, + { + "const": "ben", + "title": "Bengali" + }, + { + "const": "bih", + "title": "Bihari" + }, + { + "const": "bis", + "title": "Bislama" + }, + { + "const": "bre", + "title": "Breton" + }, + { + "const": "bul", + "title": "Bulgarian" + }, + { + "const": "bel", + "title": "Byelorussian" + }, + { + "const": "cat", + "title": "Catalan" + }, + { + "const": "chi", + "title": "Chinese" + }, + { + "const": "cos", + "title": "Corsican" + }, + { + "const": "dan", + "title": "Danish" + }, + { + "const": "dzo", + "title": "Dzongkha" + }, + { + "const": "eng", + "title": "English" + }, + { + "const": "fra", + "title": "French" + }, + { + "const": "epo", + "title": "Esperanto" + }, + { + "const": "est", + "title": "Estonian" + }, + { + "const": "fao", + "title": "Faroese" + }, + { + "const": "fij", + "title": "Fijian" + }, + { + "const": "fin", + "title": "Finnish" + }, + { + "const": "fry", + "title": "Frisian" + }, + { + "const": "glg", + "title": "Gallegan" + }, + { + "const": "ger", + "title": "German" + }, + { + "const": "gre", + "title": "Greek" + }, + { + "const": "kal", + "title": "Greenlandic" + }, + { + "const": "grn", + "title": "Guarani" + }, + { + "const": "guj", + "title": "Gujarati" + }, + { + "const": "hau", + "title": "Hausa" + }, + { + "const": "heb", + "title": "Hebrew" + }, + { + "const": "hin", + "title": "Hindi" + }, + { + "const": "hun", + "title": "Hungarian" + }, + { + "const": "ind", + "title": "Indonesian" + }, + { + "const": "ina", + "title": "Interlingua (International Auxiliary language Association)" + }, + { + "const": "iku", + "title": "Inuktitut" + }, + { + "const": "ipk", + "title": "Inupiak" + }, + { + "const": "ita", + "title": "Italian" + }, + { + "const": "jpn", + "title": "Japanese" + }, + { + "const": "kan", + "title": "Kannada" + }, + { + "const": "kas", + "title": "Kashmiri" + }, + { + "const": "kaz", + "title": "Kazakh" + }, + { + "const": "khm", + "title": "Khmer" + }, + { + "const": "kin", + "title": "Kinyarwanda" + }, + { + "const": "kir", + "title": "Kirghiz" + }, + { + "const": "kor", + "title": "Korean" + }, + { + "const": "kur", + "title": "Kurdish" + }, + { + "const": "oci", + "title": "Langue d 'Oc (post 1500)" + }, + { + "const": "lao", + "title": "Lao" + }, + { + "const": "lat", + "title": "Latin" + }, + { + "const": "lav", + "title": "Latvian" + }, + { + "const": "lin", + "title": "Lingala" + }, + { + "const": "lit", + "title": "Lithuanian" + }, + { + "const": "mlg", + "title": "Malagasy" + }, + { + "const": "mlt", + "title": "Maltese" + }, + { + "const": "mar", + "title": "Marathi" + }, + { + "const": "mol", + "title": "Moldavian" + }, + { + "const": "mon", + "title": "Mongolian" + }, + { + "const": "nau", + "title": "Nauru" + }, + { + "const": "nep", + "title": "Nepali" + }, + { + "const": "nor", + "title": "Norwegian" + }, + { + "const": "ori", + "title": "Oriya" + }, + { + "const": "orm", + "title": "Oromo" + }, + { + "const": "pan", + "title": "Panjabi" + }, + { + "const": "pol", + "title": "Polish" + }, + { + "const": "por", + "title": "Portuguese" + }, + { + "const": "pus", + "title": "Pushto" + }, + { + "const": "que", + "title": "Quechua" + }, + { + "const": "roh", + "title": "Rhaeto-Romance" + }, + { + "const": "run", + "title": "Rundi" + }, + { + "const": "rus", + "title": "Russian" + }, + { + "const": "smo", + "title": "Samoan" + }, + { + "const": "sag", + "title": "Sango" + }, + { + "const": "san", + "title": "Sanskrit" + }, + { + "const": "scr", + "title": "Serbo-Croatian" + }, + { + "const": "sna", + "title": "Shona" + }, + { + "const": "snd", + "title": "Sindhi" + }, + { + "const": "sin", + "title": "Singhalese" + }, + { + "const": "ssw", + "title": "Siswant" + }, + { + "const": "slv", + "title": "Slovenian" + }, + { + "const": "som", + "title": "Somali" + }, + { + "const": "sot", + "title": "Sotho" + }, + { + "const": "spa", + "title": "Spanish" + }, + { + "const": "sun", + "title": "Sudanese" + }, + { + "const": "swa", + "title": "Swahili" + }, + { + "const": "tgl", + "title": "Tagalog" + }, + { + "const": "tgk", + "title": "Tajik" + }, + { + "const": "tam", + "title": "Tamil" + }, + { + "const": "tat", + "title": "Tatar" + }, + { + "const": "tel", + "title": "Telugu" + }, + { + "const": "tha", + "title": "Thai" + }, + { + "const": "tir", + "title": "Tigrinya" + }, + { + "const": "tog", + "title": "Tonga (Nyasa)" + }, + { + "const": "tso", + "title": "Tsonga" + }, + { + "const": "tsn", + "title": "Tswana" + }, + { + "const": "tur", + "title": "Turkish" + }, + { + "const": "tuk", + "title": "Turkmen" + }, + { + "const": "twi", + "title": "Twi" + }, + { + "const": "uig", + "title": "Uighur" + }, + { + "const": "ukr", + "title": "Ukrainian" + }, + { + "const": "urd", + "title": "Urdu" + }, + { + "const": "uzb", + "title": "Uzbek" + }, + { + "const": "vie", + "title": "Vietnamese" + }, + { + "const": "vol", + "title": "Volapük" + }, + { + "const": "wol", + "title": "Wolof" + }, + { + "const": "xho", + "title": "Xhosa" + }, + { + "const": "yid", + "title": "Yiddish" + }, + { + "const": "yor", + "title": "Yoruba" + }, + { + "const": "zha", + "title": "Zhuang" + }, + { + "const": "zul", + "title": "Zulu" + } + ], + "default": "eng" + }, + "supplemental_information": { + "type": "string", + "title": "supplemental information", + "description": "any other descriptive information about the dataset", + "maxLength": 2000, + "defult": "No information provided", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "data_quality_statement": { + "type": "string", + "title": "data quality statement", + "description": "general explanation of the data producer's knowledge about the lineage of a dataset", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "srid": { + "type": "string", + "description": "The SRID of the resource", + "maxLength": 30, + "default": "EPSG:4326", + "ui:widget": "hidden" + }, + "csw_typename": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW typename", + "maxLength": 32, + "default": "gmd:MD_Metadata", + "ui:widget": "hidden" + }, + "csw_schema": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW schema", + "maxLength": 64, + "default": "http://www.isotc211.org/2005/gmd", + "ui:widget": "hidden" + }, + "csw_mdsource": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW source", + "maxLength": 256, + "default": "local", + "ui:widget": "hidden" + }, + "csw_type": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW type", + "maxLength": 32, + "default": "dataset", + "oneOf": [ + { + "const": "series", + "title": "series" + }, + { + "const": "software", + "title": "computer program or routine" + }, + { + "const": "featureType", + "title": "feature type" + }, + { + "const": "model", + "title": "copy or imitation of an existing or hypothetical object" + }, + { + "const": "collectionHardware", + "title": "collection hardware" + }, + { + "const": "collectionSession", + "title": "collection session" + }, + { + "const": "nonGeographicDataset", + "title": "non-geographic data" + }, + { + "const": "propertyType", + "title": "property type" + }, + { + "const": "fieldSession", + "title": "field session" + }, + { + "const": "dataset", + "title": "dataset" + }, + { + "const": "service", + "title": "service interfaces" + }, + { + "const": "attribute", + "title": "attribute class" + }, + { + "const": "attributeType", + "title": "characteristic of a feature" + }, + { + "const": "tile", + "title": "tile or spatial subset of geographic data" + }, + { + "const": "feature", + "title": "feature" + }, + { + "const": "dimensionGroup", + "title": "dimension group" + } + ], + "ui:widget": "hidden" + }, + "csw_anytext": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW anytext", + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "ui:widget": "hidden" + }, + "csw_wkt_geometry": { + "$comment": "CSW specific fields", + "type": "string", + "title": "CSW WKT geometry", + "default": "POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "ui:widget": "hidden" + }, + "metadata uploaded": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "default": false, + "ui:widget": "hidden" + }, + "Metadata uploaded preserve": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "default": false + }, + "metadata xml": { + "$comment": "metadata XML specific fields", + "type": "string", + "default": "'", + "ui:widget": "hidden" + }, + "popular_count": { + "$comment": "metadata XML specific fields", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "share_count": { + "$comment": "metadata XML specific fields", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "featured": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Featured", + "description": "Should this resource be advertised in home page?", + "default": false + }, + "was_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was published", + "description": "Previous Published state.", + "default": true + }, + "is_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Is published", + "description": "Should this resource be published and searchable?", + "default": true + }, + "was_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was Approved", + "description": "Previous Approved state.", + "default": true + }, + "is_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Approved", + "description": "Is this resource validated from a publisher or editor?", + "default": true + }, + "advertised": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Advertised", + "description": "If False, will hide the resource from search results and catalog listings", + "default": true + }, + "thumbnail_url": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail url", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "thumbnail_path": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail path", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "rating": { + "$comment": "fields necessary for the apis", + "type": "integer", + "default": 0, + "ui:widget": "hidden" + }, + "state": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "State", + "description": "Hold the resource processing state.", + "default": "READY", + "maxLength": 16, + "oneOf": [ + { + "const": "READY", + "title": "READY" + }, + { + "const": "RUNNING", + "title": "RUNNING" + }, + { + "const": "PENDING", + "title": "PENDING" + }, + { + "const": "WAITING", + "title": "WAITING" + }, + { + "const": "INCOMPLETE", + "title": "INCOMPLETE" + }, + { + "const": "COMPLETE", + "title": "COMPLETE" + }, + { + "const": "INVALID", + "title": "INVALID" + }, + { + "const": "PROCESSED", + "title": "PROCESSED" + } + ] + }, + "sourcetype": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Source Type", + "description": "The resource source type, which can be one of 'LOCAL', 'REMOTE' or 'COPYREMOTE'.", + "default": "LOCAL", + "maxLength": 16, + "oneOf": [ + { + "const": "LOCAL", + "title": "LOCAL" + }, + { + "const": "REMOTE", + "title": "REMOTE" + }, + { + "const": "COPYREMOTE", + "title": "COPYREMOTE" + } + ] + }, + "remote_typename": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Remote Service Typenamee", + "description": "Name of the Remote Service if any.", + "maxLength": 512, + "ui:widget": "hidden" + }, + "dirty_state": { + "$comment": "fields controlling security state", + "type": "boolean", + "title": "Dirty State", + "description": "Security Rules Are Not Synched with GeoServer!", + "default": false, + "ui:widget": "hidden" + }, + "resource_type": { + "$comment": "fields controlling security state", + "type": "string", + "title": "Resource Type", + "maxLength": 1024, + "ui:widget": "hidden" + }, + "metadata_only": { + "$comment": "fields controlling security state", + "type": "boolean", + "title": "Metadata", + "description": "If true, will be excluded from search", + "default": false + }, + "subtype": { + "$comment": "fields controlling security state", + "type": "string", + "maxLength": 128, + "ui:widget": "hidden" + } + }, + "required": [ + "title" + ] +} \ No newline at end of file diff --git a/geonode/metadata/jsonschema_examples/full_schema_forUI.json b/geonode/metadata/jsonschema_examples/full_schema_forUI.json deleted file mode 100644 index be6cb4a321d..00000000000 --- a/geonode/metadata/jsonschema_examples/full_schema_forUI.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "$id": "{GEONODE_SITE}/resource.json", - "type": "object", - "properties": { - "uuid": { - "type": "string", - "description": "The UUID of the resource", - "maxLength": 36 - }, - "title": { - "type": "string", - "description": "Νame by which the cited resource is known", - "maxLength": 255 - }, - "abstract": { - "type": "string", - "description": "Brief narrative summary of the content of the resource(s)", - "maxLength": 2000 - }, - "purpose": { - "type": "string", - "description": "Summary of the intentions with which the resource(s) was developed", - "maxLength": 500 - }, - "alternate": { - "type": "string", - "description": "The alternate of the resource", - "maxLength": 255 - }, - "edition": { - "type": "string", - "description": "The edition of the resource" - }, - "date type": { - "type": "string", - "maxLength": 255, - "enum": [ - "Creation", - "Publication", - "Revision" - ], - "default": "Publication" - }, - "language": { - "type": "string", - "description": "Language used within the dataset", - "maxLength": 255, - "oneOf": [ - { - "const": "abk", - "title": "Abkhazian" - }, - { - "const": "aar", - "title": "Afar" - }, - { - "const": "afr", - "title": "Africaans" - } - ], - "default": "eng" - }, - "srid": { - "type": "string", - "description": "The SRID of the resource", - "maxLength": 255, - "default": "EPSG:4326" - } - }, - "required": [ - "title" - ] -} \ No newline at end of file From b28305e3496e7519d394df769d5179f8aaf32e81 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 17 Oct 2024 11:02:23 +0300 Subject: [PATCH 09/91] adding the metadata/schema endpoint under api/v2 --- geonode/metadata/api/urls.py | 24 +++++++++++++++++ geonode/metadata/api/views.py | 49 +++++++++++++++++++++++++++++++++++ geonode/metadata/urls.py | 9 +++---- geonode/metadata/views.py | 36 +------------------------ geonode/urls.py | 4 +-- 5 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 geonode/metadata/api/urls.py create mode 100644 geonode/metadata/api/views.py diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py new file mode 100644 index 00000000000..4e88d503a31 --- /dev/null +++ b/geonode/metadata/api/urls.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework import routers +from geonode.metadata.api import views + +router = routers.DefaultRouter() + +router.register(r"metadata", views.MetadataViewSet, basename="metadata") diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py new file mode 100644 index 00000000000..56994875e2c --- /dev/null +++ b/geonode/metadata/api/views.py @@ -0,0 +1,49 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.metadata.manager import metadata_manager +from django.views.decorators.csrf import csrf_exempt +from rest_framework.viewsets import ViewSet +from rest_framework.decorators import action +from django.http import JsonResponse + +class MetadataViewSet(ViewSet): + """ + Simple viewset that return the metadata JSON schema + """ + + # serializer_class = MetadataSerializer + + # Get the JSON schema + @action(detail=False, methods=['get']) + def get_schema(self, request): + ''' + The user is able to export her/his keys with + resource scope. + ''' + + schema = metadata_manager.get_schema() + + if schema: + return JsonResponse(schema) + + else: + response = {"Message": "Schema not found"} + return JsonResponse(response) + diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py index a98150e91a3..b4bd5f64e2a 100644 --- a/geonode/metadata/urls.py +++ b/geonode/metadata/urls.py @@ -1,7 +1,4 @@ -from django.urls import re_path -from geonode.metadata.views import get_schema +from geonode.metadata.api.urls import router +from django.urls import path, include - -urlpatterns = [ - re_path(r"^metadata/schema/$", get_schema, name="get_schema"), -] \ No newline at end of file +urlpatterns = [] + router.urls \ No newline at end of file diff --git a/geonode/metadata/views.py b/geonode/metadata/views.py index 975ae37c8db..44ea94be1d1 100644 --- a/geonode/metadata/views.py +++ b/geonode/metadata/views.py @@ -1,36 +1,2 @@ -######################################################################### -# -# Copyright (C) 2024 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### - -from geonode.metadata.manager import metadata_manager -from django.views.decorators.csrf import csrf_exempt -from django.http import HttpResponse, JsonResponse -from django.core. exceptions import ObjectDoesNotExist - -@csrf_exempt -def get_schema(request): - - schema = metadata_manager.get_schema() - - if schema: - return JsonResponse(schema) - - else: - response = {"Message": "Schema not found"} - return JsonResponse(response) +# Create your views here diff --git a/geonode/urls.py b/geonode/urls.py index 30a912adb8c..6e13b26a028 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -129,9 +129,9 @@ re_path(r"^api/v2/api-auth/", include("rest_framework.urls", namespace="geonode_rest_framework")), re_path(r"^api/v2/", include("geonode.facets.urls")), re_path(r"^api/v2/", include("geonode.assets.urls")), - re_path(r"", include(api.urls)), # metadata views - re_path(r"", include("geonode.metadata.urls")), + re_path(r"^api/v2/", include("geonode.metadata.urls")), + re_path(r"", include(api.urls)), ] # tinymce WYSIWYG HTML Editor From 972d638d27e3a35b9b3eb8010cab4f0d783e6a06 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 17 Oct 2024 11:07:11 +0300 Subject: [PATCH 10/91] rename the action of getting schema --- geonode/metadata/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 56994875e2c..b0958acd977 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -32,7 +32,7 @@ class MetadataViewSet(ViewSet): # Get the JSON schema @action(detail=False, methods=['get']) - def get_schema(self, request): + def schema(self, request): ''' The user is able to export her/his keys with resource scope. From 28ede60d9b76b750b9a344b8bdba35ea50e5090d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 18 Oct 2024 18:07:09 +0300 Subject: [PATCH 11/91] adding the metadata/instance/{pk} endpoint --- geonode/metadata/api/serializers.py | 50 +++++++++++++++++++ geonode/metadata/api/urls.py | 3 +- geonode/metadata/api/views.py | 24 +++++++-- geonode/metadata/handlers.py | 10 ++-- .../jsonschema_examples/core_schema.json | 10 ++-- geonode/metadata/manager.py | 29 +++++++++++ geonode/metadata/urls.py | 2 +- 7 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 geonode/metadata/api/serializers.py diff --git a/geonode/metadata/api/serializers.py b/geonode/metadata/api/serializers.py new file mode 100644 index 00000000000..56cfb9e55d1 --- /dev/null +++ b/geonode/metadata/api/serializers.py @@ -0,0 +1,50 @@ +from rest_framework import serializers +from geonode.base.models import ResourceBase + +class MetadataSerializer(serializers.ModelSerializer): + class Meta: + model = ResourceBase + fields = [ + "uuid", + "title", + "abstract", + "purpose", + "alternate", + "date_type", + "edition", + "attribution", + "doi", + "maintenance_frequency", + "constraints_other", + "language", + "supplemental_information", + "data_quality_statement", + "srid", + "csw_typename", + "csw_schema", + "csw_mdsource", + "csw_type", + "csw_anytext", + "csw_wkt_geometry", + "metadata_uploaded", + "metadata_uploaded_preserve", + "metadata_xml", + "popular_count", + "share_count", + "featured", + "was_published", + "is_published", + "was_approved", + "is_approved", + "advertised", + "thumbnail_url", + "thumbnail_path", + "rating", + "state", + "sourcetype", + "remote_typename", + "dirty_state", + "resource_type", + "metadata_only", + "subtype", + ] \ No newline at end of file diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 4e88d503a31..11746f0cfd9 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -20,5 +20,4 @@ from geonode.metadata.api import views router = routers.DefaultRouter() - -router.register(r"metadata", views.MetadataViewSet, basename="metadata") +router.register(r"metadata", views.MetadataViewSet, basename = "metadata") diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b0958acd977..c9d282e1211 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -18,17 +18,24 @@ ######################################################################### from geonode.metadata.manager import metadata_manager -from django.views.decorators.csrf import csrf_exempt +from geonode.metadata.api.serializers import MetadataSerializer from rest_framework.viewsets import ViewSet +from geonode.base.models import ResourceBase from rest_framework.decorators import action from django.http import JsonResponse +from rest_framework.response import Response class MetadataViewSet(ViewSet): """ Simple viewset that return the metadata JSON schema """ + + queryset = ResourceBase.objects.all() + serializer_class = MetadataSerializer - # serializer_class = MetadataSerializer + def list(self, request): + serializer = self.serializer_class(many=True) + return Response(serializer.data) # Get the JSON schema @action(detail=False, methods=['get']) @@ -41,9 +48,18 @@ def schema(self, request): schema = metadata_manager.get_schema() if schema: - return JsonResponse(schema) + return Response(schema) else: response = {"Message": "Schema not found"} - return JsonResponse(response) + return Response(response) + + def retrieve(self, request, pk=None): + data = self.queryset.filter(pk=pk) + #serializer = self._get_and_validate_serializer(data=data) + # resource = metadata_manager.get_resource_base(data, self.serializer_class) + schema_instance = metadata_manager.build_schema_instance(data) + serialized_resource = self.serializer_class(data=schema_instance) + serialized_resource.is_valid(raise_exception=True) + return Response(serialized_resource.data) \ No newline at end of file diff --git a/geonode/metadata/handlers.py b/geonode/metadata/handlers.py index f602da6c526..b449af33dda 100644 --- a/geonode/metadata/handlers.py +++ b/geonode/metadata/handlers.py @@ -86,14 +86,16 @@ def update_schema(self, jsonschema): return jsonschema - def get_jsonschema_instance(resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + + field_value = resource.values().first()[field_name] - pass + return field_value - def update_resource(resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): pass - def load_context(resource: ResourceBase, context: dict): + def load_context(self, resource: ResourceBase, context: dict): pass diff --git a/geonode/metadata/jsonschema_examples/core_schema.json b/geonode/metadata/jsonschema_examples/core_schema.json index 034b44d99ff..0cc228eac1e 100644 --- a/geonode/metadata/jsonschema_examples/core_schema.json +++ b/geonode/metadata/jsonschema_examples/core_schema.json @@ -121,7 +121,7 @@ } ] }, - "other_constrains": { + "constraints_other": { "type": "string", "title": "Other constrains", "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", @@ -604,7 +604,7 @@ "title": "supplemental information", "description": "any other descriptive information about the dataset", "maxLength": 2000, - "defult": "No information provided", + "default": "No information provided", "ui:options": { "widget": "textarea", "rows": 5 @@ -746,18 +746,18 @@ }, "ui:widget": "hidden" }, - "metadata uploaded": { + "metadata_uploaded": { "$comment": "metadata XML specific fields", "type": "boolean", "default": false, "ui:widget": "hidden" }, - "Metadata uploaded preserve": { + "metadata_uploaded_preserve": { "$comment": "metadata XML specific fields", "type": "boolean", "default": false }, - "metadata xml": { + "metadata_xml": { "$comment": "metadata XML specific fields", "type": "string", "default": "'", diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 87baae9fe60..7ce7b330437 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -21,6 +21,7 @@ from abc import ABCMeta, abstractmethod from geonode.metadata.handlers import CoreHandler from geonode.metadata.settings import MODEL_SCHEMA +from geonode.metadata.api.serializers import MetadataSerializer logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ def __init__(self): self.jsonschema = MODEL_SCHEMA self.schema = None self.handlers = [] + self.serializer_class = MetadataSerializer def add_handler(self, handler): @@ -64,5 +66,32 @@ def get_schema(self): self.build_schema() return self.schema + + def build_schema_instance(self, resource): + + instance = {} + + # serialized_resource = self.get_resource_base(resource) + schema = self.get_schema() + + for fieldname, field in schema["properties"].items(): + handler_id = field["geonode:handler"] + # temp + handler = self.handlers[0] + content = handler.get_jsonschema_instance(resource, fieldname) + instance[fieldname] = content + + return instance + + + + def resource_base_serialization(self, resource): + """ + Get a serialized dataset from the ResourceBase model + """ + serializer = self.serializer_class + + serialized_data = serializer(resource, many=True).data + return serialized_data metadata_manager = MetadataManager() \ No newline at end of file diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py index b4bd5f64e2a..89461c03f5f 100644 --- a/geonode/metadata/urls.py +++ b/geonode/metadata/urls.py @@ -1,4 +1,4 @@ from geonode.metadata.api.urls import router -from django.urls import path, include +from django.urls import path, include, re_path urlpatterns = [] + router.urls \ No newline at end of file From 9b71a995d3d475d31451d34a0b84671b10770d0f Mon Sep 17 00:00:00 2001 From: gpetrak Date: Sun, 20 Oct 2024 18:35:29 +0300 Subject: [PATCH 12/91] adding handlers registry --- geonode/metadata/api/views.py | 6 +++--- geonode/metadata/apps.py | 18 ++++++------------ geonode/metadata/manager.py | 31 +++++++++++++++---------------- geonode/metadata/registry.py | 27 +++++++++++++++++++++++++++ geonode/metadata/settings.py | 6 +++--- 5 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 geonode/metadata/registry.py diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index c9d282e1211..bdcf5e19708 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -60,6 +60,6 @@ def retrieve(self, request, pk=None): #serializer = self._get_and_validate_serializer(data=data) # resource = metadata_manager.get_resource_base(data, self.serializer_class) schema_instance = metadata_manager.build_schema_instance(data) - serialized_resource = self.serializer_class(data=schema_instance) - serialized_resource.is_valid(raise_exception=True) - return Response(serialized_resource.data) \ No newline at end of file + #serialized_resource = self.serializer_class(data=schema_instance) + #serialized_resource.is_valid(raise_exception=True) + return Response(schema_instance) \ No newline at end of file diff --git a/geonode/metadata/apps.py b/geonode/metadata/apps.py index 2675ae37ce1..02b8e17492c 100644 --- a/geonode/metadata/apps.py +++ b/geonode/metadata/apps.py @@ -1,6 +1,4 @@ from django.apps import AppConfig -from django.utils.module_loading import import_string -import logging class MetadataConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" @@ -12,16 +10,12 @@ def ready(self): super(MetadataConfig, self).ready() def run_setup_hooks(*args, **kwargs): - from geonode.metadata.settings import METADATA_HANDLERS + from geonode.metadata.registry import metadata_registry from geonode.metadata.manager import metadata_manager - logger = logging.getLogger(__name__) + # registry initialization + metadata_registry.init_registry() + handlers = metadata_registry.handler_registry - _handlers = [ - import_string(module_path) for module_path in METADATA_HANDLERS - ] - for _handler in _handlers: - metadata_manager.add_handler(_handler) - logger.info( - f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}" - ) \ No newline at end of file + for handler_id, handler in handlers.items(): + metadata_manager.add_handler(handler_id, handler) \ No newline at end of file diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 7ce7b330437..fa82bf9886f 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -19,9 +19,9 @@ import logging from abc import ABCMeta, abstractmethod -from geonode.metadata.handlers import CoreHandler from geonode.metadata.settings import MODEL_SCHEMA from geonode.metadata.api.serializers import MetadataSerializer +from geonode.metadata.registry import metadata_registry logger = logging.getLogger(__name__) @@ -41,21 +41,25 @@ class MetadataManager(MetadataManagerInterface): def __init__(self): self.jsonschema = MODEL_SCHEMA self.schema = None - self.handlers = [] self.serializer_class = MetadataSerializer + self.instance = {} + self.handlers = {} - def add_handler(self, handler): + def add_handler(self, handler_id, handler): - # Handlers initialization - handler_obj = handler() - self.handlers.append(handler_obj) + handler_instance = handler() + + self.handlers[handler_id] = handler_instance + def build_schema(self): - for handler in self.handlers: + + for handler in self.handlers.values(): if self.schema: + subschema = handler.update_schema(self.jsonschema)["properties"] # Update the properties key of the current schema with the properties of the new handler - self.schema["properties"].update(handler.update_schema(self.jsonschema)["properties"]) + self.schema["properties"].update(subschema) else: self.schema = handler.update_schema(self.jsonschema) @@ -68,22 +72,17 @@ def get_schema(self): return self.schema def build_schema_instance(self, resource): - - instance = {} # serialized_resource = self.get_resource_base(resource) schema = self.get_schema() for fieldname, field in schema["properties"].items(): handler_id = field["geonode:handler"] - # temp - handler = self.handlers[0] + handler = self.handlers[handler_id] content = handler.get_jsonschema_instance(resource, fieldname) - instance[fieldname] = content - - return instance - + self.instance[fieldname] = content + return self.instance def resource_base_serialization(self, resource): """ diff --git a/geonode/metadata/registry.py b/geonode/metadata/registry.py new file mode 100644 index 00000000000..651771497cd --- /dev/null +++ b/geonode/metadata/registry.py @@ -0,0 +1,27 @@ +from django.utils.module_loading import import_string +from geonode.metadata.settings import METADATA_HANDLERS +import logging + +logger = logging.getLogger(__name__) + + +class MetadataHandlersRegistry: + + handler_registry = {} + + def init_registry(self): + self.register() + logger.info( + f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}" + ) + + def register(self): + for handler_id, module_path in METADATA_HANDLERS.items(): + self.handler_registry[handler_id] = import_string(module_path) + + @classmethod + def get_registry(cls): + return MetadataHandlersRegistry.handler_registry + + +metadata_registry = MetadataHandlersRegistry() \ No newline at end of file diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 88b77fa28a7..7025c699284 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -13,6 +13,6 @@ # The base schema is defined as a file in order to be customizable from other GeoNode instances JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/core_schema.json") -METADATA_HANDLERS = [ - 'geonode.metadata.handlers.CoreHandler', -] \ No newline at end of file +METADATA_HANDLERS = { + "base": "geonode.metadata.handlers.CoreHandler", +} \ No newline at end of file From da3b29f81d7969b96afb09e9760cc91959c1e78d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 21 Oct 2024 11:38:48 +0300 Subject: [PATCH 13/91] update metadata manager --- geonode/metadata/api/serializers.py | 11 - geonode/metadata/api/views.py | 18 +- geonode/metadata/handlers.py | 31 +- .../jsonschema_examples/base_schema.json | 308 ++++++++++++++++++ ...core_schema.json => base_schema_full.json} | 0 geonode/metadata/manager.py | 1 + geonode/metadata/settings.py | 4 +- 7 files changed, 344 insertions(+), 29 deletions(-) create mode 100644 geonode/metadata/jsonschema_examples/base_schema.json rename geonode/metadata/jsonschema_examples/{core_schema.json => base_schema_full.json} (100%) diff --git a/geonode/metadata/api/serializers.py b/geonode/metadata/api/serializers.py index 56cfb9e55d1..d1293536b4a 100644 --- a/geonode/metadata/api/serializers.py +++ b/geonode/metadata/api/serializers.py @@ -5,7 +5,6 @@ class MetadataSerializer(serializers.ModelSerializer): class Meta: model = ResourceBase fields = [ - "uuid", "title", "abstract", "purpose", @@ -20,17 +19,8 @@ class Meta: "supplemental_information", "data_quality_statement", "srid", - "csw_typename", - "csw_schema", - "csw_mdsource", - "csw_type", - "csw_anytext", - "csw_wkt_geometry", "metadata_uploaded", "metadata_uploaded_preserve", - "metadata_xml", - "popular_count", - "share_count", "featured", "was_published", "is_published", @@ -39,7 +29,6 @@ class Meta: "advertised", "thumbnail_url", "thumbnail_path", - "rating", "state", "sourcetype", "remote_typename", diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index bdcf5e19708..e30c811d870 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -34,8 +34,7 @@ class MetadataViewSet(ViewSet): serializer_class = MetadataSerializer def list(self, request): - serializer = self.serializer_class(many=True) - return Response(serializer.data) + pass # Get the JSON schema @action(detail=False, methods=['get']) @@ -57,9 +56,12 @@ def schema(self, request): def retrieve(self, request, pk=None): data = self.queryset.filter(pk=pk) - #serializer = self._get_and_validate_serializer(data=data) - # resource = metadata_manager.get_resource_base(data, self.serializer_class) - schema_instance = metadata_manager.build_schema_instance(data) - #serialized_resource = self.serializer_class(data=schema_instance) - #serialized_resource.is_valid(raise_exception=True) - return Response(schema_instance) \ No newline at end of file + + if data.exists(): + schema_instance = metadata_manager.build_schema_instance(data) + serialized_resource = self.serializer_class(data=schema_instance) + serialized_resource.is_valid(raise_exception=True) + return Response(serialized_resource.data) + else: + result = {"message": "The dataset was not found"} + return Response(result) \ No newline at end of file diff --git a/geonode/metadata/handlers.py b/geonode/metadata/handlers.py index b449af33dda..c69dd149718 100644 --- a/geonode/metadata/handlers.py +++ b/geonode/metadata/handlers.py @@ -22,6 +22,7 @@ from abc import ABCMeta, abstractmethod from geonode.base.models import ResourceBase from geonode.metadata.settings import JSONSCHEMA_BASE +from geonode.base.enumerations import ALL_LANGUAGES logger = logging.getLogger(__name__) @@ -65,7 +66,7 @@ def load_context(resource: ResourceBase, context: dict): pass -class CoreHandler(Handler): +class BaseHandler(Handler): """ The base handler builds a valid empty schema with the simple fields of the ResourceBase model @@ -73,16 +74,30 @@ class CoreHandler(Handler): def __init__(self): self.json_base_schema = JSONSCHEMA_BASE - self.basic_schema = None + self.base_schema = None def update_schema(self, jsonschema): with open(self.json_base_schema) as f: - self.basic_schema = json.load(f) + self.base_schema = json.load(f) # building the full base schema - for key, val in dict.items(self.basic_schema): - # add the base handler identity to the dictionary - val.update({"geonode:handler": "base"}) - jsonschema["properties"].update({key: val}) + for property, values in self.base_schema.items(): + + jsonschema["properties"].update({property: values}) + + # add the base handler identity to the dictionary if it doesn't exist + if "geonode:handler" not in values: + values.update({"geonode:handler": "base"}) + + # build the language choices + if property == "language": + values["oneOf"] = [] + for key, val in dict(ALL_LANGUAGES).items(): + langChoice = { + "const": key, + "title": val + } + values["oneOf"].append(langChoice) + return jsonschema @@ -91,7 +106,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): field_value = resource.values().first()[field_name] return field_value - + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): pass diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json new file mode 100644 index 00000000000..e3d146330f1 --- /dev/null +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -0,0 +1,308 @@ +{ + "uuid": { + "type": "string", + "description": "The UUID of the resource", + "maxLength": 36, + "ui:widget": "hidden", + "geonode:handler": "base" + }, + "title": { + "type": "string", + "title": "title", + "description": "Νame by which the cited resource is known", + "maxLength": 255, + "geonode:handler": "base" + }, + "abstract": { + "type": "string", + "title": "abstract", + "description": "Brief narrative summary of the content of the resource(s)", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "geonode:handler": "base" + }, + "purpose": { + "type": "string", + "title": "purpose", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500, + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "geonode:handler": "base" + }, + "alternate": { + "type": "string", + "maxLength": 255, + "ui:widget": "hidden" + }, + "date_type": { + "type": "string", + "title": "date type", + "maxLength": 255, + "enum": [ + "Creation", + "Publication", + "Revision" + ], + "default": "Publication" + }, + "edition": { + "type": "string", + "title": "edition", + "description": "Version of the cited resource", + "maxLength": 255 + }, + "attribution": { + "type": "string", + "title": "Attribution", + "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", + "maxLength": 2048 + }, + "doi": { + "type": "string", + "title": "DOI", + "description": "a DOI will be added by Admin before publication.", + "maxLength": 255 + }, + "maintenance_frequency": { + "type": "string", + "title": "maintenance frequency", + "description": "frequency with which modifications and deletions are made to the data after it is first produced", + "maxLength": 255, + "oneOf": [ + { + "const": "unknown", + "title": "frequency of maintenance for the data is not known" + }, + { + "const": "continual", + "title": "data is repeatedly and frequently updated" + }, + { + "const": "notPlanned", + "title": "there are no plans to update the data" + }, + { + "const": "daily", + "title": "data is updated each day" + }, + { + "const": "annually", + "title": "data is updated every year" + }, + { + "const": "asNeeded", + "title": "data is updated as deemed necessary" + }, + { + "const": "monthly", + "title": "data is updated each month" + }, + { + "const": "fortnightly", + "title": "data is updated every two weeks" + }, + { + "const": "irregular", + "title": "data is updated in intervals that are uneven in duration" + }, + { + "const": "weekly", + "title": "data is updated on a weekly basis" + }, + { + "const": "biannually", + "title": "data is updated twice each year" + }, + { + "const": "quarterly", + "title": "data is updated every three months" + } + ] + }, + "constraints_other": { + "type": "string", + "title": "Other constrains", + "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "language": { + "type": "string", + "title": "language", + "description": "Language used within the dataset", + "maxLength": 255, + "default": "eng" + }, + "supplemental_information": { + "type": "string", + "title": "supplemental information", + "description": "any other descriptive information about the dataset", + "maxLength": 2000, + "default": "No information provided", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "data_quality_statement": { + "type": "string", + "title": "data quality statement", + "description": "general explanation of the data producer's knowledge about the lineage of a dataset", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "srid": { + "type": "string", + "description": "The SRID of the resource", + "maxLength": 30, + "default": "EPSG:4326", + "ui:widget": "hidden" + }, + "metadata_uploaded_preserve": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "default": false + }, + "featured": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Featured", + "description": "Should this resource be advertised in home page?", + "default": false + }, + "was_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was published", + "description": "Previous Published state.", + "default": true + }, + "is_published": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Is published", + "description": "Should this resource be published and searchable?", + "default": true + }, + "was_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Was Approved", + "description": "Previous Approved state.", + "default": true + }, + "is_approved": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Approved", + "description": "Is this resource validated from a publisher or editor?", + "default": true + }, + "advertised": { + "$comment": "metadata XML specific fields", + "type": "boolean", + "title": "Advertised", + "description": "If False, will hide the resource from search results and catalog listings", + "default": true + }, + "thumbnail_url": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail url", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "thumbnail_path": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Thumbnail path", + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "state": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "State", + "description": "Hold the resource processing state.", + "default": "READY", + "maxLength": 16, + "oneOf": [ + { + "const": "READY", + "title": "READY" + }, + { + "const": "RUNNING", + "title": "RUNNING" + }, + { + "const": "PENDING", + "title": "PENDING" + }, + { + "const": "WAITING", + "title": "WAITING" + }, + { + "const": "INCOMPLETE", + "title": "INCOMPLETE" + }, + { + "const": "COMPLETE", + "title": "COMPLETE" + }, + { + "const": "INVALID", + "title": "INVALID" + }, + { + "const": "PROCESSED", + "title": "PROCESSED" + } + ] + }, + "sourcetype": { + "$comment": "fields necessary for the apis", + "type": "string", + "title": "Source Type", + "description": "The resource source type, which can be one of 'LOCAL', 'REMOTE' or 'COPYREMOTE'.", + "default": "LOCAL", + "maxLength": 16, + "oneOf": [ + { + "const": "LOCAL", + "title": "LOCAL" + }, + { + "const": "REMOTE", + "title": "REMOTE" + }, + { + "const": "COPYREMOTE", + "title": "COPYREMOTE" + } + ] + }, + "metadata_only": { + "$comment": "fields controlling security state", + "type": "boolean", + "title": "Metadata", + "description": "If true, will be excluded from search", + "default": false + } +} \ No newline at end of file diff --git a/geonode/metadata/jsonschema_examples/core_schema.json b/geonode/metadata/jsonschema_examples/base_schema_full.json similarity index 100% rename from geonode/metadata/jsonschema_examples/core_schema.json rename to geonode/metadata/jsonschema_examples/base_schema_full.json diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index fa82bf9886f..62eea911406 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -79,6 +79,7 @@ def build_schema_instance(self, resource): for fieldname, field in schema["properties"].items(): handler_id = field["geonode:handler"] handler = self.handlers[handler_id] + #TODO see if the resource exists content = handler.get_jsonschema_instance(resource, fieldname) self.instance[fieldname] = content diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 7025c699284..74120d35d9a 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -11,8 +11,8 @@ } # The base schema is defined as a file in order to be customizable from other GeoNode instances -JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/core_schema.json") +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/base_schema.json") METADATA_HANDLERS = { - "base": "geonode.metadata.handlers.CoreHandler", + "base": "geonode.metadata.handlers.BaseHandler", } \ No newline at end of file From 7be46be97170b6ecbc311094b2dc310ec1a43392 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 21 Oct 2024 16:17:16 +0300 Subject: [PATCH 14/91] update the metadata/{pk} to metadata/instance/{pk} --- geonode/metadata/api/views.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index e30c811d870..0a653d163a8 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -37,7 +37,9 @@ def list(self, request): pass # Get the JSON schema - @action(detail=False, methods=['get']) + @action(detail=False, + methods=['get'], + ) def schema(self, request): ''' The user is able to export her/his keys with @@ -53,10 +55,15 @@ def schema(self, request): response = {"Message": "Schema not found"} return Response(response) - def retrieve(self, request, pk=None): + # Get the JSON schema + @action(detail=False, + methods=['get'], + url_path="instance/(?P\d+)" + ) + def instance(self, request, pk=None): data = self.queryset.filter(pk=pk) - + if data.exists(): schema_instance = metadata_manager.build_schema_instance(data) serialized_resource = self.serializer_class(data=schema_instance) From 98542ee3a837b8d9e4a2c0a65d8903594d85b7ff Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 21 Oct 2024 17:21:37 +0300 Subject: [PATCH 15/91] update the /metadata/schema endpoint --- geonode/metadata/api/views.py | 4 +++- geonode/metadata/jsonschema_examples/base_schema.json | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 0a653d163a8..e8cda39fc90 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -37,10 +37,12 @@ def list(self, request): pass # Get the JSON schema + # A pk argument is set for futured multiple schemas @action(detail=False, methods=['get'], + url_path="schema(?:/(?P\d+))?" ) - def schema(self, request): + def schema(self, request, pk=None): ''' The user is able to export her/his keys with resource scope. diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index e3d146330f1..63de580376a 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -9,7 +9,7 @@ "title": { "type": "string", "title": "title", - "description": "Νame by which the cited resource is known", + "description": "Name by which the cited resource is known", "maxLength": 255, "geonode:handler": "base" }, From d99183e437a419a794c7b1dcc7dc36e9110b6d93 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 21 Oct 2024 17:04:38 +0200 Subject: [PATCH 16/91] Handlers refactoring, i18n --- .../{handlers.py => handlers/abstract.py} | 58 +------------ geonode/metadata/handlers/base.py | 84 +++++++++++++++++++ geonode/metadata/settings.py | 2 +- 3 files changed, 88 insertions(+), 56 deletions(-) rename geonode/metadata/{handlers.py => handlers/abstract.py} (55%) create mode 100644 geonode/metadata/handlers/base.py diff --git a/geonode/metadata/handlers.py b/geonode/metadata/handlers/abstract.py similarity index 55% rename from geonode/metadata/handlers.py rename to geonode/metadata/handlers/abstract.py index c69dd149718..460c6a1f90e 100644 --- a/geonode/metadata/handlers.py +++ b/geonode/metadata/handlers/abstract.py @@ -17,23 +17,21 @@ # ######################################################################### -import json import logging from abc import ABCMeta, abstractmethod from geonode.base.models import ResourceBase -from geonode.metadata.settings import JSONSCHEMA_BASE -from geonode.base.enumerations import ALL_LANGUAGES logger = logging.getLogger(__name__) -class Handler(metaclass=ABCMeta): + +class MetadataHandler(metaclass=ABCMeta): """ Handlers take care of reading, storing, encoding, decoding subschemas of the main Resource """ @abstractmethod - def update_schema(self, jsonschema: dict = {}): + def update_schema(self, jsonschema: dict): """ It is called by the MetadataManager when creating the JSON Schema It adds the subschema handled by the handler, and returns the @@ -64,53 +62,3 @@ def load_context(resource: ResourceBase, context: dict): Called before calls to update_resource in order to initialize info needed by the handler """ pass - - -class BaseHandler(Handler): - """ - The base handler builds a valid empty schema with the simple - fields of the ResourceBase model - """ - - def __init__(self): - self.json_base_schema = JSONSCHEMA_BASE - self.base_schema = None - - def update_schema(self, jsonschema): - with open(self.json_base_schema) as f: - self.base_schema = json.load(f) - # building the full base schema - for property, values in self.base_schema.items(): - - jsonschema["properties"].update({property: values}) - - # add the base handler identity to the dictionary if it doesn't exist - if "geonode:handler" not in values: - values.update({"geonode:handler": "base"}) - - # build the language choices - if property == "language": - values["oneOf"] = [] - for key, val in dict(ALL_LANGUAGES).items(): - langChoice = { - "const": key, - "title": val - } - values["oneOf"].append(langChoice) - - - return jsonschema - - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): - - field_value = resource.values().first()[field_name] - - return field_value - - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): - - pass - - def load_context(self, resource: ResourceBase, context: dict): - - pass diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py new file mode 100644 index 00000000000..1e5d01e8bfa --- /dev/null +++ b/geonode/metadata/handlers/base.py @@ -0,0 +1,84 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.settings import JSONSCHEMA_BASE +from geonode.base.enumerations import ALL_LANGUAGES +from django.utils.translation import gettext as _ + + +logger = logging.getLogger(__name__) + + +class BaseHandler(MetadataHandler): + """ + The base handler builds a valid empty schema with the simple + fields of the ResourceBase model + """ + + def __init__(self): + self.json_base_schema = JSONSCHEMA_BASE + self.base_schema = None + + def update_schema(self, jsonschema): + def localize(subschema: dict, annotation_name): + if annotation_name in subschema: + subschema[annotation_name] = _(subschema[annotation_name]) + + with open(self.json_base_schema) as f: + self.base_schema = json.load(f) + # building the full base schema + for subschema_name, subschema_def in self.base_schema.items(): + localize(subschema_def, 'title') + localize(subschema_def, 'abstract') + + jsonschema["properties"].update({subschema_name: subschema_def}) + + # add the base handler identity to the dictionary if it doesn't exist + if "geonode:handler" not in subschema_def: + subschema_def.update({"geonode:handler": "base"}) + + # build the language choices + if subschema_name == "language": + subschema_def["oneOf"] = [] + for key, val in dict(ALL_LANGUAGES).items(): + langChoice = { + "const": key, + "title": val + } + subschema_def["oneOf"].append(langChoice) + + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + + field_value = resource.values().first()[field_name] + + return field_value + + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + + pass + + def load_context(self, resource: ResourceBase, context: dict): + + pass diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 74120d35d9a..5edcb7fd752 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -14,5 +14,5 @@ JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/base_schema.json") METADATA_HANDLERS = { - "base": "geonode.metadata.handlers.BaseHandler", + "base": "geonode.metadata.handlers.base.BaseHandler", } \ No newline at end of file From 44238168fda0bbceb9bdccefb7d57dc9e65496d3 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 21 Oct 2024 18:09:52 +0200 Subject: [PATCH 17/91] Add TKeywords subschema --- geonode/metadata/handlers/thesaurus.py | 123 +++++++++++++++++++++++++ geonode/metadata/settings.py | 1 + 2 files changed, 124 insertions(+) create mode 100644 geonode/metadata/handlers/thesaurus.py diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py new file mode 100644 index 00000000000..606bf343463 --- /dev/null +++ b/geonode/metadata/handlers/thesaurus.py @@ -0,0 +1,123 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging + +from rest_framework.reverse import reverse + +from django.db.models import Q +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.settings import JSONSCHEMA_BASE +from geonode.base.enumerations import ALL_LANGUAGES + + +logger = logging.getLogger(__name__) + + +class TKeywordsHandler(MetadataHandler): + """ + The base handler builds a valid empty schema with the simple + fields of the ResourceBase model + """ + + def __init__(self): + self.json_base_schema = JSONSCHEMA_BASE + self.base_schema = None + + def update_schema(self, jsonschema): + + from geonode.base.models import Thesaurus + + # this query return the list of thesaurus X the list of localized titles + q = ( + Thesaurus.objects.filter(~Q(card_max=0)) + .values("identifier", "title", "description", "order", "card_min", "card_max", + "rel_thesaurus__label", "rel_thesaurus__lang") + .order_by("order") + ) + + thesauri = {} + for r in q.all(): + identifier = r["identifier"] + logger.info(f"Adding Thesaurus {identifier} to JSON Schema") + + thesaurus = {} + thesauri[identifier] = thesaurus + + thesaurus["type"] = "object" + + title = r["title"] ## todo i18n + thesaurus["title"] = title + thesaurus["description"] = r["description"] # not localized in db + + keywords = { + "type": "array", + "minItems": r["card_min"], + "maxItems": r["card_max"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "title": "keyword id", + "description": "The id of the keyword (usually a URI)", + }, + "label": { + "type": "string", + "title": "Label", + "description": "localized label for the keyword", + } + } + }, + "ui:options": { + 'geonode-ui:autocomplete': reverse("thesaurus_autocomplete") + } + } + + thesaurus["properties"] = {"keywords": keywords} + + tkeywords = { + "type": "object", + "title": _("Thesaurus keywords"), + "description": _("Keywords from controlled vocabularies"), + "geonode:handler": "thesaurus", + "properties": thesauri, + } + + jsonschema["tkeywords"] = tkeywords + + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + + field_value = resource.values().first()[field_name] + + return field_value + + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + + pass + + def load_context(self, resource: ResourceBase, context: dict): + + pass diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 5edcb7fd752..8aeedfa897f 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -15,4 +15,5 @@ METADATA_HANDLERS = { "base": "geonode.metadata.handlers.base.BaseHandler", + "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", } \ No newline at end of file From 4adabbefa2516e4fa2564700aa409ddbf6a44821 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 21 Oct 2024 18:15:25 +0200 Subject: [PATCH 18/91] Metadata TKeywords: fix max card --- geonode/metadata/handlers/thesaurus.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 606bf343463..9b8dbdd1b1f 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -72,8 +72,13 @@ def update_schema(self, jsonschema): keywords = { "type": "array", - "minItems": r["card_min"], - "maxItems": r["card_max"], + "minItems": r["card_min"] + } + + if r["card_max"] != -1: + keywords["maxItems"] = r["card_max"] + + keywords.update({ "items": { "type": "object", "properties": { @@ -92,7 +97,7 @@ def update_schema(self, jsonschema): "ui:options": { 'geonode-ui:autocomplete': reverse("thesaurus_autocomplete") } - } + }) thesaurus["properties"] = {"keywords": keywords} From 9450eab418c4b53349f4b80bb1fd3c36770e744d Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 21 Oct 2024 18:21:24 +0200 Subject: [PATCH 19/91] TKeywords: Fix schema --- geonode/metadata/handlers/thesaurus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 9b8dbdd1b1f..32fa19d9a76 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -109,7 +109,7 @@ def update_schema(self, jsonschema): "properties": thesauri, } - jsonschema["tkeywords"] = tkeywords + jsonschema["properties"]["tkeywords"] = tkeywords return jsonschema From 9bbed4e4116ddbc9e3b0d41d9c7a641a3545fc03 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 21 Oct 2024 18:30:16 +0200 Subject: [PATCH 20/91] Tkeywords: void get_jsonschema_instance --- geonode/metadata/handlers/thesaurus.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 32fa19d9a76..7cfd403670e 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -115,9 +115,7 @@ def update_schema(self, jsonschema): def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): - field_value = resource.values().first()[field_name] - - return field_value + return None def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): From 1b1f589a3d788077804d6304c5431b8fcba29d55 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 22 Oct 2024 18:13:42 +0200 Subject: [PATCH 21/91] TKeywords: Fix autocomplete; localization --- geonode/base/api/urls.py | 1 + geonode/base/api/views.py | 70 ++++++++++++++++++++++++-- geonode/base/urls.py | 6 --- geonode/base/views.py | 26 ---------- geonode/metadata/api/views.py | 16 +++--- geonode/metadata/handlers/abstract.py | 8 +-- geonode/metadata/handlers/base.py | 4 +- geonode/metadata/handlers/thesaurus.py | 10 ++-- geonode/metadata/manager.py | 26 +++++----- 9 files changed, 104 insertions(+), 63 deletions(-) diff --git a/geonode/base/api/urls.py b/geonode/base/api/urls.py index 69121b96f6f..a2575f8c36c 100644 --- a/geonode/base/api/urls.py +++ b/geonode/base/api/urls.py @@ -26,6 +26,7 @@ router.register(r"categories", views.TopicCategoryViewSet, "categories") router.register(r"keywords", views.HierarchicalKeywordViewSet, "keywords") router.register(r"tkeywords", views.ThesaurusKeywordViewSet, "tkeywords") +router.register(r"thesaurus", views.ThesaurusViewSet, "thesaurus") router.register(r"regions", views.RegionViewSet, "regions") urlpatterns = [] diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 8fbbdf71407..37f038fa0cd 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -24,17 +24,21 @@ from uuid import uuid4 from urllib.parse import urljoin, urlparse from PIL import Image +from dal import autocomplete from django.apps import apps from django.core.validators import URLValidator +from django.http import JsonResponse from django.shortcuts import get_object_or_404 from django.urls import reverse from django.conf import settings from django.db.models import Subquery, QuerySet from django.http.request import QueryDict from django.contrib.auth import get_user_model +from django.utils.translation import get_language +from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema +from drf_spectacular.utils import extend_schema, OpenApiParameter from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter @@ -54,7 +58,7 @@ from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.favorite.models import Favorite -from geonode.base.models import Configuration, ExtraMetadata, LinkedResource +from geonode.base.models import Configuration, ExtraMetadata, LinkedResource, ThesaurusKeywordLabel, Thesaurus from geonode.thumbs.exceptions import ThumbnailError from geonode.thumbs.thumbnails import create_thumbnail from geonode.thumbs.utils import _decode_base64, BASE64_PATTERN @@ -104,7 +108,7 @@ ) from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination -from geonode.base.utils import validate_extra_metadata +from geonode.base.utils import validate_extra_metadata, remove_country_from_languagecode import logging @@ -227,6 +231,66 @@ class ThesaurusKeywordViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveM pagination_class = GeoNodeApiPagination +class ThesaurusViewSet(DynamicModelViewSet): + + queryset = Thesaurus.objects.all() + serializer_class = ThesaurusKeywordSerializer + + @extend_schema( + methods=["get"], + description="API endpoint allowing to retrieve the published Resources.", + ) + @action( + detail=False, + methods=["get"], + url_path="(?P\d+)/keywords/autocomplete", # noqa + url_name="keywords_autocomplete", + ) + def tkeywords_autocomplete(self, request, thesaurusid): + + lang = get_language() + all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again + all_localized_keywords_qs = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=all_keywords_qs + ).values("keyword_id") + if not all_localized_keywords_qs.exists(): + lang = remove_country_from_languagecode(lang) + all_localized_keywords_qs = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=all_keywords_qs + ).values("keyword_id") + + # consider all the keywords that do not have a translation in the requested language + keywords_not_translated_qs = ( + ThesaurusKeywordLabel.objects.exclude(keyword_id__in=all_localized_keywords_qs) + .order_by("keyword_id") + .distinct("keyword_id") + .values("keyword_id") + ) + qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs) + if q:=request.query_params.get("q", None): + qs = qs.filter(label__istartswith=q) + + ret = [] + for tkl in qs.all(): + ret.append( + { + "id": tkl.keyword.pk, + "text": tkl.label, + "selected_text": tkl.label, + }) + for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).all(): + ret.append( + { + "id": tk.pk, + "text": f"! {tk.alt_label}", + "selected_text": f"! {tk.alt_label}", + }) + + return JsonResponse({"results":ret}) + + class TopicCategoryViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): """ API endpoint that lists categories. diff --git a/geonode/base/urls.py b/geonode/base/urls.py index 28e2b37aacf..cc0572f19eb 100644 --- a/geonode/base/urls.py +++ b/geonode/base/urls.py @@ -26,7 +26,6 @@ OwnerRightsRequestView, ResourceBaseAutocomplete, HierarchicalKeywordAutocomplete, - ThesaurusKeywordLabelAutocomplete, LinkedResourcesAutocomplete, ) @@ -57,11 +56,6 @@ ThesaurusAvailable.as_view(), name="thesaurus_available", ), - re_path( - r"^thesaurus_autocomplete/$", - ThesaurusKeywordLabelAutocomplete.as_view(), - name="thesaurus_autocomplete", - ), re_path( r"^datasets_autocomplete/$", DatasetsAutocomplete.as_view(), diff --git a/geonode/base/views.py b/geonode/base/views.py index 31f4bf2f677..b08ca239a68 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -334,32 +334,6 @@ class HierarchicalKeywordAutocomplete(SimpleSelect2View): filter_arg = "slug__icontains" -class ThesaurusKeywordLabelAutocomplete(autocomplete.Select2QuerySetView): - def get_queryset(self): - thesaurus = settings.THESAURUS - tname = thesaurus["name"] - lang = "en" - - # Filters thesaurus results based on thesaurus name and language - qs = ThesaurusKeywordLabel.objects.all().filter(keyword__thesaurus__identifier=tname, lang=lang) - - if self.q: - qs = qs.filter(label__icontains=self.q) - - return qs - - # Overides the get results method to return custom json to frontend - def get_results(self, context): - return [ - { - "id": self.get_result_value(result.keyword), - "text": self.get_result_label(result), - "selected_text": self.get_selected_result_label(result), - } - for result in context["object_list"] - ] - - class DatasetsAutocomplete(SimpleSelect2View): model = Dataset filter_arg = "title__icontains" diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index e8cda39fc90..17fd8be3b03 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -17,14 +17,17 @@ # ######################################################################### -from geonode.metadata.manager import metadata_manager -from geonode.metadata.api.serializers import MetadataSerializer from rest_framework.viewsets import ViewSet -from geonode.base.models import ResourceBase from rest_framework.decorators import action -from django.http import JsonResponse from rest_framework.response import Response +from django.utils.translation.trans_real import get_language_from_request + +from geonode.base.models import ResourceBase +from geonode.metadata.manager import metadata_manager +from geonode.metadata.api.serializers import MetadataSerializer + + class MetadataViewSet(ViewSet): """ Simple viewset that return the metadata JSON schema @@ -47,8 +50,9 @@ def schema(self, request, pk=None): The user is able to export her/his keys with resource scope. ''' - - schema = metadata_manager.get_schema() + + lang = get_language_from_request(request)[:2] + schema = metadata_manager.get_schema(lang) if schema: return Response(schema) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 460c6a1f90e..ef14ba1bbf9 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -31,7 +31,7 @@ class MetadataHandler(metaclass=ABCMeta): """ @abstractmethod - def update_schema(self, jsonschema: dict): + def update_schema(self, jsonschema: dict, lang=None): """ It is called by the MetadataManager when creating the JSON Schema It adds the subschema handled by the handler, and returns the @@ -40,7 +40,7 @@ def update_schema(self, jsonschema: dict): pass @abstractmethod - def get_jsonschema_instance(resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): """ Called when reading metadata, returns the instance of the sub-schema associated with the field field_name. @@ -48,7 +48,7 @@ def get_jsonschema_instance(resource: ResourceBase, field_name: str): pass @abstractmethod - def update_resource(resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): """ Called when persisting data, updates the field field_name of the resource with the content content, where json_instance is the full JSON Schema instance, @@ -57,7 +57,7 @@ def update_resource(resource: ResourceBase, field_name: str, content: dict, json pass @abstractmethod - def load_context(resource: ResourceBase, context: dict): + def load_context(self, resource: ResourceBase, context: dict): """ Called before calls to update_resource in order to initialize info needed by the handler """ diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 1e5d01e8bfa..272b1651d22 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -39,7 +39,7 @@ def __init__(self): self.json_base_schema = JSONSCHEMA_BASE self.base_schema = None - def update_schema(self, jsonschema): + def update_schema(self, jsonschema, lang=None): def localize(subschema: dict, annotation_name): if annotation_name in subschema: subschema[annotation_name] = _(subschema[annotation_name]) @@ -51,7 +51,7 @@ def localize(subschema: dict, annotation_name): localize(subschema_def, 'title') localize(subschema_def, 'abstract') - jsonschema["properties"].update({subschema_name: subschema_def}) + jsonschema["properties"][subschema_name] = subschema_def # add the base handler identity to the dictionary if it doesn't exist if "geonode:handler" not in subschema_def: diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 7cfd403670e..afb4dc90881 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -44,14 +44,14 @@ def __init__(self): self.json_base_schema = JSONSCHEMA_BASE self.base_schema = None - def update_schema(self, jsonschema): + def update_schema(self, jsonschema, lang=None): from geonode.base.models import Thesaurus # this query return the list of thesaurus X the list of localized titles q = ( Thesaurus.objects.filter(~Q(card_max=0)) - .values("identifier", "title", "description", "order", "card_min", "card_max", + .values("id", "identifier", "title", "description", "order", "card_min", "card_max", "rel_thesaurus__label", "rel_thesaurus__lang") .order_by("order") ) @@ -59,7 +59,7 @@ def update_schema(self, jsonschema): thesauri = {} for r in q.all(): identifier = r["identifier"] - logger.info(f"Adding Thesaurus {identifier} to JSON Schema") + logger.info(f"Adding Thesaurus {identifier} to JSON Schema lang {lang}") thesaurus = {} thesauri[identifier] = thesaurus @@ -95,7 +95,9 @@ def update_schema(self, jsonschema): } }, "ui:options": { - 'geonode-ui:autocomplete': reverse("thesaurus_autocomplete") + 'geonode-ui:autocomplete': reverse( + "thesaurus-keywords_autocomplete", + kwargs={"thesaurusid": r["id"]}) } }) diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 62eea911406..e6c22b02eba 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -18,17 +18,22 @@ ######################################################################### import logging -from abc import ABCMeta, abstractmethod +from abc import ABCMeta + +from django.utils.translation import gettext as _ + from geonode.metadata.settings import MODEL_SCHEMA from geonode.metadata.api.serializers import MetadataSerializer -from geonode.metadata.registry import metadata_registry + logger = logging.getLogger(__name__) + class MetadataManagerInterface(metaclass=ABCMeta): pass + class MetadataManager(MetadataManagerInterface): """ The metadata manager is the bridge between the API and the geonode model. @@ -52,22 +57,18 @@ def add_handler(self, handler_id, handler): self.handlers[handler_id] = handler_instance - def build_schema(self): + def build_schema(self, lang=None): + self.schema = self.jsonschema.copy() + self.schema["title"] = _(self.schema["title"]) for handler in self.handlers.values(): - - if self.schema: - subschema = handler.update_schema(self.jsonschema)["properties"] - # Update the properties key of the current schema with the properties of the new handler - self.schema["properties"].update(subschema) - else: - self.schema = handler.update_schema(self.jsonschema) + self.schema = handler.update_schema(self.schema, lang) return self.schema - def get_schema(self): + def get_schema(self, lang=None): if not self.schema: - self.build_schema() + self.build_schema(lang) return self.schema @@ -94,4 +95,5 @@ def resource_base_serialization(self, resource): serialized_data = serializer(resource, many=True).data return serialized_data + metadata_manager = MetadataManager() \ No newline at end of file From 028067c5bfafd48ecb1ff2c6f85b06c8533c45fc Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 22 Oct 2024 19:56:27 +0200 Subject: [PATCH 22/91] Thesaurus schema: Improve localization --- geonode/metadata/handlers/thesaurus.py | 48 ++++++++++++++++---------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index afb4dc90881..76983c92715 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -56,29 +56,38 @@ def update_schema(self, jsonschema, lang=None): .order_by("order") ) - thesauri = {} + # We don't know if we have the title for the requested lang: so let's loop on all retrieved translations + collected_thesauri = {} for r in q.all(): identifier = r["identifier"] - logger.info(f"Adding Thesaurus {identifier} to JSON Schema lang {lang}") - + thesaurus = collected_thesauri.get(identifier, {}) + if not thesaurus: + # init + logger.debug(f"Initializing Thesaurus {identifier} JSON Schema") + collected_thesauri[identifier] = thesaurus + thesaurus["id"] = r["id"] + thesaurus["card"] = {} + thesaurus["card"]["minItems"] = r["card_min"] + if r["card_max"] != -1: + thesaurus["card"]["maxItems"] = r["card_max"] + thesaurus["title"] = r["title"] # default title + thesaurus["description"] = r["description"] # not localized in db + + # check if this is the localized record we're looking for + if r["rel_thesaurus__lang"] == lang: + logger.debug(f"Localizing Thesaurus {identifier} JSON Schema for lang {lang}") + thesaurus["title"] = r["rel_thesaurus__label"] + + # copy info to json schema + thesauri = {} + for id,ct in collected_thesauri.items(): thesaurus = {} - thesauri[identifier] = thesaurus - thesaurus["type"] = "object" - - title = r["title"] ## todo i18n - thesaurus["title"] = title - thesaurus["description"] = r["description"] # not localized in db + thesaurus["title"] = ct["title"] + thesaurus["description"] = ct["description"] keywords = { "type": "array", - "minItems": r["card_min"] - } - - if r["card_max"] != -1: - keywords["maxItems"] = r["card_max"] - - keywords.update({ "items": { "type": "object", "properties": { @@ -97,11 +106,12 @@ def update_schema(self, jsonschema, lang=None): "ui:options": { 'geonode-ui:autocomplete': reverse( "thesaurus-keywords_autocomplete", - kwargs={"thesaurusid": r["id"]}) + kwargs={"thesaurusid": ct["id"]}) } - }) - + } + keywords.update(ct["card"]) thesaurus["properties"] = {"keywords": keywords} + thesauri[id] = thesaurus tkeywords = { "type": "object", From b161aa0b9cf9b7105694c49ddbc5e356866e472a Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 23 Oct 2024 10:34:05 +0200 Subject: [PATCH 23/91] TKeywords: Improve autocomplete --- geonode/base/api/views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 37f038fa0cd..3656b33db5d 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -277,15 +277,13 @@ def tkeywords_autocomplete(self, request, thesaurusid): ret.append( { "id": tkl.keyword.pk, - "text": tkl.label, - "selected_text": tkl.label, + "label": tkl.label, }) for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).all(): ret.append( { "id": tk.pk, - "text": f"! {tk.alt_label}", - "selected_text": f"! {tk.alt_label}", + "label": f"! {tk.alt_label}", }) return JsonResponse({"results":ret}) From 963f480ebf1c079bc943b47dac2f059961163992 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 22 Oct 2024 17:59:33 +0300 Subject: [PATCH 24/91] adding PUT functionality to the endpoint metadata/instance/{pk} --- geonode/metadata/api/serializers.py | 7 +------ geonode/metadata/api/views.py | 31 +++++++++++++++++++++-------- geonode/metadata/handlers/base.py | 7 ++++--- geonode/metadata/manager.py | 27 ++++++++++++++----------- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/geonode/metadata/api/serializers.py b/geonode/metadata/api/serializers.py index d1293536b4a..73745bcbacc 100644 --- a/geonode/metadata/api/serializers.py +++ b/geonode/metadata/api/serializers.py @@ -19,7 +19,6 @@ class Meta: "supplemental_information", "data_quality_statement", "srid", - "metadata_uploaded", "metadata_uploaded_preserve", "featured", "was_published", @@ -31,9 +30,5 @@ class Meta: "thumbnail_path", "state", "sourcetype", - "remote_typename", - "dirty_state", - "resource_type", "metadata_only", - "subtype", - ] \ No newline at end of file + ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 17fd8be3b03..229336711b5 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -63,18 +63,33 @@ def schema(self, request, pk=None): # Get the JSON schema @action(detail=False, - methods=['get'], + methods=['get', 'put'], url_path="instance/(?P\d+)" ) def instance(self, request, pk=None): + + try: + resource = ResourceBase.objects.get(pk=pk) + + # schema_instance is defined outside of the if statement in order to be used + # also by the PUT method + schema_instance = metadata_manager.build_schema_instance(resource) - data = self.queryset.filter(pk=pk) + if request.method == 'GET': + serialized_resource = self.serializer_class(data=schema_instance) + serialized_resource.is_valid(raise_exception=True) + + return Response(serialized_resource.data) + + elif request.method == 'PUT': - if data.exists(): - schema_instance = metadata_manager.build_schema_instance(data) - serialized_resource = self.serializer_class(data=schema_instance) - serialized_resource.is_valid(raise_exception=True) - return Response(serialized_resource.data) - else: + content = request.data + serialized_content = self.serializer_class(data=content) + serialized_content.is_valid(raise_exception=True) + result = metadata_manager.update_schema_instance(resource, serialized_content.data, schema_instance) + + return Response(result) + + except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} return Response(result) \ No newline at end of file diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 272b1651d22..67ca52f3076 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -71,13 +71,14 @@ def localize(subschema: dict, annotation_name): def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): - field_value = resource.values().first()[field_name] + field_value = getattr(resource, field_name) return field_value def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): - - pass + + if field_name in content: + setattr(resource, field_name, content[field_name]) def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index e6c22b02eba..eb4860715d1 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -23,8 +23,7 @@ from django.utils.translation import gettext as _ from geonode.metadata.settings import MODEL_SCHEMA -from geonode.metadata.api.serializers import MetadataSerializer - +from geonode.metadata.registry import metadata_registry logger = logging.getLogger(__name__) @@ -46,7 +45,6 @@ class MetadataManager(MetadataManagerInterface): def __init__(self): self.jsonschema = MODEL_SCHEMA self.schema = None - self.serializer_class = MetadataSerializer self.instance = {} self.handlers = {} @@ -74,26 +72,31 @@ def get_schema(self, lang=None): def build_schema_instance(self, resource): - # serialized_resource = self.get_resource_base(resource) schema = self.get_schema() for fieldname, field in schema["properties"].items(): handler_id = field["geonode:handler"] handler = self.handlers[handler_id] - #TODO see if the resource exists content = handler.get_jsonschema_instance(resource, fieldname) self.instance[fieldname] = content return self.instance - def resource_base_serialization(self, resource): - """ - Get a serialized dataset from the ResourceBase model - """ - serializer = self.serializer_class + def update_schema_instance(self, resource, content, json_instance): + + schema = self.get_schema() + + for fieldname, field in schema["properties"].items(): + handler_id = field["geonode:handler"] + handler = self.handlers[handler_id] + handler.update_resource(resource, fieldname, content, json_instance) + + try: + resource.save() + return {"message": "The resource was updated successfully"} - serialized_data = serializer(resource, many=True).data - return serialized_data + except: + return {"message": "Something went wrong... The resource was not updated"} metadata_manager = MetadataManager() \ No newline at end of file From 6fc00473be7e422f459bee885711ba2931e08f33 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 23 Oct 2024 11:07:23 +0300 Subject: [PATCH 25/91] rename the view of metadata/instance/{pk} endpoint --- geonode/metadata/api/views.py | 4 +--- geonode/metadata/handlers/base.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 229336711b5..64a576a3d7d 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -66,13 +66,11 @@ def schema(self, request, pk=None): methods=['get', 'put'], url_path="instance/(?P\d+)" ) - def instance(self, request, pk=None): + def schema_instance(self, request, pk=None): try: resource = ResourceBase.objects.get(pk=pk) - # schema_instance is defined outside of the if statement in order to be used - # also by the PUT method schema_instance = metadata_manager.build_schema_instance(resource) if request.method == 'GET': diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 67ca52f3076..9e2052d4789 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -78,6 +78,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): if field_name in content: + # insert the content value to the corresponding field_name setattr(resource, field_name, content[field_name]) def load_context(self, resource: ResourceBase, context: dict): From 612a1f7cb0cd5e25894d597b46d67cb6fdffd94d Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 23 Oct 2024 12:27:13 +0200 Subject: [PATCH 26/91] TKeywords: Improve autocomplete --- geonode/base/api/views.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 3656b33db5d..fb1edde5b84 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -252,23 +252,24 @@ def tkeywords_autocomplete(self, request, thesaurusid): all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again - all_localized_keywords_qs = ThesaurusKeywordLabel.objects.filter( + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter( lang=lang, keyword_id__in=all_keywords_qs ).values("keyword_id") - if not all_localized_keywords_qs.exists(): + if not localized_k_ids_qs.exists(): lang = remove_country_from_languagecode(lang) - all_localized_keywords_qs = ThesaurusKeywordLabel.objects.filter( + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter( lang=lang, keyword_id__in=all_keywords_qs ).values("keyword_id") # consider all the keywords that do not have a translation in the requested language keywords_not_translated_qs = ( - ThesaurusKeywordLabel.objects.exclude(keyword_id__in=all_localized_keywords_qs) - .order_by("keyword_id") - .distinct("keyword_id") - .values("keyword_id") + all_keywords_qs.exclude(id__in=localized_k_ids_qs) + .order_by("id") + .distinct("id") + .values("id") ) - qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs) + + qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).order_by("label") if q:=request.query_params.get("q", None): qs = qs.filter(label__istartswith=q) @@ -276,13 +277,13 @@ def tkeywords_autocomplete(self, request, thesaurusid): for tkl in qs.all(): ret.append( { - "id": tkl.keyword.pk, + "id": tkl.keyword.about, "label": tkl.label, }) - for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).all(): + for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).order_by("alt_label").all(): ret.append( { - "id": tk.pk, + "id": tk.about, "label": f"! {tk.alt_label}", }) From f0ea8e8963fd17ece5188476552db5c1f5935710 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 23 Oct 2024 19:13:24 +0200 Subject: [PATCH 27/91] TKeywords: move tkeywords just under category field --- geonode/metadata/handlers/thesaurus.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 76983c92715..d93ba578c33 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -121,7 +121,14 @@ def update_schema(self, jsonschema, lang=None): "properties": thesauri, } - jsonschema["properties"]["tkeywords"] = tkeywords + # add thesauri after category + ret_properties = {} + for key,val in jsonschema["properties"].items(): + ret_properties[key] = val + if key == "category": + ret_properties["tkeywords"] = tkeywords + + jsonschema["properties"] = ret_properties return jsonschema From 8e4bd6a31570b43394704cb833665079ced472d6 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 23 Oct 2024 19:17:36 +0200 Subject: [PATCH 28/91] Many improvements and addings to the base handler --- geonode/metadata/handlers/base.py | 79 ++++-- .../jsonschema_examples/base_schema.json | 265 ++++-------------- geonode/metadata/manager.py | 33 ++- 3 files changed, 127 insertions(+), 250 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 9e2052d4789..eb1ef8860ec 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -19,16 +19,56 @@ import json import logging -from geonode.base.models import ResourceBase +from geonode.base.models import ResourceBase, TopicCategory, License, Region from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE -from geonode.base.enumerations import ALL_LANGUAGES +from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES from django.utils.translation import gettext as _ logger = logging.getLogger(__name__) +class CategorySubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + # subschema["title"] = _("topiccategory") + subschema["oneOf"] = [{"const": tc.identifier,"title": tc.gn_description, "description": tc.description} + for tc in TopicCategory.objects.order_by("gn_description")] + +class FrequencySubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": key,"title": val} + for key, val in dict(UPDATE_FREQUENCIES).items()] + +class LanguageSubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": key,"title": val} + for key, val in dict(ALL_LANGUAGES).items()] + +class LicenseSubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": tc.identifier,"title": tc.name, "description": tc.description} + for tc in License.objects.order_by("name")] + +class RegionSubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["items"]["anyOf"] = [{"const": tc.code,"title": tc.name} + for tc in Region.objects.order_by("name")] + +SUBHANDLERS = { + "category": CategorySubHandler, + "language": LanguageSubHandler, + "license": LicenseSubHandler, + "maintenance_frequency": FrequencySubHandler, + "regions": RegionSubHandler, +} + + class BaseHandler(MetadataHandler): """ The base handler builds a valid empty schema with the simple @@ -47,25 +87,21 @@ def localize(subschema: dict, annotation_name): with open(self.json_base_schema) as f: self.base_schema = json.load(f) # building the full base schema - for subschema_name, subschema_def in self.base_schema.items(): - localize(subschema_def, 'title') - localize(subschema_def, 'abstract') - - jsonschema["properties"][subschema_name] = subschema_def - - # add the base handler identity to the dictionary if it doesn't exist - if "geonode:handler" not in subschema_def: - subschema_def.update({"geonode:handler": "base"}) - - # build the language choices - if subschema_name == "language": - subschema_def["oneOf"] = [] - for key, val in dict(ALL_LANGUAGES).items(): - langChoice = { - "const": key, - "title": val - } - subschema_def["oneOf"].append(langChoice) + for property_name, subschema in self.base_schema.items(): + localize(subschema, 'title') + localize(subschema, 'abstract') + + jsonschema["properties"][property_name] = subschema + + # add the base handler info to the dictionary if it doesn't exist + if "geonode:handler" not in subschema: + subschema.update({"geonode:handler": "base"}) + + # perform further specific initializations + + if subhandler := SUBHANDLERS.get(property_name, None): + logger.debug(f"Running subhandler for base field {property_name}") + subhandler.update_subschema(subschema, lang) return jsonschema @@ -84,3 +120,4 @@ def update_resource(self, resource: ResourceBase, field_name: str, content: dict def load_context(self, resource: ResourceBase, context: dict): pass + diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 63de580376a..6e36cf5b0a8 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -3,7 +3,8 @@ "type": "string", "description": "The UUID of the resource", "maxLength": 36, - "ui:widget": "hidden", + "readOnly": true, + "NO_ui:widget": "hidden", "geonode:handler": "base" }, "title": { @@ -24,21 +25,10 @@ }, "geonode:handler": "base" }, - "purpose": { - "type": "string", - "title": "purpose", - "description": "Summary of the intentions with which the resource(s) was developed", - "maxLength": 500, - "ui:options": { - "widget": "textarea", - "rows": 5 - }, - "geonode:handler": "base" - }, - "alternate": { + "date": { "type": "string", - "maxLength": 255, - "ui:widget": "hidden" + "format": "date-time", + "title": "Date" }, "date_type": { "type": "string", @@ -51,18 +41,62 @@ ], "default": "Publication" }, - "edition": { + "category": { "type": "string", - "title": "edition", - "description": "Version of the cited resource", + "title": "category", + "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets", "maxLength": 255 }, + "language": { + "type": "string", + "title": "language", + "description": "Language used within the dataset", + "maxLength": 255, + "default": "eng" + }, + "license": { + "type": "string", + "title": "license", + "description": "license of the dataset", + "maxLength": 255, + "default": "eng" + }, "attribution": { "type": "string", "title": "Attribution", "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", "maxLength": 2048 }, + "regions": { + "type": "array", + "title": "Regions", + "description": "keyword identifies a location", + "items": { + "type": "string" + } + }, + "purpose": { + "type": "string", + "title": "purpose", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500, + "ui:options": { + "widget": "textarea", + "rows": 5 + }, + "geonode:handler": "base" + }, + "alternate": { + "type": "string", + "maxLength": 255, + "ui:widget": "hidden" + }, + "edition": { + "type": "string", + "title": "edition", + "description": "Version of the cited resource", + "maxLength": 255 + }, "doi": { "type": "string", "title": "DOI", @@ -73,57 +107,7 @@ "type": "string", "title": "maintenance frequency", "description": "frequency with which modifications and deletions are made to the data after it is first produced", - "maxLength": 255, - "oneOf": [ - { - "const": "unknown", - "title": "frequency of maintenance for the data is not known" - }, - { - "const": "continual", - "title": "data is repeatedly and frequently updated" - }, - { - "const": "notPlanned", - "title": "there are no plans to update the data" - }, - { - "const": "daily", - "title": "data is updated each day" - }, - { - "const": "annually", - "title": "data is updated every year" - }, - { - "const": "asNeeded", - "title": "data is updated as deemed necessary" - }, - { - "const": "monthly", - "title": "data is updated each month" - }, - { - "const": "fortnightly", - "title": "data is updated every two weeks" - }, - { - "const": "irregular", - "title": "data is updated in intervals that are uneven in duration" - }, - { - "const": "weekly", - "title": "data is updated on a weekly basis" - }, - { - "const": "biannually", - "title": "data is updated twice each year" - }, - { - "const": "quarterly", - "title": "data is updated every three months" - } - ] + "maxLength": 255 }, "constraints_other": { "type": "string", @@ -134,13 +118,6 @@ "rows": 5 } }, - "language": { - "type": "string", - "title": "language", - "description": "Language used within the dataset", - "maxLength": 255, - "default": "eng" - }, "supplemental_information": { "type": "string", "title": "supplemental information", @@ -168,141 +145,5 @@ "maxLength": 30, "default": "EPSG:4326", "ui:widget": "hidden" - }, - "metadata_uploaded_preserve": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "default": false - }, - "featured": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Featured", - "description": "Should this resource be advertised in home page?", - "default": false - }, - "was_published": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Was published", - "description": "Previous Published state.", - "default": true - }, - "is_published": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Is published", - "description": "Should this resource be published and searchable?", - "default": true - }, - "was_approved": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Was Approved", - "description": "Previous Approved state.", - "default": true - }, - "is_approved": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Approved", - "description": "Is this resource validated from a publisher or editor?", - "default": true - }, - "advertised": { - "$comment": "metadata XML specific fields", - "type": "boolean", - "title": "Advertised", - "description": "If False, will hide the resource from search results and catalog listings", - "default": true - }, - "thumbnail_url": { - "$comment": "fields necessary for the apis", - "type": "string", - "title": "Thumbnail url", - "ui:options": { - "widget": "textarea", - "rows": 5 - } - }, - "thumbnail_path": { - "$comment": "fields necessary for the apis", - "type": "string", - "title": "Thumbnail path", - "ui:options": { - "widget": "textarea", - "rows": 5 - } - }, - "state": { - "$comment": "fields necessary for the apis", - "type": "string", - "title": "State", - "description": "Hold the resource processing state.", - "default": "READY", - "maxLength": 16, - "oneOf": [ - { - "const": "READY", - "title": "READY" - }, - { - "const": "RUNNING", - "title": "RUNNING" - }, - { - "const": "PENDING", - "title": "PENDING" - }, - { - "const": "WAITING", - "title": "WAITING" - }, - { - "const": "INCOMPLETE", - "title": "INCOMPLETE" - }, - { - "const": "COMPLETE", - "title": "COMPLETE" - }, - { - "const": "INVALID", - "title": "INVALID" - }, - { - "const": "PROCESSED", - "title": "PROCESSED" - } - ] - }, - "sourcetype": { - "$comment": "fields necessary for the apis", - "type": "string", - "title": "Source Type", - "description": "The resource source type, which can be one of 'LOCAL', 'REMOTE' or 'COPYREMOTE'.", - "default": "LOCAL", - "maxLength": 16, - "oneOf": [ - { - "const": "LOCAL", - "title": "LOCAL" - }, - { - "const": "REMOTE", - "title": "REMOTE" - }, - { - "const": "COPYREMOTE", - "title": "COPYREMOTE" - } - ] - }, - "metadata_only": { - "$comment": "fields controlling security state", - "type": "boolean", - "title": "Metadata", - "description": "If true, will be excluded from search", - "default": false } } \ No newline at end of file diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index eb4860715d1..773f4f2a093 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -18,6 +18,7 @@ ######################################################################### import logging +import copy from abc import ABCMeta from django.utils.translation import gettext as _ @@ -43,44 +44,42 @@ class MetadataManager(MetadataManagerInterface): """ def __init__(self): - self.jsonschema = MODEL_SCHEMA - self.schema = None - self.instance = {} + self.root_schema = MODEL_SCHEMA + self.cached_schema = None self.handlers = {} def add_handler(self, handler_id, handler): - - handler_instance = handler() - - self.handlers[handler_id] = handler_instance - + self.handlers[handler_id] = handler() def build_schema(self, lang=None): - self.schema = self.jsonschema.copy() - self.schema["title"] = _(self.schema["title"]) + schema = copy.deepcopy(self.root_schema) + schema["title"] = _(schema["title"]) for handler in self.handlers.values(): - self.schema = handler.update_schema(self.schema, lang) + schema = handler.update_schema(schema, lang) - return self.schema + return schema def get_schema(self, lang=None): - if not self.schema: - self.build_schema(lang) + return self.build_schema(lang) + #### we dont want caching for the moment + if not self.cached_schema: + self.cached_schema = self.build_schema(lang) - return self.schema + return self.cached_schema def build_schema_instance(self, resource): schema = self.get_schema() + instance = {} for fieldname, field in schema["properties"].items(): handler_id = field["geonode:handler"] handler = self.handlers[handler_id] content = handler.get_jsonschema_instance(resource, fieldname) - self.instance[fieldname] = content + instance[fieldname] = content - return self.instance + return instance def update_schema_instance(self, resource, content, json_instance): From 00d3c93c0ded04aac857d8d2c32cb35aa244aa47 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 23 Oct 2024 19:53:18 +0200 Subject: [PATCH 29/91] Some more improvements and addings to the base handler --- geonode/metadata/handlers/base.py | 18 +++- .../jsonschema_examples/base_schema.json | 87 ++++++++++++------- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index eb1ef8860ec..31ffa6c1346 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -19,7 +19,8 @@ import json import logging -from geonode.base.models import ResourceBase, TopicCategory, License, Region +from geonode.base.models import ResourceBase, TopicCategory, License, Region, RestrictionCodeType, \ + SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES @@ -60,12 +61,27 @@ def update_subschema(cls, subschema, lang=None): subschema["items"]["anyOf"] = [{"const": tc.code,"title": tc.name} for tc in Region.objects.order_by("name")] +class RestrictionsSubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} + for tc in RestrictionCodeType.objects.order_by("identifier")] + +class SpatialRepresentationTypeSubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} + for tc in SpatialRepresentationType.objects.order_by("identifier")] + + SUBHANDLERS = { "category": CategorySubHandler, "language": LanguageSubHandler, "license": LicenseSubHandler, "maintenance_frequency": FrequencySubHandler, "regions": RegionSubHandler, + "restriction_code_type": RestrictionsSubHandler, + "spatial_representation_type": SpatialRepresentationTypeSubHandler, } diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 6e36cf5b0a8..45e14206500 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -41,6 +41,16 @@ ], "default": "Publication" }, + "group": { + "type": "string", + "title": "group", + "todo": true + }, + "keywords": { + "type": "string", + "title": "keywords", + "todo": true + }, "category": { "type": "string", "title": "category", @@ -75,21 +85,30 @@ "type": "string" } }, - "purpose": { + "data_quality_statement": { "type": "string", - "title": "purpose", - "description": "Summary of the intentions with which the resource(s) was developed", - "maxLength": 500, + "title": "data quality statement", + "description": "general explanation of the data producer's knowledge about the lineage of a dataset", + "maxLength": 2000, "ui:options": { "widget": "textarea", "rows": 5 - }, - "geonode:handler": "base" + } }, - "alternate": { + "restriction_code_type": { "type": "string", - "maxLength": 255, - "ui:widget": "hidden" + "title": "restriction_code_type", + "description": "limitation(s) placed upon the access or use of the data.", + "maxLength": 255 + }, + "constraints_other": { + "type": "string", + "title": "Other constrains", + "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", + "ui:options": { + "widget": "textarea", + "rows": 5 + } }, "edition": { "type": "string", @@ -103,20 +122,16 @@ "description": "a DOI will be added by Admin before publication.", "maxLength": 255 }, - "maintenance_frequency": { - "type": "string", - "title": "maintenance frequency", - "description": "frequency with which modifications and deletions are made to the data after it is first produced", - "maxLength": 255 - }, - "constraints_other": { + "purpose": { "type": "string", - "title": "Other constrains", - "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", + "title": "purpose", + "description": "Summary of the intentions with which the resource(s) was developed", + "maxLength": 500, "ui:options": { "widget": "textarea", "rows": 5 - } + }, + "geonode:handler": "base" }, "supplemental_information": { "type": "string", @@ -129,21 +144,29 @@ "rows": 5 } }, - "data_quality_statement": { + "temporal_extent_start": { "type": "string", - "title": "data quality statement", - "description": "general explanation of the data producer's knowledge about the lineage of a dataset", - "maxLength": 2000, - "ui:options": { - "widget": "textarea", - "rows": 5 - } + "format": "date-time", + "title": "temporal_extent_start", + "description": "time period covered by the content of the dataset (start)" }, - "srid": { + "temporal_extent_end": { "type": "string", - "description": "The SRID of the resource", - "maxLength": 30, - "default": "EPSG:4326", - "ui:widget": "hidden" + "format": "date-time", + "title": "temporal_extent_end", + "description": "time period covered by the content of the dataset (end)" + }, + "maintenance_frequency": { + "type": "string", + "title": "maintenance frequency", + "description": "frequency with which modifications and deletions are made to the data after it is first produced", + "maxLength": 255 + }, + "spatial_representation_type": { + "type": "string", + "title": "spatial_representation_type", + "description": "method used to represent geographic information in the dataset.", + "maxLength": 255 } + } \ No newline at end of file From ed7df68e7b13a47ad752c61c26c16a0c539f24b7 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 13:27:27 +0200 Subject: [PATCH 30/91] Return proper json schema instance --- geonode/metadata/api/views.py | 10 +-- geonode/metadata/handlers/base.py | 65 +++++++++++++++++-- .../jsonschema_examples/base_schema.json | 8 +-- 3 files changed, 62 insertions(+), 21 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 64a576a3d7d..4f7ce898fd7 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # ######################################################################### - +from django.http import JsonResponse from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response @@ -70,14 +70,10 @@ def schema_instance(self, request, pk=None): try: resource = ResourceBase.objects.get(pk=pk) - schema_instance = metadata_manager.build_schema_instance(resource) if request.method == 'GET': - serialized_resource = self.serializer_class(data=schema_instance) - serialized_resource.is_valid(raise_exception=True) - - return Response(serialized_resource.data) + return JsonResponse(schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent":3}) elif request.method == 'PUT': @@ -85,7 +81,7 @@ def schema_instance(self, request, pk=None): serialized_content = self.serializer_class(data=content) serialized_content.is_valid(raise_exception=True) result = metadata_manager.update_schema_instance(resource, serialized_content.data, schema_instance) - + return Response(result) except ResourceBase.DoesNotExist: diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 31ffa6c1346..6a7c9b4b922 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -19,6 +19,8 @@ import json import logging +from datetime import datetime + from geonode.base.models import ResourceBase, TopicCategory, License, Region, RestrictionCodeType, \ SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler @@ -30,44 +32,85 @@ logger = logging.getLogger(__name__) -class CategorySubHandler: +class SubHandler: + @classmethod + def update_subschema(cls, subschema, lang=None): + pass + + @classmethod + def serialize(cls, db_value): + return db_value + + +class CategorySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): # subschema["title"] = _("topiccategory") subschema["oneOf"] = [{"const": tc.identifier,"title": tc.gn_description, "description": tc.description} for tc in TopicCategory.objects.order_by("gn_description")] -class FrequencySubHandler: +class DateTypeSubHandler(SubHandler): + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [{"const": i.lower(), "title": _(i)} + for i in ["Creation", "Publication", "Revision"]] + subschema["default"] = "Publication" + +class DateSubHandler(SubHandler): + @classmethod + def serialize(cls, value): + if isinstance(value, datetime): + return value.isoformat() + return value + +class FrequencySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": key,"title": val} for key, val in dict(UPDATE_FREQUENCIES).items()] -class LanguageSubHandler: +class LanguageSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": key,"title": val} for key, val in dict(ALL_LANGUAGES).items()] -class LicenseSubHandler: +class LicenseSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": tc.identifier,"title": tc.name, "description": tc.description} for tc in License.objects.order_by("name")] -class RegionSubHandler: + @classmethod + def serialize(cls, db_value): + if isinstance(db_value, License): + return db_value.identifier + return db_value + + +class KeywordsSubHandler(SubHandler): + @classmethod + def serialize(cls, value): + return "TODO!!!" + +class RegionSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["items"]["anyOf"] = [{"const": tc.code,"title": tc.name} for tc in Region.objects.order_by("name")] + @classmethod + def serialize(cls, db_value): + # TODO + return None -class RestrictionsSubHandler: + +class RestrictionsSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} for tc in RestrictionCodeType.objects.order_by("identifier")] -class SpatialRepresentationTypeSubHandler: +class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} @@ -76,8 +119,11 @@ def update_subschema(cls, subschema, lang=None): SUBHANDLERS = { "category": CategorySubHandler, + "date_type": DateTypeSubHandler, + "date": DateSubHandler, "language": LanguageSubHandler, "license": LicenseSubHandler, + "keywords": KeywordsSubHandler, "maintenance_frequency": FrequencySubHandler, "regions": RegionSubHandler, "restriction_code_type": RestrictionsSubHandler, @@ -125,6 +171,11 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): field_value = getattr(resource, field_name) + # perform specific transformation if any + if subhandler := SUBHANDLERS.get(field_name, None): + logger.debug(f"Serializing base field {field_name}") + field_value = subhandler.serialize(field_value) + return field_value def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 45e14206500..5c98d7b934e 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -33,13 +33,7 @@ "date_type": { "type": "string", "title": "date type", - "maxLength": 255, - "enum": [ - "Creation", - "Publication", - "Revision" - ], - "default": "Publication" + "maxLength": 255 }, "group": { "type": "string", From 41435ef42f21e4c505815daa42b054c5e29fb2ef Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 13:48:15 +0200 Subject: [PATCH 31/91] Return proper json schema instance --- geonode/metadata/handlers/base.py | 2 +- geonode/metadata/handlers/thesaurus.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 6a7c9b4b922..9e767935a2d 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -101,7 +101,7 @@ def update_subschema(cls, subschema, lang=None): @classmethod def serialize(cls, db_value): # TODO - return None + return [] class RestrictionsSubHandler(SubHandler): diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index d93ba578c33..3382d02ef8b 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -119,6 +119,9 @@ def update_schema(self, jsonschema, lang=None): "description": _("Keywords from controlled vocabularies"), "geonode:handler": "thesaurus", "properties": thesauri, + # "ui:options": { + # 'geonode-ui:group': 'Thesauri grop' + # } } # add thesauri after category @@ -134,7 +137,7 @@ def update_schema(self, jsonschema, lang=None): def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): - return None + return {} def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): From caae3d9f94d4f4dfe03d0a1ed8fd7e5f15b806c5 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 24 Oct 2024 17:36:45 +0300 Subject: [PATCH 32/91] adding a handler for the regions field: RegionsHandler --- geonode/metadata/handlers/base.py | 14 +-- geonode/metadata/handlers/region.py | 94 +++++++++++++++++++ .../jsonschema_examples/base_schema.json | 8 -- geonode/metadata/settings.py | 1 + 4 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 geonode/metadata/handlers/region.py diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 9e767935a2d..e479abe56ac 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -21,7 +21,7 @@ import logging from datetime import datetime -from geonode.base.models import ResourceBase, TopicCategory, License, Region, RestrictionCodeType, \ +from geonode.base.models import ResourceBase, TopicCategory, License, RestrictionCodeType, \ SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE @@ -93,17 +93,6 @@ class KeywordsSubHandler(SubHandler): def serialize(cls, value): return "TODO!!!" -class RegionSubHandler(SubHandler): - @classmethod - def update_subschema(cls, subschema, lang=None): - subschema["items"]["anyOf"] = [{"const": tc.code,"title": tc.name} - for tc in Region.objects.order_by("name")] - @classmethod - def serialize(cls, db_value): - # TODO - return [] - - class RestrictionsSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): @@ -125,7 +114,6 @@ def update_subschema(cls, subschema, lang=None): "license": LicenseSubHandler, "keywords": KeywordsSubHandler, "maintenance_frequency": FrequencySubHandler, - "regions": RegionSubHandler, "restriction_code_type": RestrictionsSubHandler, "spatial_representation_type": SpatialRepresentationTypeSubHandler, } diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py new file mode 100644 index 00000000000..1fb1bccc54c --- /dev/null +++ b/geonode/metadata/handlers/region.py @@ -0,0 +1,94 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.settings import JSONSCHEMA_BASE +from geonode.base.views import RegionAutocomplete + +logger = logging.getLogger(__name__) + + +class RegionHandler(MetadataHandler): + """ + The RegionsHandler adds the Regions model options to the schema + """ + + def __init__(self): + self.json_base_schema = JSONSCHEMA_BASE + self.base_schema = None + + def update_schema(self, jsonschema, lang=None): + + from geonode.base.models import Region + + subschema = [{"const": tc.code,"title": tc.name} + for tc in Region.objects.order_by("name")] + + regions = { + "type": "array", + "title": _("Regions"), + "description": _("keyword identifies a location"), + "items": { + "type": "string", + "anyOf": subschema + }, + "geonode:handler": "region", + #TODO autocomplete + # "ui:options": { + #"geonode-ui:autocomplete": reverse( + # "regions_autocomplete", + #kwargs={"regionsid": ct["id"]} + # ) + # }, + } + + # add regions after Attribution + + ret_properties = {} + for key, val in jsonschema["properties"].items(): + ret_properties[key] = val + if key == "attribution": + ret_properties["regions"] = regions + + jsonschema["properties"] = ret_properties + + return jsonschema + + @classmethod + def serialize(cls, db_value): + # TODO + return [] + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + + return None + + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + + pass + + def load_context(self, resource: ResourceBase, context: dict): + + pass \ No newline at end of file diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 5c98d7b934e..6be386f5d26 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -71,14 +71,6 @@ "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", "maxLength": 2048 }, - "regions": { - "type": "array", - "title": "Regions", - "description": "keyword identifies a location", - "items": { - "type": "string" - } - }, "data_quality_statement": { "type": "string", "title": "data quality statement", diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 8aeedfa897f..7b593f63b30 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -16,4 +16,5 @@ METADATA_HANDLERS = { "base": "geonode.metadata.handlers.base.BaseHandler", "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", + "region": "geonode.metadata.handlers.region.RegionHandler", } \ No newline at end of file From ff45f109aa76101e6eafa39ece41dc61dd659f4c Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 16:47:26 +0200 Subject: [PATCH 33/91] Add DOI handler --- geonode/metadata/handlers/abstract.py | 11 ++++ geonode/metadata/handlers/base.py | 9 ++-- geonode/metadata/handlers/doi.py | 54 +++++++++++++++++++ geonode/metadata/handlers/thesaurus.py | 8 +-- .../jsonschema_examples/base_schema.json | 6 --- 5 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 geonode/metadata/handlers/doi.py diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index ef14ba1bbf9..29e726a9b7a 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -62,3 +62,14 @@ def load_context(self, resource: ResourceBase, context: dict): Called before calls to update_resource in order to initialize info needed by the handler """ pass + + def _add_after(self, jsonschema, after_what, property_name, subschema): + # add thesauri after category + ret_properties = {} + for key,val in jsonschema["properties"].items(): + ret_properties[key] = val + if key == after_what: + ret_properties[property_name] = subschema + + jsonschema["properties"] = ret_properties + diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index e479abe56ac..0ee395afbf8 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -148,10 +148,9 @@ def localize(subschema: dict, annotation_name): subschema.update({"geonode:handler": "base"}) # perform further specific initializations - - if subhandler := SUBHANDLERS.get(property_name, None): + if property_name in SUBHANDLERS: logger.debug(f"Running subhandler for base field {property_name}") - subhandler.update_subschema(subschema, lang) + SUBHANDLERS[property_name].update_subschema(subschema, lang) return jsonschema @@ -160,9 +159,9 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): field_value = getattr(resource, field_name) # perform specific transformation if any - if subhandler := SUBHANDLERS.get(field_name, None): + if field_name in SUBHANDLERS: logger.debug(f"Serializing base field {field_name}") - field_value = subhandler.serialize(field_value) + field_value = SUBHANDLERS[field_name].serialize(field_value) return field_value diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py new file mode 100644 index 00000000000..4dd64fe35c5 --- /dev/null +++ b/geonode/metadata/handlers/doi.py @@ -0,0 +1,54 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler + + +logger = logging.getLogger(__name__) + + +class DOIHandler(MetadataHandler): + + def update_schema(self, jsonschema, lang=None): + + doi_schema = { + "type": "string", + "title": "DOI", + "description": "a DOI will be added by Admin before publication.", + "maxLength": 255 + } + + # add DOI after edition + self._add_after(jsonschema, "edition", "doi", doi_schema) + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + + return resource.doi + + def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + + pass + + def load_context(self, resource: ResourceBase, context: dict): + + pass diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 3382d02ef8b..06413692f0e 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -125,13 +125,7 @@ def update_schema(self, jsonschema, lang=None): } # add thesauri after category - ret_properties = {} - for key,val in jsonschema["properties"].items(): - ret_properties[key] = val - if key == "category": - ret_properties["tkeywords"] = tkeywords - - jsonschema["properties"] = ret_properties + self._add_after(jsonschema, "category", "tkeywords", tkeywords) return jsonschema diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 6be386f5d26..05424b369cc 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -102,12 +102,6 @@ "description": "Version of the cited resource", "maxLength": 255 }, - "doi": { - "type": "string", - "title": "DOI", - "description": "a DOI will be added by Admin before publication.", - "maxLength": 255 - }, "purpose": { "type": "string", "title": "purpose", From 9bc613d7abeddef6eb06189f4911cd27f50fe6cb Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 16:55:44 +0200 Subject: [PATCH 34/91] Improvements and fixes --- geonode/metadata/handlers/doi.py | 3 ++- geonode/metadata/handlers/region.py | 14 +------------- geonode/metadata/manager.py | 5 ++++- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 4dd64fe35c5..971b9f3e4d2 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -34,7 +34,8 @@ def update_schema(self, jsonschema, lang=None): "type": "string", "title": "DOI", "description": "a DOI will be added by Admin before publication.", - "maxLength": 255 + "maxLength": 255, + "geonode:handler": "doi", } # add DOI after edition diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 1fb1bccc54c..6955a51ff65 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -35,10 +35,6 @@ class RegionHandler(MetadataHandler): The RegionsHandler adds the Regions model options to the schema """ - def __init__(self): - self.json_base_schema = JSONSCHEMA_BASE - self.base_schema = None - def update_schema(self, jsonschema, lang=None): from geonode.base.models import Region @@ -65,15 +61,7 @@ def update_schema(self, jsonschema, lang=None): } # add regions after Attribution - - ret_properties = {} - for key, val in jsonschema["properties"].items(): - ret_properties[key] = val - if key == "attribution": - ret_properties["regions"] = regions - - jsonschema["properties"] = ret_properties - + self._add_after(jsonschema, "attribution", "regions", regions) return jsonschema @classmethod diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 773f4f2a093..f3c82b8f2da 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -74,7 +74,10 @@ def build_schema_instance(self, resource): instance = {} for fieldname, field in schema["properties"].items(): - handler_id = field["geonode:handler"] + handler_id = field.get("geonode:handler", None) + if not handler_id: + logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") + continue handler = self.handlers[handler_id] content = handler.get_jsonschema_instance(resource, fieldname) instance[fieldname] = content From 69570074816c9bfdf73e8744033a1f84cd0e9cf3 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 24 Oct 2024 17:57:49 +0300 Subject: [PATCH 35/91] fixing Region autocomplete --- geonode/metadata/handlers/region.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 1fb1bccc54c..996d8dd7dc2 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -55,13 +55,11 @@ def update_schema(self, jsonschema, lang=None): "anyOf": subschema }, "geonode:handler": "region", - #TODO autocomplete - # "ui:options": { - #"geonode-ui:autocomplete": reverse( - # "regions_autocomplete", - #kwargs={"regionsid": ct["id"]} - # ) - # }, + "ui:options": { + "geonode-ui:autocomplete": reverse( + "autocomplete_region" + ) + }, } # add regions after Attribution From 94fd9ec9cbd94283d639e5ec689a44bf5b7976f6 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 17:32:45 +0200 Subject: [PATCH 36/91] Add DOI handler --- geonode/metadata/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 7b593f63b30..5c3a08365af 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -17,4 +17,5 @@ "base": "geonode.metadata.handlers.base.BaseHandler", "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", "region": "geonode.metadata.handlers.region.RegionHandler", + "doi": "geonode.metadata.handlers.doi.DOIHandler", } \ No newline at end of file From 88a6a53189271939e11c3533f8295e8ca8e76907 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 24 Oct 2024 17:34:57 +0200 Subject: [PATCH 37/91] Simplify tkeywords schema --- geonode/metadata/handlers/thesaurus.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 06413692f0e..09eb6c4b69f 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -81,13 +81,10 @@ def update_schema(self, jsonschema, lang=None): # copy info to json schema thesauri = {} for id,ct in collected_thesauri.items(): - thesaurus = {} - thesaurus["type"] = "object" - thesaurus["title"] = ct["title"] - thesaurus["description"] = ct["description"] - - keywords = { + thesaurus = { "type": "array", + "title": ct["title"], + "description": ct["description"], "items": { "type": "object", "properties": { @@ -103,14 +100,14 @@ def update_schema(self, jsonschema, lang=None): } } }, - "ui:options": { - 'geonode-ui:autocomplete': reverse( - "thesaurus-keywords_autocomplete", - kwargs={"thesaurusid": ct["id"]}) - } + "ui:options": { + 'geonode-ui:autocomplete': reverse( + "thesaurus-keywords_autocomplete", + kwargs={"thesaurusid": ct["id"]}) + } } - keywords.update(ct["card"]) - thesaurus["properties"] = {"keywords": keywords} + + thesaurus.update(ct["card"]) thesauri[id] = thesaurus tkeywords = { From 3f137d52e3a895aece3c01d6d60886d6e0578279 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 25 Oct 2024 08:54:46 +0300 Subject: [PATCH 38/91] adding serialize method to other FKs of the BaseHandler --- geonode/metadata/handlers/base.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 0ee395afbf8..5052601fad2 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -48,6 +48,12 @@ def update_subschema(cls, subschema, lang=None): # subschema["title"] = _("topiccategory") subschema["oneOf"] = [{"const": tc.identifier,"title": tc.gn_description, "description": tc.description} for tc in TopicCategory.objects.order_by("gn_description")] + + @classmethod + def serialize(cls, db_value): + if isinstance(db_value, TopicCategory): + return db_value.identifier + return db_value class DateTypeSubHandler(SubHandler): @classmethod @@ -98,12 +104,24 @@ class RestrictionsSubHandler(SubHandler): def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} for tc in RestrictionCodeType.objects.order_by("identifier")] + + @classmethod + def serialize(cls, db_value): + if isinstance(db_value, RestrictionCodeType): + return db_value.identifier + return db_value class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} for tc in SpatialRepresentationType.objects.order_by("identifier")] + + @classmethod + def serialize(cls, db_value): + if isinstance(db_value, SpatialRepresentationType): + return db_value.identifier + return db_value SUBHANDLERS = { From 4bc01af287696be0caa276933600778d3dfb005a Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 25 Oct 2024 12:05:19 +0300 Subject: [PATCH 39/91] Extending PUT and removing serialization --- geonode/metadata/api/serializers.py | 34 ----------------------------- geonode/metadata/api/views.py | 11 +++------- geonode/metadata/handlers/base.py | 11 ++++++++-- 3 files changed, 12 insertions(+), 44 deletions(-) delete mode 100644 geonode/metadata/api/serializers.py diff --git a/geonode/metadata/api/serializers.py b/geonode/metadata/api/serializers.py deleted file mode 100644 index 73745bcbacc..00000000000 --- a/geonode/metadata/api/serializers.py +++ /dev/null @@ -1,34 +0,0 @@ -from rest_framework import serializers -from geonode.base.models import ResourceBase - -class MetadataSerializer(serializers.ModelSerializer): - class Meta: - model = ResourceBase - fields = [ - "title", - "abstract", - "purpose", - "alternate", - "date_type", - "edition", - "attribution", - "doi", - "maintenance_frequency", - "constraints_other", - "language", - "supplemental_information", - "data_quality_statement", - "srid", - "metadata_uploaded_preserve", - "featured", - "was_published", - "is_published", - "was_approved", - "is_approved", - "advertised", - "thumbnail_url", - "thumbnail_path", - "state", - "sourcetype", - "metadata_only", - ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 4f7ce898fd7..a9148513f26 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -25,7 +25,6 @@ from geonode.base.models import ResourceBase from geonode.metadata.manager import metadata_manager -from geonode.metadata.api.serializers import MetadataSerializer class MetadataViewSet(ViewSet): @@ -34,7 +33,6 @@ class MetadataViewSet(ViewSet): """ queryset = ResourceBase.objects.all() - serializer_class = MetadataSerializer def list(self, request): pass @@ -76,13 +74,10 @@ def schema_instance(self, request, pk=None): return JsonResponse(schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent":3}) elif request.method == 'PUT': + + updated_content = metadata_manager.update_schema_instance(resource, request.data, schema_instance) - content = request.data - serialized_content = self.serializer_class(data=content) - serialized_content.is_valid(raise_exception=True) - result = metadata_manager.update_schema_instance(resource, serialized_content.data, schema_instance) - - return Response(result) + return Response(updated_content) except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 5052601fad2..bafe11f156e 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -186,8 +186,15 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): if field_name in content: - # insert the content value to the corresponding field_name - setattr(resource, field_name, content[field_name]) + + field_value = content[field_name] + + if field_name in SUBHANDLERS: + logger.debug(f"Deserializing base field {field_name}") + # Serialize the field_value before setting it in the resource object + field_value = SUBHANDLERS[field_name].serialize(field_value) + + setattr(resource, field_name, field_value) def load_context(self, resource: ResourceBase, context: dict): From e5b05a60810f9fc9cf9d73ca4bb3bc8a7d19da72 Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 25 Oct 2024 12:04:47 +0200 Subject: [PATCH 40/91] Fix PUT/PATCH --- geonode/metadata/api/views.py | 20 +++++++++++++------- geonode/metadata/manager.py | 13 +++++++------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index a9148513f26..e38d8889fb5 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -17,6 +17,9 @@ # ######################################################################### from django.http import JsonResponse +import json +import logging + from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response @@ -27,6 +30,9 @@ from geonode.metadata.manager import metadata_manager +logger = logging.getLogger(__name__) + + class MetadataViewSet(ViewSet): """ Simple viewset that return the metadata JSON schema @@ -61,23 +67,23 @@ def schema(self, request, pk=None): # Get the JSON schema @action(detail=False, - methods=['get', 'put'], + methods=['get', 'put', 'patch'], url_path="instance/(?P\d+)" ) def schema_instance(self, request, pk=None): try: resource = ResourceBase.objects.get(pk=pk) - schema_instance = metadata_manager.build_schema_instance(resource) if request.method == 'GET': + schema_instance = metadata_manager.build_schema_instance(resource) return JsonResponse(schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent":3}) - elif request.method == 'PUT': - - updated_content = metadata_manager.update_schema_instance(resource, request.data, schema_instance) - - return Response(updated_content) + elif request.method in ('PUT', "PATCH"): + logger.info(f"handling request {request.method}") + logger.info(f"handling content {request.data}") + update_response = metadata_manager.update_schema_instance(resource, request.data) + return Response(update_response) except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index f3c82b8f2da..aadfefaf0ed 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -84,14 +84,15 @@ def build_schema_instance(self, resource): return instance - def update_schema_instance(self, resource, content, json_instance): - + def update_schema_instance(self, resource, json_instance): + + logger.info(f"RECEIVED INSTANCE {json_instance}") + schema = self.get_schema() - for fieldname, field in schema["properties"].items(): - handler_id = field["geonode:handler"] - handler = self.handlers[handler_id] - handler.update_resource(resource, fieldname, content, json_instance) + for fieldname, subschema in schema["properties"].items(): + handler = self.handlers[subschema["geonode:handler"]] + handler.update_resource(resource, fieldname, json_instance[fieldname], json_instance) try: resource.save() From aa5afc80e13e0fd0a05475184edd3f63996272cd Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 25 Oct 2024 12:17:57 +0200 Subject: [PATCH 41/91] Fixes: now patch returns without major errors --- geonode/metadata/handlers/abstract.py | 2 +- geonode/metadata/handlers/base.py | 21 ++++++++++++--------- geonode/metadata/handlers/doi.py | 2 +- geonode/metadata/handlers/region.py | 2 +- geonode/metadata/handlers/thesaurus.py | 4 ++-- geonode/metadata/manager.py | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 29e726a9b7a..6e2a2d45a66 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -48,7 +48,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): pass @abstractmethod - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): """ Called when persisting data, updates the field field_name of the resource with the content content, where json_instance is the full JSON Schema instance, diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index bafe11f156e..bdc96f76e02 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -183,18 +183,21 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): return field_value - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - if field_name in content: + if field_name in json_instance: + try: + field_value = json_instance[field_name] - field_value = content[field_name] + if field_name in SUBHANDLERS: + logger.debug(f"Deserializing base field {field_name}") + # Serialize the field_value before setting it in the resource object + field_value = SUBHANDLERS[field_name].serialize(field_value) + + setattr(resource, field_name, field_value) + except Exception as e: + logger.warning(f"Error setting field {field_name}={field_value}: {e}") - if field_name in SUBHANDLERS: - logger.debug(f"Deserializing base field {field_name}") - # Serialize the field_value before setting it in the resource object - field_value = SUBHANDLERS[field_name].serialize(field_value) - - setattr(resource, field_name, field_value) def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 971b9f3e4d2..5b157ba6ed3 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -46,7 +46,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): return resource.doi - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): pass diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index e19e52374a9..e574740005b 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -71,7 +71,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): return None - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): pass diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 09eb6c4b69f..24608e8f2ad 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -130,10 +130,10 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): return {} - def update_resource(self, resource: ResourceBase, field_name: str, content: dict, json_instance: dict): - + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): pass + def load_context(self, resource: ResourceBase, context: dict): pass diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index aadfefaf0ed..51968131519 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -92,7 +92,7 @@ def update_schema_instance(self, resource, json_instance): for fieldname, subschema in schema["properties"].items(): handler = self.handlers[subschema["geonode:handler"]] - handler.update_resource(resource, fieldname, json_instance[fieldname], json_instance) + handler.update_resource(resource, fieldname, json_instance) try: resource.save() From 037ea1fe834e35849b55fa1bee15ecf97382f9f7 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 25 Oct 2024 14:25:22 +0300 Subject: [PATCH 42/91] Storing FKs to the resource model --- geonode/metadata/handlers/base.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index bdc96f76e02..053cfab8019 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -54,6 +54,11 @@ def serialize(cls, db_value): if isinstance(db_value, TopicCategory): return db_value.identifier return db_value + + @classmethod + def get_fk_instance(cls, field_value): + db_value = TopicCategory.objects.get(identifier=field_value) + return db_value class DateTypeSubHandler(SubHandler): @classmethod @@ -92,8 +97,12 @@ def serialize(cls, db_value): if isinstance(db_value, License): return db_value.identifier return db_value - - + + @classmethod + def get_fk_instance(cls, field_value): + db_value = License.objects.get(identifier=field_value) + return db_value + class KeywordsSubHandler(SubHandler): @classmethod def serialize(cls, value): @@ -110,6 +119,11 @@ def serialize(cls, db_value): if isinstance(db_value, RestrictionCodeType): return db_value.identifier return db_value + + @classmethod + def get_fk_instance(cls, field_value): + db_value = RestrictionCodeType.objects.get(identifier=field_value) + return db_value class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod @@ -123,6 +137,10 @@ def serialize(cls, db_value): return db_value.identifier return db_value + @classmethod + def get_fk_instance(cls, field_value): + db_value = SpatialRepresentationType.objects.get(identifier=field_value) + return db_value SUBHANDLERS = { "category": CategorySubHandler, @@ -191,8 +209,8 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance if field_name in SUBHANDLERS: logger.debug(f"Deserializing base field {field_name}") - # Serialize the field_value before setting it in the resource object - field_value = SUBHANDLERS[field_name].serialize(field_value) + # Set the field_value of a foreign key to the resource object + field_value = SUBHANDLERS[field_name].get_fk_instance(field_value) setattr(resource, field_name, field_value) except Exception as e: From 3d7e062826042fd3b83ee9d865cbfd85dac7ee1c Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 25 Oct 2024 15:59:02 +0300 Subject: [PATCH 43/91] small improvements to store FK values --- geonode/metadata/handlers/base.py | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 053cfab8019..de1b4addfd3 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -40,6 +40,10 @@ def update_subschema(cls, subschema, lang=None): @classmethod def serialize(cls, db_value): return db_value + + @classmethod + def deserialize(cls, field_value): + return field_value class CategorySubHandler(SubHandler): @@ -56,9 +60,9 @@ def serialize(cls, db_value): return db_value @classmethod - def get_fk_instance(cls, field_value): - db_value = TopicCategory.objects.get(identifier=field_value) - return db_value + def deserialize(cls, field_value): + field_value = TopicCategory.objects.get(identifier=field_value) + return field_value class DateTypeSubHandler(SubHandler): @classmethod @@ -99,9 +103,9 @@ def serialize(cls, db_value): return db_value @classmethod - def get_fk_instance(cls, field_value): - db_value = License.objects.get(identifier=field_value) - return db_value + def deserialize(cls, field_value): + field_value = License.objects.get(identifier=field_value) + return field_value class KeywordsSubHandler(SubHandler): @classmethod @@ -121,9 +125,9 @@ def serialize(cls, db_value): return db_value @classmethod - def get_fk_instance(cls, field_value): - db_value = RestrictionCodeType.objects.get(identifier=field_value) - return db_value + def deserialize(cls, field_value): + field_value = RestrictionCodeType.objects.get(identifier=field_value) + return field_value class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod @@ -138,9 +142,9 @@ def serialize(cls, db_value): return db_value @classmethod - def get_fk_instance(cls, field_value): - db_value = SpatialRepresentationType.objects.get(identifier=field_value) - return db_value + def deserialize(cls, field_value): + field_value = SpatialRepresentationType.objects.get(identifier=field_value) + return field_value SUBHANDLERS = { "category": CategorySubHandler, @@ -209,8 +213,8 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance if field_name in SUBHANDLERS: logger.debug(f"Deserializing base field {field_name}") - # Set the field_value of a foreign key to the resource object - field_value = SUBHANDLERS[field_name].get_fk_instance(field_value) + # Deserialize field values before setting them to the ResourceBase + field_value = SUBHANDLERS[field_name].deserialize(field_value) setattr(resource, field_name, field_value) except Exception as e: From 69dcc10be6f6c7536d0a9b85975f884bdf2d0ed2 Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 25 Oct 2024 17:58:39 +0200 Subject: [PATCH 44/91] TKeywords get and patch working. Added i18n to instance request --- geonode/metadata/api/views.py | 5 +-- geonode/metadata/handlers/abstract.py | 2 +- geonode/metadata/handlers/base.py | 25 ++++++++------- geonode/metadata/handlers/doi.py | 2 +- geonode/metadata/handlers/region.py | 2 +- geonode/metadata/handlers/thesaurus.py | 44 +++++++++++++++++++------- geonode/metadata/manager.py | 4 +-- 7 files changed, 55 insertions(+), 29 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index e38d8889fb5..f2df63a8d70 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -55,7 +55,7 @@ def schema(self, request, pk=None): resource scope. ''' - lang = get_language_from_request(request)[:2] + lang = request.query_params.get("lang", get_language_from_request(request)[:2]) schema = metadata_manager.get_schema(lang) if schema: @@ -76,7 +76,8 @@ def schema_instance(self, request, pk=None): resource = ResourceBase.objects.get(pk=pk) if request.method == 'GET': - schema_instance = metadata_manager.build_schema_instance(resource) + lang = request.query_params.get("lang", get_language_from_request(request)[:2]) + schema_instance = metadata_manager.build_schema_instance(resource, lang) return JsonResponse(schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent":3}) elif request.method in ('PUT', "PATCH"): diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 6e2a2d45a66..6bb950064fd 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -40,7 +40,7 @@ def update_schema(self, jsonschema: dict, lang=None): pass @abstractmethod - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): """ Called when reading metadata, returns the instance of the sub-schema associated with the field field_name. diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index de1b4addfd3..cea8c193dfb 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -61,8 +61,8 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - field_value = TopicCategory.objects.get(identifier=field_value) - return field_value + return TopicCategory.objects.get(identifier=field_value) + class DateTypeSubHandler(SubHandler): @classmethod @@ -78,18 +78,21 @@ def serialize(cls, value): return value.isoformat() return value + class FrequencySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": key,"title": val} for key, val in dict(UPDATE_FREQUENCIES).items()] + class LanguageSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): subschema["oneOf"] = [{"const": key,"title": val} for key, val in dict(ALL_LANGUAGES).items()] + class LicenseSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): @@ -104,14 +107,15 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - field_value = License.objects.get(identifier=field_value) - return field_value + return License.objects.get(identifier=field_value) + class KeywordsSubHandler(SubHandler): @classmethod def serialize(cls, value): return "TODO!!!" + class RestrictionsSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): @@ -126,8 +130,8 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - field_value = RestrictionCodeType.objects.get(identifier=field_value) - return field_value + return RestrictionCodeType.objects.get(identifier=field_value) + class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod @@ -143,8 +147,8 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - field_value = SpatialRepresentationType.objects.get(identifier=field_value) - return field_value + return SpatialRepresentationType.objects.get(identifier=field_value) + SUBHANDLERS = { "category": CategorySubHandler, @@ -194,7 +198,7 @@ def localize(subschema: dict, annotation_name): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): field_value = getattr(resource, field_name) @@ -208,9 +212,8 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): if field_name in json_instance: + field_value = json_instance[field_name] try: - field_value = json_instance[field_name] - if field_name in SUBHANDLERS: logger.debug(f"Deserializing base field {field_name}") # Deserialize field values before setting them to the ResourceBase diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 5b157ba6ed3..c84751cc86b 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -42,7 +42,7 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "edition", "doi", doi_schema) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): return resource.doi diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index e574740005b..fa88d3ee867 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -67,7 +67,7 @@ def serialize(cls, db_value): # TODO return [] - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): return None diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 24608e8f2ad..33806f62f09 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -17,7 +17,6 @@ # ######################################################################### -import json import logging from rest_framework.reverse import reverse @@ -25,25 +24,23 @@ from django.db.models import Q from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase +from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE -from geonode.base.enumerations import ALL_LANGUAGES logger = logging.getLogger(__name__) +TKEYWORDS = "tkeywords" + + class TKeywordsHandler(MetadataHandler): """ The base handler builds a valid empty schema with the simple fields of the ResourceBase model """ - def __init__(self): - self.json_base_schema = JSONSCHEMA_BASE - self.base_schema = None - def update_schema(self, jsonschema, lang=None): from geonode.base.models import Thesaurus @@ -122,16 +119,41 @@ def update_schema(self, jsonschema, lang=None): } # add thesauri after category - self._add_after(jsonschema, "category", "tkeywords", tkeywords) + self._add_after(jsonschema, "category", TKEYWORDS, tkeywords) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): + + tks = {} + for tk in resource.tkeywords.all(): + tks[tk.id] = tk + tkls = ThesaurusKeywordLabel.objects.filter(keyword__id__in=tks.keys(), lang=lang) # read all entries in a single query + + ret = {} + for tkl in tkls: + keywords = ret.setdefault(tkl.keyword.thesaurus.identifier, []) + keywords.append({"id": tkl.keyword.about, "label": tkl.label}) + del tks[tkl.keyword.id] - return {} + if tks: + logger.info(f"Returning untraslated '{lang}' keywords: {tks}") + for tk in tks.values(): + keywords = ret.setdefault(tk.thesaurus.identifier, []) + keywords.append({"id": tk.about, "label": tk.alt_label}) + + return ret def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - pass + + kids = [] + for thes_id, keywords in json_instance.get(TKEYWORDS, {}).items(): + logger.info(f"Getting info for thesaurus {thes_id}") + for keyword in keywords: + kids.append(keyword["id"]) + + kw_requested = ThesaurusKeyword.objects.filter(about__in=kids) + resource.tkeywords.set(kw_requested) def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 51968131519..7eba3f2619d 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -68,7 +68,7 @@ def get_schema(self, lang=None): return self.cached_schema - def build_schema_instance(self, resource): + def build_schema_instance(self, resource, lang=None): schema = self.get_schema() instance = {} @@ -79,7 +79,7 @@ def build_schema_instance(self, resource): logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") continue handler = self.handlers[handler_id] - content = handler.get_jsonschema_instance(resource, fieldname) + content = handler.get_jsonschema_instance(resource, fieldname, lang) instance[fieldname] = content return instance From 481f5eadc6a59717baa940f410a06151fac335a2 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 28 Oct 2024 10:29:26 +0100 Subject: [PATCH 45/91] Cleanup: black and flake --- geonode/base/api/views.py | 31 ++++++------- geonode/metadata/admin.py | 3 -- geonode/metadata/api/urls.py | 2 +- geonode/metadata/api/views.py | 38 +++++++--------- geonode/metadata/apps.py | 4 +- geonode/metadata/handlers/abstract.py | 17 ++++--- geonode/metadata/handlers/base.py | 63 ++++++++++++++------------ geonode/metadata/handlers/doi.py | 10 ++-- geonode/metadata/handlers/region.py | 22 +++------ geonode/metadata/handlers/thesaurus.py | 39 ++++++++++------ geonode/metadata/manager.py | 35 +++++++------- geonode/metadata/models.py | 3 -- geonode/metadata/registry.py | 6 +-- geonode/metadata/settings.py | 15 +++--- geonode/metadata/tests.py | 3 -- geonode/metadata/urls.py | 3 +- geonode/metadata/views.py | 1 - 17 files changed, 138 insertions(+), 157 deletions(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index fb1edde5b84..daf9b4b9c87 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -24,7 +24,6 @@ from uuid import uuid4 from urllib.parse import urljoin, urlparse from PIL import Image -from dal import autocomplete from django.apps import apps from django.core.validators import URLValidator @@ -36,9 +35,8 @@ from django.http.request import QueryDict from django.contrib.auth import get_user_model from django.utils.translation import get_language -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import extend_schema from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter @@ -252,25 +250,22 @@ def tkeywords_autocomplete(self, request, thesaurusid): all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again - localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter( - lang=lang, keyword_id__in=all_keywords_qs - ).values("keyword_id") + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( + "keyword_id" + ) if not localized_k_ids_qs.exists(): lang = remove_country_from_languagecode(lang) - localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter( - lang=lang, keyword_id__in=all_keywords_qs - ).values("keyword_id") + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( + "keyword_id" + ) # consider all the keywords that do not have a translation in the requested language keywords_not_translated_qs = ( - all_keywords_qs.exclude(id__in=localized_k_ids_qs) - .order_by("id") - .distinct("id") - .values("id") + all_keywords_qs.exclude(id__in=localized_k_ids_qs).order_by("id").distinct("id").values("id") ) qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).order_by("label") - if q:=request.query_params.get("q", None): + if q := request.query_params.get("q", None): qs = qs.filter(label__istartswith=q) ret = [] @@ -279,15 +274,17 @@ def tkeywords_autocomplete(self, request, thesaurusid): { "id": tkl.keyword.about, "label": tkl.label, - }) + } + ) for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).order_by("alt_label").all(): ret.append( { "id": tk.about, "label": f"! {tk.alt_label}", - }) + } + ) - return JsonResponse({"results":ret}) + return JsonResponse({"results": ret}) class TopicCategoryViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): diff --git a/geonode/metadata/admin.py b/geonode/metadata/admin.py index 8c38f3f3dad..e69de29bb2d 100644 --- a/geonode/metadata/admin.py +++ b/geonode/metadata/admin.py @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 11746f0cfd9..85341eda3ca 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -20,4 +20,4 @@ from geonode.metadata.api import views router = routers.DefaultRouter() -router.register(r"metadata", views.MetadataViewSet, basename = "metadata") +router.register(r"metadata", views.MetadataViewSet, basename="metadata") diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index f2df63a8d70..fc604f3d7ff 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -16,14 +16,14 @@ # along with this program. If not, see . # ######################################################################### -from django.http import JsonResponse -import json import logging + from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response +from django.http import JsonResponse from django.utils.translation.trans_real import get_language_from_request from geonode.base.models import ResourceBase @@ -37,7 +37,7 @@ class MetadataViewSet(ViewSet): """ Simple viewset that return the metadata JSON schema """ - + queryset = ResourceBase.objects.all() def list(self, request): @@ -45,19 +45,16 @@ def list(self, request): # Get the JSON schema # A pk argument is set for futured multiple schemas - @action(detail=False, - methods=['get'], - url_path="schema(?:/(?P\d+))?" - ) + @action(detail=False, methods=["get"], url_path=r"schema(?:/(?P\d+))?") def schema(self, request, pk=None): - ''' + """ The user is able to export her/his keys with resource scope. - ''' + """ lang = request.query_params.get("lang", get_language_from_request(request)[:2]) schema = metadata_manager.get_schema(lang) - + if schema: return Response(schema) @@ -66,26 +63,25 @@ def schema(self, request, pk=None): return Response(response) # Get the JSON schema - @action(detail=False, - methods=['get', 'put', 'patch'], - url_path="instance/(?P\d+)" - ) + @action(detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)") def schema_instance(self, request, pk=None): - + try: resource = ResourceBase.objects.get(pk=pk) - if request.method == 'GET': + if request.method == "GET": lang = request.query_params.get("lang", get_language_from_request(request)[:2]) schema_instance = metadata_manager.build_schema_instance(resource, lang) - return JsonResponse(schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent":3}) - - elif request.method in ('PUT', "PATCH"): + return JsonResponse( + schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent": 3} + ) + + elif request.method in ("PUT", "PATCH"): logger.info(f"handling request {request.method}") logger.info(f"handling content {request.data}") update_response = metadata_manager.update_schema_instance(resource, request.data) return Response(update_response) - + except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} - return Response(result) \ No newline at end of file + return Response(result) diff --git a/geonode/metadata/apps.py b/geonode/metadata/apps.py index 02b8e17492c..f47f12ad6c8 100644 --- a/geonode/metadata/apps.py +++ b/geonode/metadata/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class MetadataConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "geonode.metadata" @@ -9,6 +10,7 @@ def ready(self): run_setup_hooks() super(MetadataConfig, self).ready() + def run_setup_hooks(*args, **kwargs): from geonode.metadata.registry import metadata_registry from geonode.metadata.manager import metadata_manager @@ -18,4 +20,4 @@ def run_setup_hooks(*args, **kwargs): handlers = metadata_registry.handler_registry for handler_id, handler in handlers.items(): - metadata_manager.add_handler(handler_id, handler) \ No newline at end of file + metadata_manager.add_handler(handler_id, handler) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 6bb950064fd..9ccd69b4b57 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -26,23 +26,23 @@ class MetadataHandler(metaclass=ABCMeta): """ - Handlers take care of reading, storing, encoding, + Handlers take care of reading, storing, encoding, decoding subschemas of the main Resource """ @abstractmethod def update_schema(self, jsonschema: dict, lang=None): """ - It is called by the MetadataManager when creating the JSON Schema - It adds the subschema handled by the handler, and returns the + It is called by the MetadataManager when creating the JSON Schema + It adds the subschema handled by the handler, and returns the augmented instance of the JSON Schema. """ pass @abstractmethod - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): """ - Called when reading metadata, returns the instance of the sub-schema + Called when reading metadata, returns the instance of the sub-schema associated with the field field_name. """ pass @@ -50,8 +50,8 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: @abstractmethod def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): """ - Called when persisting data, updates the field field_name of the resource - with the content content, where json_instance is the full JSON Schema instance, + Called when persisting data, updates the field field_name of the resource + with the content content, where json_instance is the full JSON Schema instance, in case the handler needs some cross related data contained in the resource. """ pass @@ -66,10 +66,9 @@ def load_context(self, resource: ResourceBase, context: dict): def _add_after(self, jsonschema, after_what, property_name, subschema): # add thesauri after category ret_properties = {} - for key,val in jsonschema["properties"].items(): + for key, val in jsonschema["properties"].items(): ret_properties[key] = val if key == after_what: ret_properties[property_name] = subschema jsonschema["properties"] = ret_properties - diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index cea8c193dfb..2eb30a26cf8 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -21,8 +21,7 @@ import logging from datetime import datetime -from geonode.base.models import ResourceBase, TopicCategory, License, RestrictionCodeType, \ - SpatialRepresentationType +from geonode.base.models import ResourceBase, TopicCategory, License, RestrictionCodeType, SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES @@ -40,7 +39,7 @@ def update_subschema(cls, subschema, lang=None): @classmethod def serialize(cls, db_value): return db_value - + @classmethod def deserialize(cls, field_value): return field_value @@ -50,15 +49,17 @@ class CategorySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): # subschema["title"] = _("topiccategory") - subschema["oneOf"] = [{"const": tc.identifier,"title": tc.gn_description, "description": tc.description} - for tc in TopicCategory.objects.order_by("gn_description")] - + subschema["oneOf"] = [ + {"const": tc.identifier, "title": tc.gn_description, "description": tc.description} + for tc in TopicCategory.objects.order_by("gn_description") + ] + @classmethod def serialize(cls, db_value): if isinstance(db_value, TopicCategory): return db_value.identifier return db_value - + @classmethod def deserialize(cls, field_value): return TopicCategory.objects.get(identifier=field_value) @@ -67,10 +68,10 @@ def deserialize(cls, field_value): class DateTypeSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": i.lower(), "title": _(i)} - for i in ["Creation", "Publication", "Revision"]] + subschema["oneOf"] = [{"const": i.lower(), "title": _(i)} for i in ["Creation", "Publication", "Revision"]] subschema["default"] = "Publication" + class DateSubHandler(SubHandler): @classmethod def serialize(cls, value): @@ -82,34 +83,34 @@ def serialize(cls, value): class FrequencySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": key,"title": val} - for key, val in dict(UPDATE_FREQUENCIES).items()] + subschema["oneOf"] = [{"const": key, "title": val} for key, val in dict(UPDATE_FREQUENCIES).items()] class LanguageSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": key,"title": val} - for key, val in dict(ALL_LANGUAGES).items()] + subschema["oneOf"] = [{"const": key, "title": val} for key, val in dict(ALL_LANGUAGES).items()] class LicenseSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": tc.identifier,"title": tc.name, "description": tc.description} - for tc in License.objects.order_by("name")] + subschema["oneOf"] = [ + {"const": tc.identifier, "title": tc.name, "description": tc.description} + for tc in License.objects.order_by("name") + ] @classmethod def serialize(cls, db_value): if isinstance(db_value, License): return db_value.identifier return db_value - + @classmethod def deserialize(cls, field_value): return License.objects.get(identifier=field_value) - + class KeywordsSubHandler(SubHandler): @classmethod def serialize(cls, value): @@ -119,15 +120,17 @@ def serialize(cls, value): class RestrictionsSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} - for tc in RestrictionCodeType.objects.order_by("identifier")] - + subschema["oneOf"] = [ + {"const": tc.identifier, "title": tc.identifier, "description": tc.description} + for tc in RestrictionCodeType.objects.order_by("identifier") + ] + @classmethod def serialize(cls, db_value): if isinstance(db_value, RestrictionCodeType): return db_value.identifier return db_value - + @classmethod def deserialize(cls, field_value): return RestrictionCodeType.objects.get(identifier=field_value) @@ -136,9 +139,11 @@ def deserialize(cls, field_value): class SpatialRepresentationTypeSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [{"const": tc.identifier,"title": tc.identifier, "description": tc.description} - for tc in SpatialRepresentationType.objects.order_by("identifier")] - + subschema["oneOf"] = [ + {"const": tc.identifier, "title": tc.identifier, "description": tc.description} + for tc in SpatialRepresentationType.objects.order_by("identifier") + ] + @classmethod def serialize(cls, db_value): if isinstance(db_value, SpatialRepresentationType): @@ -182,8 +187,8 @@ def localize(subschema: dict, annotation_name): self.base_schema = json.load(f) # building the full base schema for property_name, subschema in self.base_schema.items(): - localize(subschema, 'title') - localize(subschema, 'abstract') + localize(subschema, "title") + localize(subschema, "abstract") jsonschema["properties"][property_name] = subschema @@ -198,7 +203,7 @@ def localize(subschema: dict, annotation_name): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): field_value = getattr(resource, field_name) @@ -210,7 +215,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: return field_value def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - + if field_name in json_instance: field_value = json_instance[field_name] try: @@ -223,8 +228,6 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") - def load_context(self, resource: ResourceBase, context: dict): pass - diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index c84751cc86b..bec9ff6cd36 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -31,11 +31,11 @@ class DOIHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): doi_schema = { - "type": "string", - "title": "DOI", - "description": "a DOI will be added by Admin before publication.", - "maxLength": 255, - "geonode:handler": "doi", + "type": "string", + "title": "DOI", + "description": "a DOI will be added by Admin before publication.", + "maxLength": 255, + "geonode:handler": "doi", } # add DOI after edition diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index fa88d3ee867..4b7ba7d00e4 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -24,8 +24,6 @@ from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler -from geonode.metadata.settings import JSONSCHEMA_BASE -from geonode.base.views import RegionAutocomplete logger = logging.getLogger(__name__) @@ -39,29 +37,21 @@ def update_schema(self, jsonschema, lang=None): from geonode.base.models import Region - subschema = [{"const": tc.code,"title": tc.name} - for tc in Region.objects.order_by("name")] - + subschema = [{"const": tc.code, "title": tc.name} for tc in Region.objects.order_by("name")] + regions = { "type": "array", "title": _("Regions"), "description": _("keyword identifies a location"), - "items": { - "type": "string", - "anyOf": subschema - }, + "items": {"type": "string", "anyOf": subschema}, "geonode:handler": "region", - "ui:options": { - "geonode-ui:autocomplete": reverse( - "autocomplete_region" - ) - }, + "ui:options": {"geonode-ui:autocomplete": reverse("autocomplete_region")}, } # add regions after Attribution self._add_after(jsonschema, "attribution", "regions", regions) return jsonschema - + @classmethod def serialize(cls, db_value): # TODO @@ -77,4 +67,4 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance def load_context(self, resource: ResourceBase, context: dict): - pass \ No newline at end of file + pass diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 33806f62f09..f3c87d66513 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -26,7 +26,6 @@ from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.metadata.handlers.abstract import MetadataHandler -from geonode.metadata.settings import JSONSCHEMA_BASE logger = logging.getLogger(__name__) @@ -48,8 +47,17 @@ def update_schema(self, jsonschema, lang=None): # this query return the list of thesaurus X the list of localized titles q = ( Thesaurus.objects.filter(~Q(card_max=0)) - .values("id", "identifier", "title", "description", "order", "card_min", "card_max", - "rel_thesaurus__label", "rel_thesaurus__lang") + .values( + "id", + "identifier", + "title", + "description", + "order", + "card_min", + "card_max", + "rel_thesaurus__label", + "rel_thesaurus__lang", + ) .order_by("order") ) @@ -66,7 +74,7 @@ def update_schema(self, jsonschema, lang=None): thesaurus["card"] = {} thesaurus["card"]["minItems"] = r["card_min"] if r["card_max"] != -1: - thesaurus["card"]["maxItems"] = r["card_max"] + thesaurus["card"]["maxItems"] = r["card_max"] thesaurus["title"] = r["title"] # default title thesaurus["description"] = r["description"] # not localized in db @@ -77,7 +85,7 @@ def update_schema(self, jsonschema, lang=None): # copy info to json schema thesauri = {} - for id,ct in collected_thesauri.items(): + for id, ct in collected_thesauri.items(): thesaurus = { "type": "array", "title": ct["title"], @@ -94,14 +102,14 @@ def update_schema(self, jsonschema, lang=None): "type": "string", "title": "Label", "description": "localized label for the keyword", - } - } + }, + }, + }, + "ui:options": { + "geonode-ui:autocomplete": reverse( + "thesaurus-keywords_autocomplete", kwargs={"thesaurusid": ct["id"]} + ) }, - "ui:options": { - 'geonode-ui:autocomplete': reverse( - "thesaurus-keywords_autocomplete", - kwargs={"thesaurusid": ct["id"]}) - } } thesaurus.update(ct["card"]) @@ -123,12 +131,14 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang:str=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): tks = {} for tk in resource.tkeywords.all(): tks[tk.id] = tk - tkls = ThesaurusKeywordLabel.objects.filter(keyword__id__in=tks.keys(), lang=lang) # read all entries in a single query + tkls = ThesaurusKeywordLabel.objects.filter( + keyword__id__in=tks.keys(), lang=lang + ) # read all entries in a single query ret = {} for tkl in tkls: @@ -155,7 +165,6 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance kw_requested = ThesaurusKeyword.objects.filter(about__in=kids) resource.tkeywords.set(kw_requested) - def load_context(self, resource: ResourceBase, context: dict): pass diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 7eba3f2619d..3f0e865fbbf 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -24,22 +24,20 @@ from django.utils.translation import gettext as _ from geonode.metadata.settings import MODEL_SCHEMA -from geonode.metadata.registry import metadata_registry logger = logging.getLogger(__name__) class MetadataManagerInterface(metaclass=ABCMeta): - pass class MetadataManager(MetadataManagerInterface): """ - The metadata manager is the bridge between the API and the geonode model. - The metadata manager will loop over all of the registered metadata handlers, - calling their update_schema(jsonschema) which will add the subschemas of the - fields handled by each handler. At the end of the loop, the schema will be ready + The metadata manager is the bridge between the API and the geonode model. + The metadata manager will loop over all of the registered metadata handlers, + calling their update_schema(jsonschema) which will add the subschemas of the + fields handled by each handler. At the end of the loop, the schema will be ready to be delivered to the caller. """ @@ -50,9 +48,9 @@ def __init__(self): def add_handler(self, handler_id, handler): self.handlers[handler_id] = handler() - + def build_schema(self, lang=None): - schema = copy.deepcopy(self.root_schema) + schema = copy.deepcopy(self.root_schema) schema["title"] = _(schema["title"]) for handler in self.handlers.values(): @@ -62,14 +60,14 @@ def build_schema(self, lang=None): def get_schema(self, lang=None): return self.build_schema(lang) - #### we dont want caching for the moment + # we dont want caching for the moment if not self.cached_schema: self.cached_schema = self.build_schema(lang) - + return self.cached_schema - + def build_schema_instance(self, resource, lang=None): - + schema = self.get_schema() instance = {} @@ -81,9 +79,9 @@ def build_schema_instance(self, resource, lang=None): handler = self.handlers[handler_id] content = handler.get_jsonschema_instance(resource, fieldname, lang) instance[fieldname] = content - + return instance - + def update_schema_instance(self, resource, json_instance): logger.info(f"RECEIVED INSTANCE {json_instance}") @@ -93,13 +91,14 @@ def update_schema_instance(self, resource, json_instance): for fieldname, subschema in schema["properties"].items(): handler = self.handlers[subschema["geonode:handler"]] handler.update_resource(resource, fieldname, json_instance) - + try: resource.save() return {"message": "The resource was updated successfully"} - - except: + + except Exception as e: + logger.warning(f"Error while updating schema instance: {e}") return {"message": "Something went wrong... The resource was not updated"} -metadata_manager = MetadataManager() \ No newline at end of file +metadata_manager = MetadataManager() diff --git a/geonode/metadata/models.py b/geonode/metadata/models.py index 71a83623907..e69de29bb2d 100644 --- a/geonode/metadata/models.py +++ b/geonode/metadata/models.py @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/geonode/metadata/registry.py b/geonode/metadata/registry.py index 651771497cd..6f6f6de3d4e 100644 --- a/geonode/metadata/registry.py +++ b/geonode/metadata/registry.py @@ -11,9 +11,7 @@ class MetadataHandlersRegistry: def init_registry(self): self.register() - logger.info( - f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}" - ) + logger.info(f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}") def register(self): for handler_id, module_path in METADATA_HANDLERS.items(): @@ -24,4 +22,4 @@ def get_registry(cls): return MetadataHandlersRegistry.handler_registry -metadata_registry = MetadataHandlersRegistry() \ No newline at end of file +metadata_registry = MetadataHandlersRegistry() diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 5c3a08365af..3f6e27ce83c 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -2,13 +2,12 @@ from geonode.settings import PROJECT_ROOT MODEL_SCHEMA = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "{GEONODE_SITE}/resource.json", - "title": "GeoNode resource", - "type": "object", - "properties": { - } - } + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "{GEONODE_SITE}/resource.json", + "title": "GeoNode resource", + "type": "object", + "properties": {}, +} # The base schema is defined as a file in order to be customizable from other GeoNode instances JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/base_schema.json") @@ -18,4 +17,4 @@ "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", "region": "geonode.metadata.handlers.region.RegionHandler", "doi": "geonode.metadata.handlers.doi.DOIHandler", -} \ No newline at end of file +} diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests.py index 7ce503c2dd9..e69de29bb2d 100644 --- a/geonode/metadata/tests.py +++ b/geonode/metadata/tests.py @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py index 89461c03f5f..3b4e749dfe2 100644 --- a/geonode/metadata/urls.py +++ b/geonode/metadata/urls.py @@ -1,4 +1,3 @@ from geonode.metadata.api.urls import router -from django.urls import path, include, re_path -urlpatterns = [] + router.urls \ No newline at end of file +urlpatterns = [] + router.urls diff --git a/geonode/metadata/views.py b/geonode/metadata/views.py index 44ea94be1d1..f8417b96341 100644 --- a/geonode/metadata/views.py +++ b/geonode/metadata/views.py @@ -1,2 +1 @@ # Create your views here - From e2cc1c7bddab8c1b19fda5364a0124ee34236e34 Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 4 Nov 2024 10:39:05 +0100 Subject: [PATCH 46/91] Added contacts schema. Moved tkeywords autocomplete. --- geonode/base/api/urls.py | 1 - geonode/base/api/views.py | 64 +------------- geonode/metadata/api/urls.py | 13 +++ geonode/metadata/api/views.py | 82 +++++++++++++++++- geonode/metadata/handlers/contact.py | 115 +++++++++++++++++++++++++ geonode/metadata/handlers/region.py | 5 -- geonode/metadata/handlers/thesaurus.py | 2 +- geonode/metadata/urls.py | 4 +- 8 files changed, 212 insertions(+), 74 deletions(-) create mode 100644 geonode/metadata/handlers/contact.py diff --git a/geonode/base/api/urls.py b/geonode/base/api/urls.py index a2575f8c36c..69121b96f6f 100644 --- a/geonode/base/api/urls.py +++ b/geonode/base/api/urls.py @@ -26,7 +26,6 @@ router.register(r"categories", views.TopicCategoryViewSet, "categories") router.register(r"keywords", views.HierarchicalKeywordViewSet, "keywords") router.register(r"tkeywords", views.ThesaurusKeywordViewSet, "tkeywords") -router.register(r"thesaurus", views.ThesaurusViewSet, "thesaurus") router.register(r"regions", views.RegionViewSet, "regions") urlpatterns = [] diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index daf9b4b9c87..8fbbdf71407 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -27,14 +27,12 @@ from django.apps import apps from django.core.validators import URLValidator -from django.http import JsonResponse from django.shortcuts import get_object_or_404 from django.urls import reverse from django.conf import settings from django.db.models import Subquery, QuerySet from django.http.request import QueryDict from django.contrib.auth import get_user_model -from django.utils.translation import get_language from drf_spectacular.utils import extend_schema from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin @@ -56,7 +54,7 @@ from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.favorite.models import Favorite -from geonode.base.models import Configuration, ExtraMetadata, LinkedResource, ThesaurusKeywordLabel, Thesaurus +from geonode.base.models import Configuration, ExtraMetadata, LinkedResource from geonode.thumbs.exceptions import ThumbnailError from geonode.thumbs.thumbnails import create_thumbnail from geonode.thumbs.utils import _decode_base64, BASE64_PATTERN @@ -106,7 +104,7 @@ ) from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination -from geonode.base.utils import validate_extra_metadata, remove_country_from_languagecode +from geonode.base.utils import validate_extra_metadata import logging @@ -229,64 +227,6 @@ class ThesaurusKeywordViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveM pagination_class = GeoNodeApiPagination -class ThesaurusViewSet(DynamicModelViewSet): - - queryset = Thesaurus.objects.all() - serializer_class = ThesaurusKeywordSerializer - - @extend_schema( - methods=["get"], - description="API endpoint allowing to retrieve the published Resources.", - ) - @action( - detail=False, - methods=["get"], - url_path="(?P\d+)/keywords/autocomplete", # noqa - url_name="keywords_autocomplete", - ) - def tkeywords_autocomplete(self, request, thesaurusid): - - lang = get_language() - all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) - - # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again - localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( - "keyword_id" - ) - if not localized_k_ids_qs.exists(): - lang = remove_country_from_languagecode(lang) - localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( - "keyword_id" - ) - - # consider all the keywords that do not have a translation in the requested language - keywords_not_translated_qs = ( - all_keywords_qs.exclude(id__in=localized_k_ids_qs).order_by("id").distinct("id").values("id") - ) - - qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).order_by("label") - if q := request.query_params.get("q", None): - qs = qs.filter(label__istartswith=q) - - ret = [] - for tkl in qs.all(): - ret.append( - { - "id": tkl.keyword.about, - "label": tkl.label, - } - ) - for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).order_by("alt_label").all(): - ret.append( - { - "id": tk.about, - "label": f"! {tk.alt_label}", - } - ) - - return JsonResponse({"results": ret}) - - class TopicCategoryViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): """ API endpoint that lists categories. diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 85341eda3ca..bec90e64ec3 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -16,8 +16,21 @@ # along with this program. If not, see . # ######################################################################### +from django.urls import path from rest_framework import routers + from geonode.metadata.api import views +from geonode.metadata.api.views import ProfileAutocomplete router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") + +urlpatterns = router.urls + [ + path( + r"metadata/autocomplete/thesaurus/)/keywords", + views.tkeywords_autocomplete, + name="metadata_autocomplete_tkeywords", + ), + path(r"metadata/autocomplete/users", ProfileAutocomplete.as_view(), name="metadata_autocomplete_users"), + # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), +] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index fc604f3d7ff..af511ad803e 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -18,17 +18,22 @@ ######################################################################### import logging - +from dal import autocomplete +from django.contrib.auth import get_user_model +from django.core.handlers.wsgi import WSGIRequest from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response from django.http import JsonResponse from django.utils.translation.trans_real import get_language_from_request +from django.utils.translation import get_language +from django.db.models import Q -from geonode.base.models import ResourceBase +from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.base.utils import remove_country_from_languagecode from geonode.metadata.manager import metadata_manager - +from geonode.people.utils import get_available_users logger = logging.getLogger(__name__) @@ -85,3 +90,74 @@ def schema_instance(self, request, pk=None): except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} return Response(result) + + +def tkeywords_autocomplete(request: WSGIRequest, thesaurusid): + + lang = get_language() + all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( + "keyword_id" + ) + if not localized_k_ids_qs.exists(): + lang = remove_country_from_languagecode(lang) + localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( + "keyword_id" + ) + + # consider all the keywords that do not have a translation in the requested language + keywords_not_translated_qs = ( + all_keywords_qs.exclude(id__in=localized_k_ids_qs).order_by("id").distinct("id").values("id") + ) + + qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).order_by("label") + # if q := request.query_params.get("q", None): + if q := request.GET.get("q", None): + qs = qs.filter(label__istartswith=q) + + ret = [] + for tkl in qs.all(): + ret.append( + { + "id": tkl.keyword.about, + "label": tkl.label, + } + ) + for tk in all_keywords_qs.filter(id__in=keywords_not_translated_qs).order_by("alt_label").all(): + ret.append( + { + "id": tk.about, + "label": f"! {tk.alt_label}", + } + ) + + return JsonResponse({"results": ret}) + + +class ProfileAutocomplete(autocomplete.Select2QuerySetView): + def get_queryset(self): + if self.request and self.request.user: + qs = get_available_users(self.request.user) + else: + qs = get_user_model().objects.all() + + if self.q: + qs = qs.filter( + Q(username__icontains=self.q) + | Q(email__icontains=self.q) + | Q(first_name__icontains=self.q) + | Q(last_name__icontains=self.q) + ) + + return qs + + def get_results(self, context): + def get_label(user): + names = [n for n in (user.first_name, user.last_name) if n] + postfix = f" {' '.join(names)}" if names else "" + return f"{user.username}{postfix}" + + """Return data for the 'results' key of the response.""" + return [{"id": self.get_result_value(result), "label": get_label(result)} for result in context["object_list"]] diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py new file mode 100644 index 00000000000..e4349b2f788 --- /dev/null +++ b/geonode/metadata/handlers/contact.py @@ -0,0 +1,115 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.people import Roles + +logger = logging.getLogger(__name__) + + +class ContactHandler(MetadataHandler): + """ + The RegionsHandler adds the Regions model options to the schema + """ + + def update_schema(self, jsonschema, lang=None): + + contacts = {} + for role in Roles: + card = ("1" if role.is_required else "0") + ".." + ("N" if role.is_multivalue else "1") + if role.is_multivalue: + contact = { + "type": "array", + "title": _(role.label) + " " + card, + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "title": _("User id"), + }, + "label": { + "type": "string", + "title": _("User name"), + }, + }, + }, + "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_users")}, + "xxrequired": role.is_required, + } + else: + contact = { + "type": "object", + "title": _(role.label) + " " + card, + "properties": { + "id": { + "type": "string", + "title": _("User id"), + }, + "label": { + "type": "string", + "title": _("User name"), + }, + }, + "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_users")}, + "xxrequired": role.is_required, + } + + contacts[role.name] = contact + + jsonschema["properties"]["contacts"] = { + "type": "object", + "title": _("Contacts"), + "properties": contacts, + "geonode:handler": "contact", + } + + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + def __create_user_entry(user): + return {"id": user.username, "name": f"{user.username} - {user.first_name} {user.last_name}"} + + contacts = {} + for role in Roles: + if role.is_multivalue: + content = [__create_user_entry(user) for user in resource.__get_contact_role_elements__(role) or []] + else: + users = resource.__get_contact_role_elements__(role) + if not users and role == Roles.OWNER: + users = [resource.owner] + content = __create_user_entry(users[0]) if users else None + + contacts[role.name] = content + + return contacts + + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + + pass + + def load_context(self, resource: ResourceBase, context: dict): + + pass diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 4b7ba7d00e4..33d643d1bbe 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -52,11 +52,6 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "attribution", "regions", regions) return jsonschema - @classmethod - def serialize(cls, db_value): - # TODO - return [] - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): return None diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index f3c87d66513..de2147e5bdd 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -107,7 +107,7 @@ def update_schema(self, jsonschema, lang=None): }, "ui:options": { "geonode-ui:autocomplete": reverse( - "thesaurus-keywords_autocomplete", kwargs={"thesaurusid": ct["id"]} + "metadata_autocomplete_tkeywords", kwargs={"thesaurusid": ct["id"]} ) }, } diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py index 3b4e749dfe2..06802fcd87c 100644 --- a/geonode/metadata/urls.py +++ b/geonode/metadata/urls.py @@ -1,3 +1,3 @@ -from geonode.metadata.api.urls import router +from geonode.metadata.api.urls import urlpatterns -urlpatterns = [] + router.urls +urlpatterns += [] # make flake8 happy \ No newline at end of file From b095a7aba7ed697ae7b46710e7faebd8146dda3b Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 4 Nov 2024 14:22:31 +0100 Subject: [PATCH 47/91] Load+store contacts --- geonode/metadata/api/views.py | 2 +- geonode/metadata/handlers/contact.py | 20 +++++++++++++++----- geonode/metadata/settings.py | 1 + geonode/metadata/urls.py | 2 +- geonode/people/__init__.py | 4 ++++ 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index af511ad803e..edcd70b3e84 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -156,7 +156,7 @@ def get_queryset(self): def get_results(self, context): def get_label(user): names = [n for n in (user.first_name, user.last_name) if n] - postfix = f" {' '.join(names)}" if names else "" + postfix = f" ({' '.join(names)})" if names else "" return f"{user.username}{postfix}" """Return data for the 'results' key of the response.""" diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index e4349b2f788..479f8ed2c67 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -19,6 +19,7 @@ import logging +from django.contrib.auth import get_user_model from rest_framework.reverse import reverse from django.utils.translation import gettext as _ @@ -31,11 +32,10 @@ class ContactHandler(MetadataHandler): """ - The RegionsHandler adds the Regions model options to the schema + Handles role contacts """ def update_schema(self, jsonschema, lang=None): - contacts = {} for role in Roles: card = ("1" if role.is_required else "0") + ".." + ("N" if role.is_multivalue else "1") @@ -90,7 +90,9 @@ def update_schema(self, jsonschema, lang=None): def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): def __create_user_entry(user): - return {"id": user.username, "name": f"{user.username} - {user.first_name} {user.last_name}"} + names = [n for n in (user.first_name, user.last_name) if n] + postfix = f" ({' '.join(names)})" if names else "" + return {"id": user.id, "label": f"{user.username}{postfix}"} contacts = {} for role in Roles: @@ -107,8 +109,16 @@ def __create_user_entry(user): return contacts def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - - pass + data = json_instance[field_name] + logger.info(f"CONTACTS {data}") + for rolename, users in data.items(): + if rolename == Roles.OWNER.OWNER.name: + logger.debug("Skipping role owner") + continue + role = Roles.get_role_by_name(rolename) + ids = [u["id"] for u in users] + profiles = get_user_model().objects.filter(pk__in=ids) + resource.__set_contact_role_element__(profiles, role) def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 3f6e27ce83c..69eeebc1c1d 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -17,4 +17,5 @@ "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", "region": "geonode.metadata.handlers.region.RegionHandler", "doi": "geonode.metadata.handlers.doi.DOIHandler", + "contact": "geonode.metadata.handlers.contact.ContactHandler", } diff --git a/geonode/metadata/urls.py b/geonode/metadata/urls.py index 06802fcd87c..7e96ae33c7b 100644 --- a/geonode/metadata/urls.py +++ b/geonode/metadata/urls.py @@ -1,3 +1,3 @@ from geonode.metadata.api.urls import urlpatterns -urlpatterns += [] # make flake8 happy \ No newline at end of file +urlpatterns += [] # make flake8 happy diff --git a/geonode/people/__init__.py b/geonode/people/__init__.py index 5af39af536c..37e0c8a1c30 100644 --- a/geonode/people/__init__.py +++ b/geonode/people/__init__.py @@ -79,3 +79,7 @@ def get_multivalue_ones(cls): @classmethod def get_toggled_ones(cls): return [role for role in cls if role.is_toggled_in_metadata_editor] + + @classmethod + def get_role_by_name(cls, name): + return next((role for role in cls if role.name == name)) From aaf4f2fdf4d6421495052256268c83839ca35b1e Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 4 Nov 2024 16:13:20 +0100 Subject: [PATCH 48/91] Added linked resources handler --- geonode/metadata/api/urls.py | 7 +- geonode/metadata/api/views.py | 9 +++ geonode/metadata/handlers/contact.py | 1 + geonode/metadata/handlers/linkedresource.py | 71 +++++++++++++++++++++ geonode/metadata/settings.py | 1 + 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 geonode/metadata/handlers/linkedresource.py diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index bec90e64ec3..63355613b17 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -20,7 +20,7 @@ from rest_framework import routers from geonode.metadata.api import views -from geonode.metadata.api.views import ProfileAutocomplete +from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") @@ -32,5 +32,10 @@ name="metadata_autocomplete_tkeywords", ), path(r"metadata/autocomplete/users", ProfileAutocomplete.as_view(), name="metadata_autocomplete_users"), + path( + r"metadata/autocomplete/resources", + MetadataLinkedResourcesAutocomplete.as_view(), + name="metadata_autocomplete_resources", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index edcd70b3e84..9cd8cbd185e 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -32,6 +32,7 @@ from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.base.utils import remove_country_from_languagecode +from geonode.base.views import LinkedResourcesAutocomplete from geonode.metadata.manager import metadata_manager from geonode.people.utils import get_available_users @@ -161,3 +162,11 @@ def get_label(user): """Return data for the 'results' key of the response.""" return [{"id": self.get_result_value(result), "label": get_label(result)} for result in context["object_list"]] + + +class MetadataLinkedResourcesAutocomplete(LinkedResourcesAutocomplete): + def get_results(self, context): + return [ + {"id": self.get_result_value(result), "label": self.get_result_label(result)} + for result in context["object_list"] + ] diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 479f8ed2c67..d6d07f79d3f 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -67,6 +67,7 @@ def update_schema(self, jsonschema, lang=None): "id": { "type": "string", "title": _("User id"), + "ui:widget": "hidden", }, "label": { "type": "string", diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py new file mode 100644 index 00000000000..088f7ea682c --- /dev/null +++ b/geonode/metadata/handlers/linkedresource.py @@ -0,0 +1,71 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase, LinkedResource +from geonode.metadata.handlers.abstract import MetadataHandler + +logger = logging.getLogger(__name__) + + +class LinkedResourceHandler(MetadataHandler): + + def update_schema(self, jsonschema, lang=None): + linked = { + "type": "array", + "title": _("Linked resources"), + "description": _("Resources related to this one"), + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + }, + "label": {"type": "string", "title": _("title")}, + }, + }, + "geonode:handler": "linkedresource", + "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_resources")}, + } + + jsonschema["properties"]["linkedresources"] = linked + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + return [{"id": lr.target.id, "label": lr.target.title} for lr in resource.get_linked_resources()] + + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + + data = json_instance[field_name] + new_ids = {item["id"] for item in data} + + # add requested links + for res_id in new_ids: + target = ResourceBase.objects.get(pk=res_id) + LinkedResource.objects.get_or_create(source=resource, target=target, internal=False) + + # delete remaining links + LinkedResource.objects.filter(source_id=resource.id, internal=False).exclude(target_id__in=new_ids).delete() + + def load_context(self, resource: ResourceBase, context: dict): + pass diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 69eeebc1c1d..22497ef4846 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -17,5 +17,6 @@ "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", "region": "geonode.metadata.handlers.region.RegionHandler", "doi": "geonode.metadata.handlers.doi.DOIHandler", + "linkedresource": "geonode.metadata.handlers.linkedresource.LinkedResourceHandler", "contact": "geonode.metadata.handlers.contact.ContactHandler", } From b4f11293ec38803ef275fcf86a373bc517a02c65 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 5 Nov 2024 12:29:03 +0100 Subject: [PATCH 49/91] Regions autocomplete --- geonode/metadata/api/urls.py | 8 +++++++- geonode/metadata/api/views.py | 11 ++++++++++- geonode/metadata/handlers/region.py | 21 ++++++++++++++------- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 63355613b17..b2e5dc710ca 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -20,7 +20,8 @@ from rest_framework import routers from geonode.metadata.api import views -from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete +from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete, \ + MetadataRegionsAutocomplete router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") @@ -37,5 +38,10 @@ MetadataLinkedResourcesAutocomplete.as_view(), name="metadata_autocomplete_resources", ), + path( + r"metadata/autocomplete/regions", + MetadataRegionsAutocomplete.as_view(), + name="metadata_autocomplete_regions", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 9cd8cbd185e..b26a3a99db8 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -32,7 +32,7 @@ from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.base.utils import remove_country_from_languagecode -from geonode.base.views import LinkedResourcesAutocomplete +from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete from geonode.metadata.manager import metadata_manager from geonode.people.utils import get_available_users @@ -170,3 +170,12 @@ def get_results(self, context): {"id": self.get_result_value(result), "label": self.get_result_label(result)} for result in context["object_list"] ] + + +class MetadataRegionsAutocomplete(RegionAutocomplete): + def get_results(self, context): + return [ + {"id": self.get_result_value(result), "label": self.get_result_label(result)} + for result in context["object_list"] + ] + diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 33d643d1bbe..6022455673c 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -35,17 +35,24 @@ class RegionHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): - from geonode.base.models import Region - - subschema = [{"const": tc.code, "title": tc.name} for tc in Region.objects.order_by("name")] - regions = { "type": "array", "title": _("Regions"), "description": _("keyword identifies a location"), - "items": {"type": "string", "anyOf": subschema}, + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + }, + "label": { + "type": "string", + "title": _("title") + }, + }, + }, "geonode:handler": "region", - "ui:options": {"geonode-ui:autocomplete": reverse("autocomplete_region")}, + "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_regions")}, } # add regions after Attribution @@ -54,7 +61,7 @@ def update_schema(self, jsonschema, lang=None): def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - return None + return [] def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): From d358c0a159ad2b2c038d633de3e8b376a809ae09 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 5 Nov 2024 12:43:42 +0100 Subject: [PATCH 50/91] Regions load/store --- geonode/metadata/handlers/region.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 6022455673c..81c71888244 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -22,7 +22,7 @@ from rest_framework.reverse import reverse from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase +from geonode.base.models import ResourceBase, Region from geonode.metadata.handlers.abstract import MetadataHandler logger = logging.getLogger(__name__) @@ -60,12 +60,16 @@ def update_schema(self, jsonschema, lang=None): return jsonschema def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - - return [] + return [{"id": r.id, "label": r.name} for r in resource.regions.all()] def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - pass + data = json_instance[field_name] + new_ids = {item["id"] for item in data} + logger.info(f"Regions added {data} --> {new_ids}") + + regions = Region.objects.filter(id__in=new_ids) + resource.regions.set(regions) def load_context(self, resource: ResourceBase, context: dict): From d4723684a802e2b81ea8178eba11f5fc559f9b00 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 6 Nov 2024 10:30:10 +0200 Subject: [PATCH 51/91] Extending the Regions autocomplete results --- geonode/metadata/api/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b26a3a99db8..b46a7211fde 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -174,8 +174,11 @@ def get_results(self, context): class MetadataRegionsAutocomplete(RegionAutocomplete): def get_results(self, context): + + context['object_list'] = self.object_list + return [ {"id": self.get_result_value(result), "label": self.get_result_label(result)} - for result in context["object_list"] + for result in context['object_list'] ] From 565f27d1b8a948e8cdbdba438e917dfac877e64f Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 6 Nov 2024 10:34:46 +0200 Subject: [PATCH 52/91] format fixing --- geonode/metadata/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b46a7211fde..399c5f56084 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -179,6 +179,6 @@ def get_results(self, context): return [ {"id": self.get_result_value(result), "label": self.get_result_label(result)} - for result in context['object_list'] + for result in context["object_list"] ] From 838a83164ede0983d972a5bd08724605e7af40a1 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 6 Nov 2024 14:12:51 +0200 Subject: [PATCH 53/91] update the MetadataRegionsAutocomplete class --- geonode/metadata/api/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 399c5f56084..b26a3a99db8 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -174,9 +174,6 @@ def get_results(self, context): class MetadataRegionsAutocomplete(RegionAutocomplete): def get_results(self, context): - - context['object_list'] = self.object_list - return [ {"id": self.get_result_value(result), "label": self.get_result_label(result)} for result in context["object_list"] From 05a60b70d921cd58d907c84f65e2b81ef06ccadd Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 20 Nov 2024 19:13:36 +0100 Subject: [PATCH 54/91] Metadata: review label i18n --- geonode/metadata/handlers/base.py | 2 +- geonode/metadata/handlers/doi.py | 4 ++- geonode/metadata/handlers/linkedresource.py | 2 +- geonode/metadata/handlers/thesaurus.py | 4 +-- .../jsonschema_examples/base_schema.json | 35 ++++++++----------- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 2eb30a26cf8..2f2e56d019a 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -188,7 +188,7 @@ def localize(subschema: dict, annotation_name): # building the full base schema for property_name, subschema in self.base_schema.items(): localize(subschema, "title") - localize(subschema, "abstract") + localize(subschema, "description") jsonschema["properties"][property_name] = subschema diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index bec9ff6cd36..43ecd8be55a 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -19,6 +19,8 @@ import logging +from django.utils.translation import gettext as _ + from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler @@ -33,7 +35,7 @@ def update_schema(self, jsonschema, lang=None): doi_schema = { "type": "string", "title": "DOI", - "description": "a DOI will be added by Admin before publication.", + "description": _("a DOI will be added by Admin before publication."), "maxLength": 255, "geonode:handler": "doi", } diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py index 088f7ea682c..60f74cc6e2b 100644 --- a/geonode/metadata/handlers/linkedresource.py +++ b/geonode/metadata/handlers/linkedresource.py @@ -33,7 +33,7 @@ class LinkedResourceHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): linked = { "type": "array", - "title": _("Linked resources"), + "title": _("Related resources"), "description": _("Resources related to this one"), "items": { "type": "object", diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index de2147e5bdd..9d6fddd9b4a 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -117,8 +117,8 @@ def update_schema(self, jsonschema, lang=None): tkeywords = { "type": "object", - "title": _("Thesaurus keywords"), - "description": _("Keywords from controlled vocabularies"), + "title": _("Keywords from Thesaurus"), + "description": _("List of keywords from Thesaurus"), "geonode:handler": "thesaurus", "properties": thesauri, # "ui:options": { diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 05424b369cc..1e968b01e65 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -1,7 +1,7 @@ { "uuid": { "type": "string", - "description": "The UUID of the resource", + "title": "UUID", "maxLength": 36, "readOnly": true, "NO_ui:widget": "hidden", @@ -10,14 +10,14 @@ "title": { "type": "string", "title": "title", - "description": "Name by which the cited resource is known", + "description": "name by which the cited resource is known", "maxLength": 255, "geonode:handler": "base" }, "abstract": { "type": "string", "title": "abstract", - "description": "Brief narrative summary of the content of the resource(s)", + "description": "brief narrative summary of the content of the resource(s)", "maxLength": 2000, "ui:options": { "widget": "textarea", @@ -40,27 +40,22 @@ "title": "group", "todo": true }, - "keywords": { - "type": "string", - "title": "keywords", - "todo": true - }, "category": { "type": "string", - "title": "category", - "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets", + "title": "Category", + "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.", "maxLength": 255 }, "language": { "type": "string", "title": "language", - "description": "Language used within the dataset", + "description": "language used within the dataset", "maxLength": 255, "default": "eng" }, "license": { "type": "string", - "title": "license", + "title": "License", "description": "license of the dataset", "maxLength": 255, "default": "eng" @@ -68,7 +63,7 @@ "attribution": { "type": "string", "title": "Attribution", - "description": "Authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", + "description": "authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", "maxLength": 2048 }, "data_quality_statement": { @@ -83,13 +78,13 @@ }, "restriction_code_type": { "type": "string", - "title": "restriction_code_type", + "title": "restrictions", "description": "limitation(s) placed upon the access or use of the data.", "maxLength": 255 }, "constraints_other": { "type": "string", - "title": "Other constrains", + "title": "Other constraints", "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", "ui:options": { "widget": "textarea", @@ -99,13 +94,13 @@ "edition": { "type": "string", "title": "edition", - "description": "Version of the cited resource", + "description": "version of the cited resource", "maxLength": 255 }, "purpose": { "type": "string", "title": "purpose", - "description": "Summary of the intentions with which the resource(s) was developed", + "description": "summary of the intentions with which the resource(s) was developed", "maxLength": 500, "ui:options": { "widget": "textarea", @@ -127,13 +122,13 @@ "temporal_extent_start": { "type": "string", "format": "date-time", - "title": "temporal_extent_start", + "title": "temporal extent start", "description": "time period covered by the content of the dataset (start)" }, "temporal_extent_end": { "type": "string", "format": "date-time", - "title": "temporal_extent_end", + "title": "temporal extent end", "description": "time period covered by the content of the dataset (end)" }, "maintenance_frequency": { @@ -144,7 +139,7 @@ }, "spatial_representation_type": { "type": "string", - "title": "spatial_representation_type", + "title": "spatial representation type", "description": "method used to represent geographic information in the dataset.", "maxLength": 255 } From 337e7811bb326e456c5207f65d0c9efc709d07c6 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 20 Nov 2024 19:16:18 +0100 Subject: [PATCH 55/91] Metadata: hkeywords handler - WIP --- geonode/metadata/api/urls.py | 7 ++- geonode/metadata/api/views.py | 12 +++- geonode/metadata/handlers/base.py | 7 --- geonode/metadata/handlers/hkeyword.py | 87 +++++++++++++++++++++++++++ geonode/metadata/settings.py | 1 + 5 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 geonode/metadata/handlers/hkeyword.py diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index b2e5dc710ca..1a1c01fcebe 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -21,7 +21,7 @@ from geonode.metadata.api import views from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete, \ - MetadataRegionsAutocomplete + MetadataRegionsAutocomplete, MetadataHKeywordAutocomplete router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") @@ -43,5 +43,10 @@ MetadataRegionsAutocomplete.as_view(), name="metadata_autocomplete_regions", ), + path( + r"metadata/autocomplete/hkeywords", + MetadataHKeywordAutocomplete.as_view(), + name="metadata_autocomplete_hkeywords", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b26a3a99db8..25230a74c4f 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -32,7 +32,7 @@ from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.base.utils import remove_country_from_languagecode -from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete +from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete from geonode.metadata.manager import metadata_manager from geonode.people.utils import get_available_users @@ -179,3 +179,13 @@ def get_results(self, context): for result in context["object_list"] ] + +class MetadataHKeywordAutocomplete(HierarchicalKeywordAutocomplete): + def get_results(self, context): + return [ + {"id": self.get_result_value(result), "label": self.get_result_label(result)} + for result in context["object_list"] + ] + + + diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 2f2e56d019a..6dd2f43a5a3 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -111,12 +111,6 @@ def deserialize(cls, field_value): return License.objects.get(identifier=field_value) -class KeywordsSubHandler(SubHandler): - @classmethod - def serialize(cls, value): - return "TODO!!!" - - class RestrictionsSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): @@ -161,7 +155,6 @@ def deserialize(cls, field_value): "date": DateSubHandler, "language": LanguageSubHandler, "license": LicenseSubHandler, - "keywords": KeywordsSubHandler, "maintenance_frequency": FrequencySubHandler, "restriction_code_type": RestrictionsSubHandler, "spatial_representation_type": SpatialRepresentationTypeSubHandler, diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py new file mode 100644 index 00000000000..a40189f759e --- /dev/null +++ b/geonode/metadata/handlers/hkeyword.py @@ -0,0 +1,87 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase, LinkedResource +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.resource.utils import KeywordHandler + +logger = logging.getLogger(__name__) + + +class HKeywordHandler(MetadataHandler): + + def update_schema(self, jsonschema, lang=None): + hkeywords = { + "type": "array", + "title": _("Keywords"), + "description": _("Hierarchical keywords"), + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "title": _("User id"), + "ui:widget": "hidden", + }, + "label": { + "type": "string", + "title": _("User name"), + }, + }, + + # "ui:options": { + # "geonode-ui:autocomplete": { + # "url": reverse("metadata_autocomplete_hkeywords"), + # "creatable": True, + # }, + # }, + }, + "ui:options": { + "geonode-ui:autocomplete": { + "url": reverse("metadata_autocomplete_hkeywords"), + "creatable": True, + }, + }, + + "geonode:handler": "hkeyword", + } + + # jsonschema["properties"]["hkeywords"] = hkeywords + self._add_after(jsonschema, "tkeywords", "hkeywords", hkeywords) + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + # return [keyword.name for keyword in resource.keywords.all()] + return [{"id": keyword.name, "label": keyword.name} for keyword in resource.keywords.all()] + + + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + # TODO: see also resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) + hkeywords = json_instance["hkeywords"] + cleaned = [k for k in hkeywords if k] + logger.warning(f"HKEYWORDS {hkeywords} --> {cleaned}") + KeywordHandler(resource, cleaned).set_keywords() + + def load_context(self, resource: ResourceBase, context: dict): + pass diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 22497ef4846..f1fef43e321 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -15,6 +15,7 @@ METADATA_HANDLERS = { "base": "geonode.metadata.handlers.base.BaseHandler", "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", + "hkeyword": "geonode.metadata.handlers.hkeyword.HKeywordHandler", "region": "geonode.metadata.handlers.region.RegionHandler", "doi": "geonode.metadata.handlers.doi.DOIHandler", "linkedresource": "geonode.metadata.handlers.linkedresource.LinkedResourceHandler", From a039e167dc758faff170c12d020706273449f67a Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 20 Nov 2024 19:17:02 +0100 Subject: [PATCH 56/91] Minor improvement --- geonode/metadata/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 3f0e865fbbf..48bc9eb1442 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -53,7 +53,8 @@ def build_schema(self, lang=None): schema = copy.deepcopy(self.root_schema) schema["title"] = _(schema["title"]) - for handler in self.handlers.values(): + for key, handler in self.handlers.items(): + logger.debug(f"build_schema: update schema -> {key}") schema = handler.update_schema(schema, lang) return schema @@ -72,6 +73,7 @@ def build_schema_instance(self, resource, lang=None): instance = {} for fieldname, field in schema["properties"].items(): + # logger.debug(f"build_schema_instance: getting handler for property {fieldname}") handler_id = field.get("geonode:handler", None) if not handler_id: logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") From 0ab811b83480526ff6bdce058a1b25814751403b Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 21 Nov 2024 12:41:07 +0100 Subject: [PATCH 57/91] Metadata: hkeywords handler --- geonode/metadata/api/views.py | 2 +- geonode/metadata/handlers/hkeyword.py | 26 ++------------------------ 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 25230a74c4f..7b8eb0f47d6 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -183,7 +183,7 @@ def get_results(self, context): class MetadataHKeywordAutocomplete(HierarchicalKeywordAutocomplete): def get_results(self, context): return [ - {"id": self.get_result_value(result), "label": self.get_result_label(result)} + self.get_result_label(result) for result in context["object_list"] ] diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py index a40189f759e..f2966c5c7c6 100644 --- a/geonode/metadata/handlers/hkeyword.py +++ b/geonode/metadata/handlers/hkeyword.py @@ -37,25 +37,7 @@ def update_schema(self, jsonschema, lang=None): "title": _("Keywords"), "description": _("Hierarchical keywords"), "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "title": _("User id"), - "ui:widget": "hidden", - }, - "label": { - "type": "string", - "title": _("User name"), - }, - }, - - # "ui:options": { - # "geonode-ui:autocomplete": { - # "url": reverse("metadata_autocomplete_hkeywords"), - # "creatable": True, - # }, - # }, + "type": "string", }, "ui:options": { "geonode-ui:autocomplete": { @@ -63,18 +45,14 @@ def update_schema(self, jsonschema, lang=None): "creatable": True, }, }, - "geonode:handler": "hkeyword", } - # jsonschema["properties"]["hkeywords"] = hkeywords self._add_after(jsonschema, "tkeywords", "hkeywords", hkeywords) return jsonschema def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - # return [keyword.name for keyword in resource.keywords.all()] - return [{"id": keyword.name, "label": keyword.name} for keyword in resource.keywords.all()] - + return [keyword.name for keyword in resource.keywords.all()] def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): # TODO: see also resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) From 64c41a946fb8cf2d2f4dff2a03d2dc0500972be9 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 21 Nov 2024 13:36:09 +0100 Subject: [PATCH 58/91] Metadata: group handler --- geonode/metadata/api/urls.py | 7 +- geonode/metadata/api/views.py | 26 +++++++ geonode/metadata/handlers/group.py | 75 +++++++++++++++++++ .../jsonschema_examples/base_schema.json | 5 -- geonode/metadata/settings.py | 1 + 5 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 geonode/metadata/handlers/group.py diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 1a1c01fcebe..bc1d04bc08b 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -21,7 +21,7 @@ from geonode.metadata.api import views from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete, \ - MetadataRegionsAutocomplete, MetadataHKeywordAutocomplete + MetadataRegionsAutocomplete, MetadataHKeywordAutocomplete, MetadataGroupAutocomplete router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") @@ -48,5 +48,10 @@ MetadataHKeywordAutocomplete.as_view(), name="metadata_autocomplete_hkeywords", ), + path( + r"metadata/autocomplete/groups", + MetadataGroupAutocomplete.as_view(), + name="metadata_autocomplete_groups", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 7b8eb0f47d6..6edc9e770a5 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -33,8 +33,10 @@ from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.base.utils import remove_country_from_languagecode from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete +from geonode.groups.models import GroupProfile from geonode.metadata.manager import metadata_manager from geonode.people.utils import get_available_users +from geonode.security.utils import get_user_visible_groups logger = logging.getLogger(__name__) @@ -188,4 +190,28 @@ def get_results(self, context): ] +class MetadataGroupAutocomplete(autocomplete.Select2QuerySetView): + def get_queryset(self): + user = self.request.user if self.request else None + + if not user: + qs = GroupProfile.objects.none() + elif user.is_superuser or user.is_staff: + qs = GroupProfile.objects.all() + else: + qs = GroupProfile.objects.filter(groupmember__user=user) + + qs = qs.order_by("title") + if self.q: + qs = qs.filter(title__icontains=self.q) + + return qs + + def get_results(self, context): + return [ + {"id": self.get_result_value(result), "label": result.title} + for result in context["object_list"] + ] + + diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py new file mode 100644 index 00000000000..723176b6a0d --- /dev/null +++ b/geonode/metadata/handlers/group.py @@ -0,0 +1,75 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase +from geonode.groups.models import GroupProfile +from geonode.metadata.handlers.abstract import MetadataHandler + +logger = logging.getLogger(__name__) + + +class GroupHandler(MetadataHandler): + """ + The GroupHandler handles the group FK field. + This handler is only used in the first transition to the new metadata editor, and will be then replaced by + an entry in the resource management panel + """ + + def update_schema(self, jsonschema, lang=None): + group_schema = { + "type": "object", + "title": _("group"), + "properties": { + "id": { + "type": "string", + "ui:widget": "hidden", + }, + "label": { + "type": "string", + "title": _("group"), + }, + }, + "geonode:handler": "group", + "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_groups")}, + } + + # add group after date_type + self._add_after(jsonschema, "date_type", "group", group_schema) + + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + return {"id": resource.group.groupprofile.pk, "label": resource.group.groupprofile.title} if resource.group else None + + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + data = json_instance[field_name] + id = data.get("id", None) if data else None + if id is not None: + gp = GroupProfile.objects.get(pk=id) + resource.group = gp.group + else: + resource.group = None + + def load_context(self, resource: ResourceBase, context: dict): + pass diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 1e968b01e65..2a43d446f47 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -35,11 +35,6 @@ "title": "date type", "maxLength": 255 }, - "group": { - "type": "string", - "title": "group", - "todo": true - }, "category": { "type": "string", "title": "Category", diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index f1fef43e321..4bd311dff60 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -17,6 +17,7 @@ "thesaurus": "geonode.metadata.handlers.thesaurus.TKeywordsHandler", "hkeyword": "geonode.metadata.handlers.hkeyword.HKeywordHandler", "region": "geonode.metadata.handlers.region.RegionHandler", + "group": "geonode.metadata.handlers.group.GroupHandler", "doi": "geonode.metadata.handlers.doi.DOIHandler", "linkedresource": "geonode.metadata.handlers.linkedresource.LinkedResourceHandler", "contact": "geonode.metadata.handlers.contact.ContactHandler", From d1835ad5793ecbe0190c7c902eaf3935c02af1fd Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 22 Nov 2024 19:37:44 +0100 Subject: [PATCH 59/91] Metadata: set owner fields as required --- geonode/metadata/handlers/contact.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index d6d07f79d3f..1124733ca0a 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -37,8 +37,12 @@ class ContactHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): contacts = {} + required = [] for role in Roles: - card = ("1" if role.is_required else "0") + ".." + ("N" if role.is_multivalue else "1") + card = f'[{"1" if role.is_required else "0"}..{"N" if role.is_multivalue else "1"}]' + if role.is_required: + required.append(role.name) + if role.is_multivalue: contact = { "type": "array", @@ -57,7 +61,6 @@ def update_schema(self, jsonschema, lang=None): }, }, "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_users")}, - "xxrequired": role.is_required, } else: contact = { @@ -75,7 +78,7 @@ def update_schema(self, jsonschema, lang=None): }, }, "ui:options": {"geonode-ui:autocomplete": reverse("metadata_autocomplete_users")}, - "xxrequired": role.is_required, + "required": ["id"] if role.is_required else [], } contacts[role.name] = contact @@ -84,6 +87,7 @@ def update_schema(self, jsonschema, lang=None): "type": "object", "title": _("Contacts"), "properties": contacts, + "required": required, "geonode:handler": "contact", } @@ -114,7 +118,8 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance logger.info(f"CONTACTS {data}") for rolename, users in data.items(): if rolename == Roles.OWNER.OWNER.name: - logger.debug("Skipping role owner") + resource.owner = get_user_model().objects.get(pk=users["id"]) + # logger.debug("Skipping role owner") continue role = Roles.get_role_by_name(rolename) ids = [u["id"] for u in users] From ec7761f6dec33abe3fa73304dda0c8d66e36224a Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 25 Nov 2024 12:10:13 +0100 Subject: [PATCH 60/91] Metadata: doi: implement update_resource --- geonode/metadata/handlers/doi.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 43ecd8be55a..06f67a8136a 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -45,13 +45,10 @@ def update_schema(self, jsonschema, lang=None): return jsonschema def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - return resource.doi - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): - - pass + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + resource.doi = json_instance[field_name] def load_context(self, resource: ResourceBase, context: dict): - pass From 728a25a8a6a0ace554e85001bdb5a36d8408f7f2 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 28 Nov 2024 12:59:36 +0100 Subject: [PATCH 61/91] Many improvements and fixes Sparse fields, model + handler Fix id type Handling required fields Add load_serialization_context Add null type to most optional fields Caching schema Simplified handler registration --- geonode/metadata/api/urls.py | 11 ++- geonode/metadata/api/views.py | 36 ++++---- geonode/metadata/apps.py | 22 +++-- geonode/metadata/handlers/abstract.py | 19 +++- geonode/metadata/handlers/base.py | 31 +++---- geonode/metadata/handlers/contact.py | 30 ++++--- geonode/metadata/handlers/doi.py | 4 +- geonode/metadata/handlers/group.py | 10 ++- geonode/metadata/handlers/hkeyword.py | 8 +- geonode/metadata/handlers/linkedresource.py | 6 +- geonode/metadata/handlers/region.py | 11 +-- geonode/metadata/handlers/sparse.py | 89 +++++++++++++++++++ geonode/metadata/handlers/thesaurus.py | 25 +++--- .../jsonschema_examples/base_schema.json | 18 ++-- geonode/metadata/manager.py | 72 +++++++++------ geonode/metadata/migrations/0001_initial.py | 29 ++++++ geonode/metadata/models.py | 52 +++++++++++ geonode/metadata/registry.py | 25 ------ geonode/metadata/settings.py | 1 + 19 files changed, 353 insertions(+), 146 deletions(-) create mode 100644 geonode/metadata/handlers/sparse.py create mode 100644 geonode/metadata/migrations/0001_initial.py delete mode 100644 geonode/metadata/registry.py diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index bc1d04bc08b..0b806f4efd4 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -20,15 +20,20 @@ from rest_framework import routers from geonode.metadata.api import views -from geonode.metadata.api.views import ProfileAutocomplete, MetadataLinkedResourcesAutocomplete, \ - MetadataRegionsAutocomplete, MetadataHKeywordAutocomplete, MetadataGroupAutocomplete +from geonode.metadata.api.views import ( + ProfileAutocomplete, + MetadataLinkedResourcesAutocomplete, + MetadataRegionsAutocomplete, + MetadataHKeywordAutocomplete, + MetadataGroupAutocomplete, +) router = routers.DefaultRouter() router.register(r"metadata", views.MetadataViewSet, basename="metadata") urlpatterns = router.urls + [ path( - r"metadata/autocomplete/thesaurus/)/keywords", + r"metadata/autocomplete/thesaurus//keywords", views.tkeywords_autocomplete, name="metadata_autocomplete_tkeywords", ), diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 6edc9e770a5..dea63687d19 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -17,6 +17,7 @@ # ######################################################################### import logging +import json from dal import autocomplete from django.contrib.auth import get_user_model @@ -36,7 +37,6 @@ from geonode.groups.models import GroupProfile from geonode.metadata.manager import metadata_manager from geonode.people.utils import get_available_users -from geonode.security.utils import get_user_visible_groups logger = logging.getLogger(__name__) @@ -85,10 +85,23 @@ def schema_instance(self, request, pk=None): ) elif request.method in ("PUT", "PATCH"): - logger.info(f"handling request {request.method}") - logger.info(f"handling content {request.data}") - update_response = metadata_manager.update_schema_instance(resource, request.data) - return Response(update_response) + logger.debug(f"handling request {request.method}") + # try: + # logger.debug(f"handling content {json.dumps(request.data, indent=3)}") + # except Exception as e: + # logger.warning(f"Can't parse JSON {request.data}: {e}") + errors = metadata_manager.update_schema_instance(resource, request.data) + + response = { + "message": ( + "Some errors were found while updating the resource" + if errors + else "The resource was updated successfully" + ), + "extraErrors": errors, + } + + return Response(response, status=400 if errors else 200) except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} @@ -184,10 +197,7 @@ def get_results(self, context): class MetadataHKeywordAutocomplete(HierarchicalKeywordAutocomplete): def get_results(self, context): - return [ - self.get_result_label(result) - for result in context["object_list"] - ] + return [self.get_result_label(result) for result in context["object_list"]] class MetadataGroupAutocomplete(autocomplete.Select2QuerySetView): @@ -208,10 +218,4 @@ def get_queryset(self): return qs def get_results(self, context): - return [ - {"id": self.get_result_value(result), "label": result.title} - for result in context["object_list"] - ] - - - + return [{"id": self.get_result_value(result), "label": result.title} for result in context["object_list"]] diff --git a/geonode/metadata/apps.py b/geonode/metadata/apps.py index f47f12ad6c8..f234c19a10d 100644 --- a/geonode/metadata/apps.py +++ b/geonode/metadata/apps.py @@ -1,4 +1,9 @@ +import logging + from django.apps import AppConfig +from django.utils.module_loading import import_string + +logger = logging.getLogger(__name__) class MetadataConfig(AppConfig): @@ -12,12 +17,17 @@ def ready(self): def run_setup_hooks(*args, **kwargs): - from geonode.metadata.registry import metadata_registry - from geonode.metadata.manager import metadata_manager + setup_metadata_handlers() - # registry initialization - metadata_registry.init_registry() - handlers = metadata_registry.handler_registry - for handler_id, handler in handlers.items(): +def setup_metadata_handlers(): + from geonode.metadata.manager import metadata_manager + from geonode.metadata.settings import METADATA_HANDLERS + + ids = [] + for handler_id, module_path in METADATA_HANDLERS.items(): + handler = import_string(module_path) metadata_manager.add_handler(handler_id, handler) + ids.append(handler_id) + + logger.info(f"Metadata handlers from config: {', '.join(METADATA_HANDLERS)}") diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 9ccd69b4b57..70f5cab4a1b 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -40,7 +40,7 @@ def update_schema(self, jsonschema: dict, lang=None): pass @abstractmethod - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context: dict, lang: str = None): """ Called when reading metadata, returns the instance of the sub-schema associated with the field field_name. @@ -48,7 +48,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: pass @abstractmethod - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): """ Called when persisting data, updates the field field_name of the resource with the content content, where json_instance is the full JSON Schema instance, @@ -56,8 +56,13 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance """ pass - @abstractmethod - def load_context(self, resource: ResourceBase, context: dict): + def load_serialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): + """ + Called before calls to get_jsonschema_instance in order to initialize info needed by the handler + """ + pass + + def load_deserialization_context(self, resource: ResourceBase, context: dict): """ Called before calls to update_resource in order to initialize info needed by the handler """ @@ -66,9 +71,15 @@ def load_context(self, resource: ResourceBase, context: dict): def _add_after(self, jsonschema, after_what, property_name, subschema): # add thesauri after category ret_properties = {} + added = False for key, val in jsonschema["properties"].items(): ret_properties[key] = val if key == after_what: ret_properties[property_name] = subschema + added = True + + if not added: + logger.warning(f'Could not add "{property_name}" after "{after_what}"') + ret_properties[property_name] = subschema jsonschema["properties"] = ret_properties diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 6dd2f43a5a3..8e802539ec5 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -50,7 +50,7 @@ class CategorySubHandler(SubHandler): def update_subschema(cls, subschema, lang=None): # subschema["title"] = _("topiccategory") subschema["oneOf"] = [ - {"const": tc.identifier, "title": tc.gn_description, "description": tc.description} + {"const": tc.identifier, "title": _(tc.gn_description), "description": _(tc.description)} for tc in TopicCategory.objects.order_by("gn_description") ] @@ -191,36 +191,33 @@ def localize(subschema: dict, annotation_name): # perform further specific initializations if property_name in SUBHANDLERS: - logger.debug(f"Running subhandler for base field {property_name}") + # logger.debug(f"Running subhandler for base field {property_name}") SUBHANDLERS[property_name].update_subschema(subschema, lang) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): - + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): field_value = getattr(resource, field_name) # perform specific transformation if any if field_name in SUBHANDLERS: - logger.debug(f"Serializing base field {field_name}") + # logger.debug(f"Serializing base field {field_name}") field_value = SUBHANDLERS[field_name].serialize(field_value) return field_value - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + field_value = json_instance.get(field_name, None) - if field_name in json_instance: - field_value = json_instance[field_name] - try: - if field_name in SUBHANDLERS: - logger.debug(f"Deserializing base field {field_name}") - # Deserialize field values before setting them to the ResourceBase - field_value = SUBHANDLERS[field_name].deserialize(field_value) + try: + if field_name in SUBHANDLERS: + logger.debug(f"Deserializing base field {field_name}") + # Deserialize field values before setting them to the ResourceBase + field_value = SUBHANDLERS[field_name].deserialize(field_value) - setattr(resource, field_name, field_value) - except Exception as e: - logger.warning(f"Error setting field {field_name}={field_value}: {e}") + setattr(resource, field_name, field_value) + except Exception as e: + logger.warning(f"Error setting field {field_name}={field_value}: {e}") def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 1124733ca0a..5b71c31c9a9 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -88,16 +88,17 @@ def update_schema(self, jsonschema, lang=None): "title": _("Contacts"), "properties": contacts, "required": required, + "geonode:required": bool(required), "geonode:handler": "contact", } return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): def __create_user_entry(user): names = [n for n in (user.first_name, user.last_name) if n] postfix = f" ({' '.join(names)})" if names else "" - return {"id": user.id, "label": f"{user.username}{postfix}"} + return {"id": str(user.id), "label": f"{user.username}{postfix}"} contacts = {} for role in Roles: @@ -113,18 +114,27 @@ def __create_user_entry(user): return contacts - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): data = json_instance[field_name] - logger.info(f"CONTACTS {data}") + logger.debug(f"CONTACTS {data}") for rolename, users in data.items(): if rolename == Roles.OWNER.OWNER.name: - resource.owner = get_user_model().objects.get(pk=users["id"]) + if not users: + logger.warning(f"User not specified for role '{rolename}'") + ( + errors.setdefault("contacts", {}) + .setdefault(rolename, {}) + .setdefault("__errors", []) + .append(f"User not specified for role '{rolename}'") + ) + else: + resource.owner = get_user_model().objects.get(pk=users["id"]) # logger.debug("Skipping role owner") - continue - role = Roles.get_role_by_name(rolename) - ids = [u["id"] for u in users] - profiles = get_user_model().objects.filter(pk__in=ids) - resource.__set_contact_role_element__(profiles, role) + else: + role = Roles.get_role_by_name(rolename) + ids = [u["id"] for u in users] + profiles = get_user_model().objects.filter(pk__in=ids) + resource.__set_contact_role_element__(profiles, role) def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 06f67a8136a..7d45bdee8dc 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -33,7 +33,7 @@ class DOIHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): doi_schema = { - "type": "string", + "type": ["string", "null"], "title": "DOI", "description": _("a DOI will be added by Admin before publication."), "maxLength": 255, @@ -44,7 +44,7 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "edition", "doi", doi_schema) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): return resource.doi def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py index 723176b6a0d..4e7202b1375 100644 --- a/geonode/metadata/handlers/group.py +++ b/geonode/metadata/handlers/group.py @@ -59,10 +59,14 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - return {"id": resource.group.groupprofile.pk, "label": resource.group.groupprofile.title} if resource.group else None + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + return ( + {"id": str(resource.group.groupprofile.pk), "label": resource.group.groupprofile.title} + if resource.group + else None + ) - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): data = json_instance[field_name] id = data.get("id", None) if data else None if id is not None: diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py index f2966c5c7c6..4928a60d708 100644 --- a/geonode/metadata/handlers/hkeyword.py +++ b/geonode/metadata/handlers/hkeyword.py @@ -22,7 +22,7 @@ from rest_framework.reverse import reverse from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase, LinkedResource +from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler from geonode.resource.utils import KeywordHandler @@ -51,14 +51,14 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "tkeywords", "hkeywords", hkeywords) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): return [keyword.name for keyword in resource.keywords.all()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): # TODO: see also resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) hkeywords = json_instance["hkeywords"] cleaned = [k for k in hkeywords if k] - logger.warning(f"HKEYWORDS {hkeywords} --> {cleaned}") + logger.debug(f"hkeywords: {hkeywords} --> {cleaned}") KeywordHandler(resource, cleaned).set_keywords() def load_context(self, resource: ResourceBase, context: dict): diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py index 60f74cc6e2b..d6d5e9af44a 100644 --- a/geonode/metadata/handlers/linkedresource.py +++ b/geonode/metadata/handlers/linkedresource.py @@ -51,10 +51,10 @@ def update_schema(self, jsonschema, lang=None): jsonschema["properties"]["linkedresources"] = linked return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - return [{"id": lr.target.id, "label": lr.target.title} for lr in resource.get_linked_resources()] + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + return [{"id": str(lr.target.id), "label": lr.target.title} for lr in resource.get_linked_resources()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): data = json_instance[field_name] new_ids = {item["id"] for item in data} diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 81c71888244..8a812f10b77 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -45,10 +45,7 @@ def update_schema(self, jsonschema, lang=None): "id": { "type": "string", }, - "label": { - "type": "string", - "title": _("title") - }, + "label": {"type": "string", "title": _("title")}, }, }, "geonode:handler": "region", @@ -59,10 +56,10 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "attribution", "regions", regions) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang=None): - return [{"id": r.id, "label": r.name} for r in resource.regions.all()] + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + return [{"id": str(r.id), "label": r.name} for r in resource.regions.all()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): data = json_instance[field_name] new_ids = {item["id"] for item in data} diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py new file mode 100644 index 00000000000..e7295a6eaf5 --- /dev/null +++ b/geonode/metadata/handlers/sparse.py @@ -0,0 +1,89 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from geonode.base.models import ResourceBase +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.models import SparseField + +logger = logging.getLogger(__name__) + + +class SparseFieldRegistry: + + sparse_fields = {} + + def register(self, field_name: str, schema: dict, after: str = None, init_func=None): + self.sparse_fields[field_name] = {"schema": schema, "after": after, "init_func": init_func} + + def fields(self): + return self.sparse_fields + + +sparse_field_registry = SparseFieldRegistry() + + +class SparseHandler(MetadataHandler): + """ + Handles sparse in fields in the SparseField table + """ + + def update_schema(self, jsonschema, lang=None): + # building the full base schema + + # TODO: manage i18n (thesaurus?) + + for field_name, field_info in sparse_field_registry.fields().items(): + subschema = field_info["schema"] + if after := field_info["after"]: + self._add_after(jsonschema, after, field_name, subschema) + else: + jsonschema["properties"][field_name] = subschema + + # add the handler info to the dictionary if it doesn't exist + if "geonode:handler" not in subschema: + subschema.update({"geonode:handler": "sparse"}) + + # # perform further specific initializations + # if init_func := field_info["init_func"]: + # logger.debug(f"Running init for sparse field {field_name}") + # init_func(field_name, subschema, lang) + + return jsonschema + + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): + # TODO: reading fields one by one may kill performance. We may want the manager to perform a loadcontext as a first call + # before looping on the get_jsonschema_instance calls + field = SparseField.objects.filter(resource=resource, name=field_name).first() + return field.value if field else None + + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + if field_name in json_instance: + field_value = json_instance[field_name] + try: + sf, created = SparseField.objects.update_or_create( + defaults={"value": field_value}, resource=resource, name=field_name + ) + except Exception as e: + logger.warning(f"Error setting field {field_name}={field_value}: {e}") + errors.append(f"{field_name}: error inserting value: {e}") + + def load_context(self, resource: ResourceBase, context: dict): + pass diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 9d6fddd9b4a..848e228f4c6 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -24,7 +24,7 @@ from django.db.models import Q from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.base.models import ResourceBase, Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.metadata.handlers.abstract import MetadataHandler @@ -36,17 +36,14 @@ class TKeywordsHandler(MetadataHandler): """ - The base handler builds a valid empty schema with the simple - fields of the ResourceBase model + Handles the keywords for all the Thesauri with max card > 0 """ - def update_schema(self, jsonschema, lang=None): - - from geonode.base.models import Thesaurus - + @staticmethod + def collect_thesauri(filter, lang=None): # this query return the list of thesaurus X the list of localized titles q = ( - Thesaurus.objects.filter(~Q(card_max=0)) + Thesaurus.objects.filter(filter) .values( "id", "identifier", @@ -68,7 +65,7 @@ def update_schema(self, jsonschema, lang=None): thesaurus = collected_thesauri.get(identifier, {}) if not thesaurus: # init - logger.debug(f"Initializing Thesaurus {identifier} JSON Schema") + logger.debug(f"Initializing Thesaurus {lang}/{identifier} JSON Schema") collected_thesauri[identifier] = thesaurus thesaurus["id"] = r["id"] thesaurus["card"] = {} @@ -83,6 +80,12 @@ def update_schema(self, jsonschema, lang=None): logger.debug(f"Localizing Thesaurus {identifier} JSON Schema for lang {lang}") thesaurus["title"] = r["rel_thesaurus__label"] + return collected_thesauri + + def update_schema(self, jsonschema, lang=None): + + collected_thesauri = self.collect_thesauri(~Q(card_max=0), lang=lang) + # copy info to json schema thesauri = {} for id, ct in collected_thesauri.items(): @@ -131,7 +134,7 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: str = None): + def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): tks = {} for tk in resource.tkeywords.all(): @@ -154,7 +157,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, lang: return ret - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): kids = [] for thes_id, keywords in json_instance.get(TKEYWORDS, {}).items(): diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/jsonschema_examples/base_schema.json index 2a43d446f47..354cc5c4608 100644 --- a/geonode/metadata/jsonschema_examples/base_schema.json +++ b/geonode/metadata/jsonschema_examples/base_schema.json @@ -49,20 +49,20 @@ "default": "eng" }, "license": { - "type": "string", + "type": ["string", "null"], "title": "License", "description": "license of the dataset", "maxLength": 255, "default": "eng" }, "attribution": { - "type": "string", + "type": ["string", "null"], "title": "Attribution", "description": "authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.", "maxLength": 2048 }, "data_quality_statement": { - "type": "string", + "type": ["string", "null"], "title": "data quality statement", "description": "general explanation of the data producer's knowledge about the lineage of a dataset", "maxLength": 2000, @@ -78,7 +78,7 @@ "maxLength": 255 }, "constraints_other": { - "type": "string", + "type": ["string", "null"], "title": "Other constraints", "description": "other restrictions and legal prerequisites for accessing and using the resource or metadata", "ui:options": { @@ -87,13 +87,13 @@ } }, "edition": { - "type": "string", + "type": ["string", "null"], "title": "edition", "description": "version of the cited resource", "maxLength": 255 }, "purpose": { - "type": "string", + "type": ["string", "null"], "title": "purpose", "description": "summary of the intentions with which the resource(s) was developed", "maxLength": 500, @@ -104,7 +104,7 @@ "geonode:handler": "base" }, "supplemental_information": { - "type": "string", + "type": ["string", "null"], "title": "supplemental information", "description": "any other descriptive information about the dataset", "maxLength": 2000, @@ -115,13 +115,13 @@ } }, "temporal_extent_start": { - "type": "string", + "type": ["string", "null"], "format": "date-time", "title": "temporal extent start", "description": "time period covered by the content of the dataset (start)" }, "temporal_extent_end": { - "type": "string", + "type": ["string", "null"], "format": "date-time", "title": "temporal extent end", "description": "time period covered by the content of the dataset (end)" diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 48bc9eb1442..dabec6ba9e3 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -19,7 +19,7 @@ import logging import copy -from abc import ABCMeta +from cachetools import FIFOCache from django.utils.translation import gettext as _ @@ -28,11 +28,7 @@ logger = logging.getLogger(__name__) -class MetadataManagerInterface(metaclass=ABCMeta): - pass - - -class MetadataManager(MetadataManagerInterface): +class MetadataManager: """ The metadata manager is the bridge between the API and the geonode model. The metadata manager will loop over all of the registered metadata handlers, @@ -41,66 +37,90 @@ class MetadataManager(MetadataManagerInterface): to be delivered to the caller. """ + # FIFO bc we want to renew the data once in a while + _schema_cache = FIFOCache(32) + def __init__(self): self.root_schema = MODEL_SCHEMA - self.cached_schema = None self.handlers = {} def add_handler(self, handler_id, handler): self.handlers[handler_id] = handler() def build_schema(self, lang=None): + logger.debug(f"build_schema {lang}") + schema = copy.deepcopy(self.root_schema) schema["title"] = _(schema["title"]) for key, handler in self.handlers.items(): - logger.debug(f"build_schema: update schema -> {key}") + # logger.debug(f"build_schema: update schema -> {key}") schema = handler.update_schema(schema, lang) + # Set required fields. + required = [] + for fieldname, field in schema["properties"].items(): + if field.get("geonode:required", False): + required.append(fieldname) + + if required: + schema["required"] = required return schema def get_schema(self, lang=None): - return self.build_schema(lang) - # we dont want caching for the moment - if not self.cached_schema: - self.cached_schema = self.build_schema(lang) - - return self.cached_schema + cache_key = str(lang) + ret = MetadataManager._schema_cache.get(cache_key, None) + if not ret: + logger.info(f"Building schema for {cache_key}") + ret = self.build_schema(lang) + MetadataManager._schema_cache[cache_key] = ret + logger.info("Schema built") + return ret def build_schema_instance(self, resource, lang=None): + schema = self.get_schema(lang) - schema = self.get_schema() - instance = {} + context = {} + for handler in self.handlers.values(): + handler.load_serialization_context(resource, schema, context) - for fieldname, field in schema["properties"].items(): + instance = {} + for fieldname, subschema in schema["properties"].items(): # logger.debug(f"build_schema_instance: getting handler for property {fieldname}") - handler_id = field.get("geonode:handler", None) + handler_id = subschema.get("geonode:handler", None) if not handler_id: logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") continue handler = self.handlers[handler_id] - content = handler.get_jsonschema_instance(resource, fieldname, lang) + content = handler.get_jsonschema_instance(resource, fieldname, context, lang) instance[fieldname] = content return instance - def update_schema_instance(self, resource, json_instance): + def update_schema_instance(self, resource, json_instance) -> dict: - logger.info(f"RECEIVED INSTANCE {json_instance}") + logger.debug(f"RECEIVED INSTANCE {json_instance}") schema = self.get_schema() + errors = {} for fieldname, subschema in schema["properties"].items(): handler = self.handlers[subschema["geonode:handler"]] - handler.update_resource(resource, fieldname, json_instance) - + # todo: get errors also + handler.update_resource(resource, fieldname, json_instance, errors) try: resource.save() - return {"message": "The resource was updated successfully"} - except Exception as e: logger.warning(f"Error while updating schema instance: {e}") - return {"message": "Something went wrong... The resource was not updated"} + errors.setdefault("__errors", []).append(f"Error while saving the resource: {e}") + + if "error" in resource.title.lower(): + errors.setdefault("title", {}).setdefault("__errors", []).append("this is a test error under /title") + errors.setdefault("properties", {}).setdefault("title", {}).setdefault("__errors", []).append( + "this is a test error under /properties/title" + ) + + return errors metadata_manager = MetadataManager() diff --git a/geonode/metadata/migrations/0001_initial.py b/geonode/metadata/migrations/0001_initial.py new file mode 100644 index 00000000000..498df8f6aab --- /dev/null +++ b/geonode/metadata/migrations/0001_initial.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.9 on 2024-11-25 10:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("base", "0092_migrate and_remove_resourcebase_files"), + ] + + operations = [ + migrations.CreateModel( + name="SparseField", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=64)), + ("value", models.CharField(blank=True, max_length=1024, null=True)), + ("resource", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="base.resourcebase")), + ], + options={ + "ordering": ("resource", "name"), + "unique_together": {("resource", "name")}, + }, + ), + ] diff --git a/geonode/metadata/models.py b/geonode/metadata/models.py index e69de29bb2d..6e9e2665c43 100644 --- a/geonode/metadata/models.py +++ b/geonode/metadata/models.py @@ -0,0 +1,52 @@ +import logging + +from django.db import models + +from geonode.base.models import ResourceBase + + +logger = logging.getLogger(__name__) + +# class SparseFieldDecl(models.Model): +# class Type(enum.Enum): +# STRING = 'string' +# INTEGER = 'integer' +# FLOAT = 'float' +# BOOL = 'bool' +# +# FIELD_TYPES = [] +# name = models.CharField(max_length=64, null=False, blank=False, unique=True, primary_key=True) +# +# type = models.CharField(choices=[(x.value,x.name) for x in Type], max_length=32, null=False, blank=False, unique=True, ) +# nullable = models.BooleanField(default=True, null=False) +# +# eager = models.BooleanField(default=True, null=False) + + +class SparseField(models.Model): + """ + Sparse field related to a ResourceBase + """ + + resource = models.ForeignKey(ResourceBase, on_delete=models.CASCADE, null=False) + # name = models.ForeignKey(SparseFieldDecl, on_delete=models.PROTECT, null=False) + name = models.CharField(max_length=64, null=False, blank=False) + value = models.CharField(max_length=1024, null=True, blank=True) + + def __str__(self): + return f"{self.name}={self.value}" + + @staticmethod + def get_fields(cls, resource: ResourceBase, names=None): + qs = SparseField.objects.filter(resource=resource) + if names: + qs = qs.filter(name__in=names) + + return qs + + class Meta: + ordering = ( + "resource", + "name", + ) + unique_together = (("resource", "name"),) diff --git a/geonode/metadata/registry.py b/geonode/metadata/registry.py deleted file mode 100644 index 6f6f6de3d4e..00000000000 --- a/geonode/metadata/registry.py +++ /dev/null @@ -1,25 +0,0 @@ -from django.utils.module_loading import import_string -from geonode.metadata.settings import METADATA_HANDLERS -import logging - -logger = logging.getLogger(__name__) - - -class MetadataHandlersRegistry: - - handler_registry = {} - - def init_registry(self): - self.register() - logger.info(f"The following metadata handlers have been registered: {', '.join(METADATA_HANDLERS)}") - - def register(self): - for handler_id, module_path in METADATA_HANDLERS.items(): - self.handler_registry[handler_id] = import_string(module_path) - - @classmethod - def get_registry(cls): - return MetadataHandlersRegistry.handler_registry - - -metadata_registry = MetadataHandlersRegistry() diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index 4bd311dff60..bb1d8ca985b 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -21,4 +21,5 @@ "doi": "geonode.metadata.handlers.doi.DOIHandler", "linkedresource": "geonode.metadata.handlers.linkedresource.LinkedResourceHandler", "contact": "geonode.metadata.handlers.contact.ContactHandler", + "sparse": "geonode.metadata.handlers.sparse.SparseHandler", } From 857103f060c35268b9452bada0a2406ede319235 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 28 Nov 2024 19:08:29 +0100 Subject: [PATCH 62/91] Cleanup --- geonode/metadata/handlers/abstract.py | 8 ++++++++ geonode/metadata/handlers/base.py | 3 --- geonode/metadata/handlers/contact.py | 14 ++------------ geonode/metadata/handlers/doi.py | 4 ---- geonode/metadata/handlers/group.py | 3 --- geonode/metadata/handlers/hkeyword.py | 3 --- geonode/metadata/handlers/linkedresource.py | 4 ---- geonode/metadata/handlers/region.py | 6 ------ geonode/metadata/handlers/sparse.py | 20 ++++++++------------ geonode/metadata/handlers/thesaurus.py | 4 ---- geonode/metadata/manager.py | 18 ++++++++++++++++-- 11 files changed, 34 insertions(+), 53 deletions(-) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 70f5cab4a1b..a76dad718b9 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -83,3 +83,11 @@ def _add_after(self, jsonschema, after_what, property_name, subschema): ret_properties[property_name] = subschema jsonschema["properties"] = ret_properties + + @staticmethod + def _set_error(errors: dict, path: list, msg: str): + elem = errors + for step in path: + elem = elem.setdefault(step, {}) + elem = elem.setdefault("__errors", []) + elem.append(msg) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 8e802539ec5..e7db9cfbcf5 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -218,6 +218,3 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance setattr(resource, field_name, field_value) except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") - - def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 5b71c31c9a9..9850db51d1e 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -114,28 +114,18 @@ def __create_user_entry(user): return contacts - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: dict, **kwargs): data = json_instance[field_name] logger.debug(f"CONTACTS {data}") for rolename, users in data.items(): if rolename == Roles.OWNER.OWNER.name: if not users: logger.warning(f"User not specified for role '{rolename}'") - ( - errors.setdefault("contacts", {}) - .setdefault(rolename, {}) - .setdefault("__errors", []) - .append(f"User not specified for role '{rolename}'") - ) + self._set_error(errors, ["contacts", rolename], f"User not specified for role '{rolename}'") else: resource.owner = get_user_model().objects.get(pk=users["id"]) - # logger.debug("Skipping role owner") else: role = Roles.get_role_by_name(rolename) ids = [u["id"] for u in users] profiles = get_user_model().objects.filter(pk__in=ids) resource.__set_contact_role_element__(profiles, role) - - def load_context(self, resource: ResourceBase, context: dict): - - pass diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 7d45bdee8dc..ce96512b856 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -31,7 +31,6 @@ class DOIHandler(MetadataHandler): def update_schema(self, jsonschema, lang=None): - doi_schema = { "type": ["string", "null"], "title": "DOI", @@ -49,6 +48,3 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): resource.doi = json_instance[field_name] - - def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py index 4e7202b1375..4379748afad 100644 --- a/geonode/metadata/handlers/group.py +++ b/geonode/metadata/handlers/group.py @@ -74,6 +74,3 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance resource.group = gp.group else: resource.group = None - - def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py index 4928a60d708..776355b34ea 100644 --- a/geonode/metadata/handlers/hkeyword.py +++ b/geonode/metadata/handlers/hkeyword.py @@ -60,6 +60,3 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance cleaned = [k for k in hkeywords if k] logger.debug(f"hkeywords: {hkeywords} --> {cleaned}") KeywordHandler(resource, cleaned).set_keywords() - - def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py index d6d5e9af44a..16185f9357d 100644 --- a/geonode/metadata/handlers/linkedresource.py +++ b/geonode/metadata/handlers/linkedresource.py @@ -55,7 +55,6 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte return [{"id": str(lr.target.id), "label": lr.target.title} for lr in resource.get_linked_resources()] def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): - data = json_instance[field_name] new_ids = {item["id"] for item in data} @@ -66,6 +65,3 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance # delete remaining links LinkedResource.objects.filter(source_id=resource.id, internal=False).exclude(target_id__in=new_ids).delete() - - def load_context(self, resource: ResourceBase, context: dict): - pass diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 8a812f10b77..03a62e6bd9d 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -34,7 +34,6 @@ class RegionHandler(MetadataHandler): """ def update_schema(self, jsonschema, lang=None): - regions = { "type": "array", "title": _("Regions"), @@ -60,14 +59,9 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte return [{"id": str(r.id), "label": r.name} for r in resource.regions.all()] def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): - data = json_instance[field_name] new_ids = {item["id"] for item in data} logger.info(f"Regions added {data} --> {new_ids}") regions = Region.objects.filter(id__in=new_ids) resource.regions.set(regions) - - def load_context(self, resource: ResourceBase, context: dict): - - pass diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index e7295a6eaf5..c2b0b0865f3 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -75,15 +75,11 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte return field.value if field else None def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): - if field_name in json_instance: - field_value = json_instance[field_name] - try: - sf, created = SparseField.objects.update_or_create( - defaults={"value": field_value}, resource=resource, name=field_name - ) - except Exception as e: - logger.warning(f"Error setting field {field_name}={field_value}: {e}") - errors.append(f"{field_name}: error inserting value: {e}") - - def load_context(self, resource: ResourceBase, context: dict): - pass + field_value = json_instance.get(field_name, None) + try: + sf, created = SparseField.objects.update_or_create( + defaults={"value": field_value}, resource=resource, name=field_name + ) + except Exception as e: + logger.warning(f"Error setting field {field_name}={field_value}: {e}") + self._set_error(errors, ["field_name"], f"Error setting value: {e}") diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 848e228f4c6..36509d11651 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -167,7 +167,3 @@ def update_resource(self, resource: ResourceBase, field_name: str, json_instance kw_requested = ThesaurusKeyword.objects.filter(about__in=kids) resource.tkeywords.set(kw_requested) - - def load_context(self, resource: ResourceBase, context: dict): - - pass diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index dabec6ba9e3..1deae3d2143 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -23,6 +23,7 @@ from django.utils.translation import gettext as _ +from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import MODEL_SCHEMA logger = logging.getLogger(__name__) @@ -95,6 +96,13 @@ def build_schema_instance(self, resource, lang=None): content = handler.get_jsonschema_instance(resource, fieldname, context, lang) instance[fieldname] = content + # TESTING ONLY + if "error" in resource.title.lower(): + errors = {} + MetadataHandler._set_error(errors, ["title"], "GET: test msg under /title") + MetadataHandler._set_error(errors, ["properties", "title"], "GET: test msg under /properties/title") + instance["extraErrors"] = errors + return instance def update_schema_instance(self, resource, json_instance) -> dict: @@ -114,10 +122,16 @@ def update_schema_instance(self, resource, json_instance) -> dict: logger.warning(f"Error while updating schema instance: {e}") errors.setdefault("__errors", []).append(f"Error while saving the resource: {e}") + # TESTING ONLY if "error" in resource.title.lower(): - errors.setdefault("title", {}).setdefault("__errors", []).append("this is a test error under /title") + errors.setdefault("title", {}).setdefault("__errors", []).append("PUT: this is a test error under /title") errors.setdefault("properties", {}).setdefault("title", {}).setdefault("__errors", []).append( - "this is a test error under /properties/title" + "PUT: this is a test error under /properties/title" + ) + + MetadataHandler._set_error(errors, ["title"], "PUT: this is another test msg under /title") + MetadataHandler._set_error( + errors, ["properties", "title"], "PUT: this is another test msg under /properties/title" ) return errors From dc9dd8c64216258750319e4d7d36be44292d7df8 Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 29 Nov 2024 12:21:33 +0100 Subject: [PATCH 63/91] Add error handling, Improve sparse field loading --- geonode/metadata/api/views.py | 1 - geonode/metadata/handlers/abstract.py | 11 ++++++++--- geonode/metadata/handlers/base.py | 6 +++--- geonode/metadata/handlers/contact.py | 7 +++---- geonode/metadata/handlers/doi.py | 6 ++---- geonode/metadata/handlers/group.py | 7 +++---- geonode/metadata/handlers/hkeyword.py | 7 +++---- geonode/metadata/handlers/linkedresource.py | 6 +++--- geonode/metadata/handlers/region.py | 8 ++++---- geonode/metadata/handlers/sparse.py | 18 +++++++++++------- geonode/metadata/handlers/thesaurus.py | 8 +++----- geonode/metadata/manager.py | 18 +++++++++++------- geonode/metadata/models.py | 4 ++-- 13 files changed, 56 insertions(+), 51 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index dea63687d19..47e4645662a 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -17,7 +17,6 @@ # ######################################################################### import logging -import json from dal import autocomplete from django.contrib.auth import get_user_model diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index a76dad718b9..6ceecd24e2c 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -40,7 +40,9 @@ def update_schema(self, jsonschema: dict, lang=None): pass @abstractmethod - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context: dict, lang: str = None): + def get_jsonschema_instance( + self, resource: ResourceBase, field_name: str, context: dict, errors: dict, lang: str = None + ): """ Called when reading metadata, returns the instance of the sub-schema associated with the field field_name. @@ -48,7 +50,9 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte pass @abstractmethod - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource( + self, resource: ResourceBase, field_name: str, json_instance: dict, context: dict, errors: dict, **kwargs + ): """ Called when persisting data, updates the field field_name of the resource with the content content, where json_instance is the full JSON Schema instance, @@ -62,7 +66,7 @@ def load_serialization_context(self, resource: ResourceBase, jsonschema: dict, c """ pass - def load_deserialization_context(self, resource: ResourceBase, context: dict): + def load_deserialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): """ Called before calls to update_resource in order to initialize info needed by the handler """ @@ -86,6 +90,7 @@ def _add_after(self, jsonschema, after_what, property_name, subschema): @staticmethod def _set_error(errors: dict, path: list, msg: str): + logger.warning(f"Reported message: {'/'.join(path)}: {msg} ") elem = errors for step in path: elem = elem.setdefault(step, {}) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index e7db9cfbcf5..d2e18fe79f2 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -21,7 +21,7 @@ import logging from datetime import datetime -from geonode.base.models import ResourceBase, TopicCategory, License, RestrictionCodeType, SpatialRepresentationType +from geonode.base.models import TopicCategory, License, RestrictionCodeType, SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES @@ -196,7 +196,7 @@ def localize(subschema: dict, annotation_name): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): field_value = getattr(resource, field_name) # perform specific transformation if any @@ -206,7 +206,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte return field_value - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): field_value = json_instance.get(field_name, None) try: diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 9850db51d1e..e9b00e7e191 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -18,12 +18,11 @@ ######################################################################### import logging +from rest_framework.reverse import reverse from django.contrib.auth import get_user_model -from rest_framework.reverse import reverse from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler from geonode.people import Roles @@ -94,7 +93,7 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): def __create_user_entry(user): names = [n for n in (user.first_name, user.last_name) if n] postfix = f" ({' '.join(names)})" if names else "" @@ -114,7 +113,7 @@ def __create_user_entry(user): return contacts - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: dict, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): data = json_instance[field_name] logger.debug(f"CONTACTS {data}") for rolename, users in data.items(): diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index ce96512b856..74183d5857c 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -21,10 +21,8 @@ from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler - logger = logging.getLogger(__name__) @@ -43,8 +41,8 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "edition", "doi", doi_schema) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): return resource.doi - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): resource.doi = json_instance[field_name] diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py index 4379748afad..a78775bc80e 100644 --- a/geonode/metadata/handlers/group.py +++ b/geonode/metadata/handlers/group.py @@ -18,11 +18,10 @@ ######################################################################### import logging - from rest_framework.reverse import reverse + from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase from geonode.groups.models import GroupProfile from geonode.metadata.handlers.abstract import MetadataHandler @@ -59,14 +58,14 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): return ( {"id": str(resource.group.groupprofile.pk), "label": resource.group.groupprofile.title} if resource.group else None ) - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): data = json_instance[field_name] id = data.get("id", None) if data else None if id is not None: diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py index 776355b34ea..7ca0fcdb73e 100644 --- a/geonode/metadata/handlers/hkeyword.py +++ b/geonode/metadata/handlers/hkeyword.py @@ -18,11 +18,10 @@ ######################################################################### import logging - from rest_framework.reverse import reverse + from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler from geonode.resource.utils import KeywordHandler @@ -51,10 +50,10 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "tkeywords", "hkeywords", hkeywords) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): return [keyword.name for keyword in resource.keywords.all()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): # TODO: see also resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) hkeywords = json_instance["hkeywords"] cleaned = [k for k in hkeywords if k] diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py index 16185f9357d..77520f568b7 100644 --- a/geonode/metadata/handlers/linkedresource.py +++ b/geonode/metadata/handlers/linkedresource.py @@ -18,8 +18,8 @@ ######################################################################### import logging - from rest_framework.reverse import reverse + from django.utils.translation import gettext as _ from geonode.base.models import ResourceBase, LinkedResource @@ -51,10 +51,10 @@ def update_schema(self, jsonschema, lang=None): jsonschema["properties"]["linkedresources"] = linked return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): return [{"id": str(lr.target.id), "label": lr.target.title} for lr in resource.get_linked_resources()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): data = json_instance[field_name] new_ids = {item["id"] for item in data} diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index 03a62e6bd9d..c33cac429ca 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -18,11 +18,11 @@ ######################################################################### import logging - from rest_framework.reverse import reverse + from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase, Region +from geonode.base.models import Region from geonode.metadata.handlers.abstract import MetadataHandler logger = logging.getLogger(__name__) @@ -55,10 +55,10 @@ def update_schema(self, jsonschema, lang=None): self._add_after(jsonschema, "attribution", "regions", regions) return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang=None): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): return [{"id": str(r.id), "label": r.name} for r in resource.regions.all()] - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): data = json_instance[field_name] new_ids = {item["id"] for item in data} logger.info(f"Regions added {data} --> {new_ids}") diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index c2b0b0865f3..018264634ca 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -19,7 +19,6 @@ import logging -from geonode.base.models import ResourceBase from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.models import SparseField @@ -68,13 +67,18 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): - # TODO: reading fields one by one may kill performance. We may want the manager to perform a loadcontext as a first call - # before looping on the get_jsonschema_instance calls - field = SparseField.objects.filter(resource=resource, name=field_name).first() - return field.value if field else None + def load_serialization_context(self, resource, jsonschema: dict, context: dict): + logger.debug(f"Preloading sparse fields {sparse_field_registry.fields().keys()}") + context["sparse"] = { + "fields": { + f.name: f.value for f in SparseField.get_fields(resource, names=sparse_field_registry.fields().keys()) + } + } - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): + return context["sparse"]["fields"].get(field_name, None) + + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): field_value = json_instance.get(field_name, None) try: sf, created = SparseField.objects.update_or_create( diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 36509d11651..7d83fc7178a 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -24,7 +24,7 @@ from django.db.models import Q from django.utils.translation import gettext as _ -from geonode.base.models import ResourceBase, Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.base.models import Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.metadata.handlers.abstract import MetadataHandler @@ -134,8 +134,7 @@ def update_schema(self, jsonschema, lang=None): return jsonschema - def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, context, lang: str = None): - + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): tks = {} for tk in resource.tkeywords.all(): tks[tk.id] = tk @@ -157,8 +156,7 @@ def get_jsonschema_instance(self, resource: ResourceBase, field_name: str, conte return ret - def update_resource(self, resource: ResourceBase, field_name: str, json_instance: dict, errors: list, **kwargs): - + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): kids = [] for thes_id, keywords in json_instance.get(TKEYWORDS, {}).items(): logger.info(f"Getting info for thesaurus {thes_id}") diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 1deae3d2143..67fc4719d7b 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -86,6 +86,7 @@ def build_schema_instance(self, resource, lang=None): handler.load_serialization_context(resource, schema, context) instance = {} + errors = {} for fieldname, subschema in schema["properties"].items(): # logger.debug(f"build_schema_instance: getting handler for property {fieldname}") handler_id = subschema.get("geonode:handler", None) @@ -93,12 +94,11 @@ def build_schema_instance(self, resource, lang=None): logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") continue handler = self.handlers[handler_id] - content = handler.get_jsonschema_instance(resource, fieldname, context, lang) + content = handler.get_jsonschema_instance(resource, fieldname, context, errors, lang) instance[fieldname] = content # TESTING ONLY if "error" in resource.title.lower(): - errors = {} MetadataHandler._set_error(errors, ["title"], "GET: test msg under /title") MetadataHandler._set_error(errors, ["properties", "title"], "GET: test msg under /properties/title") instance["extraErrors"] = errors @@ -106,21 +106,25 @@ def build_schema_instance(self, resource, lang=None): return instance def update_schema_instance(self, resource, json_instance) -> dict: - logger.debug(f"RECEIVED INSTANCE {json_instance}") - schema = self.get_schema() + context = {} + for handler in self.handlers.values(): + handler.load_deserialization_context(resource, schema, context) + errors = {} for fieldname, subschema in schema["properties"].items(): handler = self.handlers[subschema["geonode:handler"]] - # todo: get errors also - handler.update_resource(resource, fieldname, json_instance, errors) + try: + handler.update_resource(resource, fieldname, json_instance, context, errors) + except Exception as e: + MetadataHandler._set_error(errors, [fieldname], f"Error while processing this field: {e}") try: resource.save() except Exception as e: logger.warning(f"Error while updating schema instance: {e}") - errors.setdefault("__errors", []).append(f"Error while saving the resource: {e}") + MetadataHandler._set_error(errors, [], f"Error while saving the resource: {e}") # TESTING ONLY if "error" in resource.title.lower(): diff --git a/geonode/metadata/models.py b/geonode/metadata/models.py index 6e9e2665c43..111f76ffcb8 100644 --- a/geonode/metadata/models.py +++ b/geonode/metadata/models.py @@ -37,12 +37,12 @@ def __str__(self): return f"{self.name}={self.value}" @staticmethod - def get_fields(cls, resource: ResourceBase, names=None): + def get_fields(resource: ResourceBase, names=None): qs = SparseField.objects.filter(resource=resource) if names: qs = qs.filter(name__in=names) - return qs + return qs.all() class Meta: ordering = ( From d960746c2b03823dcdcc9e5bf33438349b80936c Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 29 Nov 2024 12:25:09 +0100 Subject: [PATCH 64/91] Initial INSPIRE app --- geonode/base/admin.py | 2 +- geonode/inspire/__init__.py | 0 geonode/inspire/apps.py | 32 ++++ geonode/inspire/inspire.py | 304 ++++++++++++++++++++++++++++++++++++ geonode/settings.py | 5 + 5 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 geonode/inspire/__init__.py create mode 100644 geonode/inspire/apps.py create mode 100644 geonode/inspire/inspire.py diff --git a/geonode/base/admin.py b/geonode/base/admin.py index 0f6c4ee859c..5d8ace0529e 100755 --- a/geonode/base/admin.py +++ b/geonode/base/admin.py @@ -247,7 +247,7 @@ def import_rdf(self, request): if request.method == "POST": try: rdf_file = request.FILES["rdf_file"] - name = slugify(rdf_file.name) + name = slugify(rdf_file.name).removesuffix("-rdf") call_command("load_thesaurus", file=rdf_file, name=name) self.message_user(request, "Your RDF file has been imported", messages.SUCCESS) return redirect("..") diff --git a/geonode/inspire/__init__.py b/geonode/inspire/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/inspire/apps.py b/geonode/inspire/apps.py new file mode 100644 index 00000000000..f32718878cb --- /dev/null +++ b/geonode/inspire/apps.py @@ -0,0 +1,32 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + +from geonode.notifications_helper import NotificationsAppConfigBase + + +class BaseAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.inspire" + + def ready(self): + super().ready() + + from geonode.inspire.inspire import init + + init() diff --git a/geonode/inspire/inspire.py b/geonode/inspire/inspire.py new file mode 100644 index 00000000000..4f1af2836db --- /dev/null +++ b/geonode/inspire/inspire.py @@ -0,0 +1,304 @@ +import logging +from rest_framework.reverse import reverse + +from django.db.models import Q +from django.utils.translation import gettext as _ + +from geonode.base.models import ResourceBase, RestrictionCodeType, ThesaurusKeywordLabel +from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.handlers.sparse import sparse_field_registry +from geonode.metadata.manager import metadata_manager +from geonode.metadata.handlers.thesaurus import TKeywordsHandler +from geonode.metadata.models import SparseField + +logger = logging.getLogger(__name__) + + +FIELD_RESTRICTION_TYPE = "restriction_code_type" +FIELD_RESTRICTION_PUBLIC_ACCESS = "constraints_other" +FIELD_RESTRICTION_ACCESS_USE = "access_and_use" + +THESAURUS_PUBLIC_ACCESS = "limitationsonpublicaccess" +THESAURUS_ACCESS_USE = "conditionsapplyingtoaccessanduse" + +FIELDVAL_ACCESS_USE_FREETEXT = "http://inspire.ec.europa.eu/metadata-codelist/ConditionsApplyingToAccessAndUse/freeText" +# Should be "otherRestrictions", see https://github.com/GeoNode/geonode/issues/12745 +FIELDVAL_RESTRICTION_TYPE_DEFAULT = "limitation not listed" + + +class INSPIREHandler(MetadataHandler): + """ + The INSPIRE Handler adds the Regions model options to the schema + """ + + def update_schema(self, jsonschema, lang=None): + # Schema overriding for INSPIRE + # Some Additional Sparse Fields are registered in geonode.inspire.inspire.init() + + # lineage is mandatory + for prop in ( + "language", # 2.2.2 TG Rq C.5 + "data_quality_statement", + ): + jsonschema["properties"][prop].update( + { + "type": "string", # exclude null + "geonode:required": True, + } + ) + + # override base schema: was a codelist, is a fixed value in the codelist + jsonschema["properties"][FIELD_RESTRICTION_TYPE] = { + "type": "string", + "title": "restrictions", + "ui:widget": "hidden", + "geonode:handler": "inspire", + } + + collected_thesauri = TKeywordsHandler.collect_thesauri( + Q(identifier__in=(THESAURUS_ACCESS_USE, THESAURUS_PUBLIC_ACCESS)), lang=lang + ) + if THESAURUS_PUBLIC_ACCESS in collected_thesauri: + ct = collected_thesauri[THESAURUS_PUBLIC_ACCESS] + # override base schema: was free text, is an entry from a thesaurus + jsonschema["properties"][FIELD_RESTRICTION_PUBLIC_ACCESS] = { + "type": "object", + "title": ct["title"], + "description": ct["description"], + "properties": { + "id": { + "type": "string", + "title": "keyword id", + }, + "label": { + "type": "string", + "title": "Label", + }, + }, + "ui:options": { + "geonode-ui:autocomplete": reverse( + "metadata_autocomplete_tkeywords", kwargs={"thesaurusid": ct["id"]} + ) + }, + "geonode:handler": "inspire", + } + else: + logger.warning(f"Missing thesaurus {THESAURUS_PUBLIC_ACCESS}") + jsonschema["properties"][FIELD_RESTRICTION_PUBLIC_ACCESS] = { + "type": "string", + "title": _("Limitations on public access"), + "readOnly": True, + "geonode:handler": "inspire", + } + + # As per §2.3.7 the conditions to access and use may be either: + # - a URI to indicate "noConditionsApply" + # - a URI to indicate "conditionsUnknown" + # - a free text with a textual description of the condition + # The standard thesaurus include only the first two options. + # This implementation foresee: + # - a dropdown to choose one of the 2 official entries and a custom added entry "http://inspire.ec.europa.eu/metadata-codelist/ConditionsApplyingToAccessAndUse/freeText" + # - a textarea for the free text + # The textarea can be populated only if the freetext entry is selected, or there will be an error returned + # by the server + # Future implementations may refine the jsonschema to enable the textarea only when the freetext entry is selected + + if THESAURUS_ACCESS_USE in collected_thesauri: + ct = collected_thesauri[THESAURUS_ACCESS_USE] + access_use_subschema = { + "type": "object", + "title": ct["title"], + "description": ct["description"], + "properties": { + "choice": { + "type": "object", + "properties": { + "id": { + "type": "string", + "title": "keyword id", + }, + "label": { + "type": "string", + "title": "Label", + }, + }, + "ui:options": { + "label": False, + "geonode-ui:autocomplete": reverse( + "metadata_autocomplete_tkeywords", kwargs={"thesaurusid": ct["id"]} + ), + }, + }, + "freetext": { + "title": "Conditions description", + "description": "Description of terms and conditions for access and use", + "type": ["string", "null"], + "ui:options": { + "widget": "textarea", + "rows": 3, + # "label": False, + }, + }, + }, + "geonode:handler": "inspire", + } + + else: + logger.warning(f"Missing thesaurus {THESAURUS_ACCESS_USE}") + access_use_subschema = { + "type": "string", + "title": _("Conditions applying to access and use"), + "readOnly": True, + "geonode:handler": "inspire", + } + + self._add_after(jsonschema, FIELD_RESTRICTION_PUBLIC_ACCESS, FIELD_RESTRICTION_ACCESS_USE, access_use_subschema) + + return jsonschema + + def load_serialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): + context["inspire"] = {"schema": jsonschema} + fields = {} + for field in SparseField.get_fields(resource, names=(FIELD_RESTRICTION_ACCESS_USE,)): + fields[field.name] = field.value + context["inspire"]["fields"] = fields + + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): + if field_name == FIELD_RESTRICTION_TYPE: + # 1) as per TG Req C.17, fixed to "otherRestrictions" + # 2) should be "otherRestrictions", see https://github.com/GeoNode/geonode/issues/12745 + return FIELDVAL_RESTRICTION_TYPE_DEFAULT + + elif field_name == FIELD_RESTRICTION_PUBLIC_ACCESS: + if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_PUBLIC_ACCESS].get("readOnly", False): + self._set_error( + errors, + [FIELD_RESTRICTION_PUBLIC_ACCESS], + f"Missing thesaurus {THESAURUS_PUBLIC_ACCESS}. Please contact your administrator.", + ) + return ( + f"Missing Thesaurus {THESAURUS_PUBLIC_ACCESS}.\n" + "Get it from https://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess" + ) + else: + return {"id": resource.constraints_other} + + elif field_name == FIELD_RESTRICTION_ACCESS_USE: + if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): + self._set_error( + errors, + [FIELD_RESTRICTION_ACCESS_USE], + f"Missing thesaurus {THESAURUS_ACCESS_USE}. Please contact your administrator.", + ) + return ( + f"Missing Thesaurus {THESAURUS_ACCESS_USE}.\n" + "Get it from https://inspire.ec.europa.eu/metadata-codelist/ConditionsApplyingToAccessAndUse" + ) + else: + val = context["inspire"]["fields"].get(FIELD_RESTRICTION_ACCESS_USE, None) + if not val: + return {"choice": {"id": None}, "freetext": None} + elif val.startswith("http"): + label = ( + ThesaurusKeywordLabel.objects.filter(keyword__about=val, lang=lang) + .values_list("label", flat=True) + .first() + ) + label = label or val.split("/")[-1] + return {"choice": {"id": val, "label": label}, "freetext": None} + else: + return {"choice": {"id": FIELDVAL_ACCESS_USE_FREETEXT}, "freetext": val} + + else: + raise Exception(f"The INSPIRE handler does not support field {field_name}") + + def load_deserialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): + context["inspire"] = {"schema": jsonschema} + + def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): + if field_name == FIELD_RESTRICTION_TYPE: + # 1) as per TG Req C.17, fixed to "otherRestrictions" + resource.restriction_code_type = RestrictionCodeType.objects.filter( + identifier=FIELDVAL_RESTRICTION_TYPE_DEFAULT + ).first() + if not resource.restriction_code_type: + logger.warning(f"Default value '{FIELDVAL_RESTRICTION_TYPE_DEFAULT}' not found.") + self._set_error( + errors, + [FIELD_RESTRICTION_TYPE], + f"Default value '{FIELDVAL_RESTRICTION_TYPE_DEFAULT}' not found. Please contact your administrator.", + ) + + elif field_name == FIELD_RESTRICTION_PUBLIC_ACCESS: + field_value = json_instance.get(field_name, None) + # This field contains a URI from a controlled vocabulary + # TODO: validate against allowed values + resource.constraints_other = field_value + + elif field_name == FIELD_RESTRICTION_ACCESS_USE: + # This field contains either a URI from a controlled voc or a free text. + # The schema contains a dropdown property (id+label) where the id is a URI from a controlled voc, + # and a string object for the freetext. + # We want the free text only if the choice contain the freetext related URI + + field_value = json_instance.get(field_name, {}) + + if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): + content = "N/A" + self._set_error( + errors, + [FIELD_RESTRICTION_ACCESS_USE], + f"Missing thesaurus {THESAURUS_ACCESS_USE}. Please contact your administrator.", + ) + else: + logger.debug(f"ACCESS AND USE --> {field_value}") + uri = field_value["choice"]["id"] + freetext = field_value.get("freetext", None) + content = None + if uri != FIELDVAL_ACCESS_USE_FREETEXT: + if freetext: + # save the text anyway, it may be hard to type it again + content = freetext + self._set_error( + errors, + [FIELD_RESTRICTION_ACCESS_USE], + "Textual content only allowed when freetext option is selected", + ) + else: + content = uri + else: + content = freetext + if not freetext: + self._set_error(errors, [FIELD_RESTRICTION_ACCESS_USE], "Textual content can't be empty") + try: + SparseField.objects.update_or_create(defaults={"value": content}, resource=resource, name=field_name) + except Exception as e: + logger.warning(f"Error setting field {field_name}={field_value}: {e}") + self._set_error(errors, ["field_name"], f"Error setting value: {e}") + else: + raise Exception(f"The INSPIRE handler does not support field {field_name}") + + +def init(): + logger.info("Initting INSPIRE hooks") + # == Add json schema + + # TG Requirement 1.5: metadata/2.0/req/datasets-and-series/spatial-resolution + schema_res = { + "type": "number", + "title": "res_distance", + "description": "Level of detail of the data set in metres", + } + sparse_field_registry.register("accuracy", schema_res, after="spatial_representation_type") + + metadata_manager.add_handler("inspire", INSPIREHandler) + + # TODO: register metadata parser + + # TODO: register metadata storer + + # TODO: set metadata template + + # TODO: check for mandatory thesauri + + # TODO: reload schema on thesaurus+thesauruskeywords signal diff --git a/geonode/settings.py b/geonode/settings.py index b79cc98f29e..58eb27889c0 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2365,3 +2365,8 @@ def get_geonode_catalogue_service(): ] INSTALLED_APPS += ("geonode.assets",) GEONODE_APPS += ("geonode.assets",) + +INSPIRE_ENABLE = ast.literal_eval(os.getenv("INSPIRE_ENABLE", "False")) +if INSPIRE_ENABLE: + INSTALLED_APPS += ("geonode.inspire",) + GEONODE_APPS += ("geonode.inspire",) \ No newline at end of file From 422cae87df0b8f521c2704c7bbf53577c7140d33 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 10 Dec 2024 11:01:20 +0100 Subject: [PATCH 65/91] May improvements: sparse fields, i18n,... - Handling complex values in sparse fields - Added i18n via thesaurus - Improved subschema handling - Renamed base schema json file --- geonode/base/api/views.py | 20 +++++-- geonode/inspire/apps.py | 2 +- geonode/inspire/{inspire.py => metadata.py} | 18 +++--- geonode/metadata/api/views.py | 11 +--- geonode/metadata/handlers/abstract.py | 43 +++++++++++++- geonode/metadata/handlers/base.py | 9 +-- geonode/metadata/handlers/contact.py | 12 ++-- geonode/metadata/handlers/doi.py | 4 +- geonode/metadata/handlers/group.py | 4 +- geonode/metadata/handlers/hkeyword.py | 4 +- geonode/metadata/handlers/linkedresource.py | 2 +- geonode/metadata/handlers/region.py | 4 +- geonode/metadata/handlers/sparse.py | 57 ++++++++++++++----- geonode/metadata/handlers/thesaurus.py | 4 +- geonode/metadata/i18n.py | 39 +++++++++++++ geonode/metadata/manager.py | 9 ++- .../base_schema.json => schemas/base.json} | 0 .../full.json} | 0 .../full_schema_V1.json | 0 geonode/metadata/settings.py | 2 +- 20 files changed, 183 insertions(+), 61 deletions(-) rename geonode/inspire/{inspire.py => metadata.py} (95%) create mode 100644 geonode/metadata/i18n.py rename geonode/metadata/{jsonschema_examples/base_schema.json => schemas/base.json} (100%) rename geonode/metadata/{jsonschema_examples/base_schema_full.json => schemas/full.json} (100%) rename geonode/metadata/{jsonschema_examples => schemas}/full_schema_V1.json (100%) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 8fbbdf71407..d45e4119bc5 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -1457,7 +1457,7 @@ def base_linked_resources(instance, user, params): return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) -def base_linked_resources_payload(instance, user, params={}): +def base_linked_resources_instances(instance, user, params={}): resource_type = params.get("resource_type", None) link_type = params.get("link_type", None) type_list = resource_type.split(",") if resource_type else [] @@ -1498,7 +1498,7 @@ def base_linked_resources_payload(instance, user, params={}): linked_to_visib_ids = linked_to_visib.values_list("id", flat=True) linked_to = [lres for lres in linked_to_over_loopable if lres.target.id in linked_to_visib_ids] - ret["linked_to"] = LinkedResourceSerializer(linked_to, embed=True, many=True).data + ret["linked_to"] = linked_to if not link_type or link_type == "linked_by": linked_by_over = instance.get_linked_resources(as_target=True) @@ -1517,11 +1517,21 @@ def base_linked_resources_payload(instance, user, params={}): linked_by_visib_ids = linked_by_visib.values_list("id", flat=True) linked_by = [lres for lres in linked_by_over_loopable if lres.source.id in linked_by_visib_ids] - ret["linked_by"] = LinkedResourceSerializer( - instance=linked_by, serialize_source=True, embed=True, many=True - ).data + ret["linked_by"] = linked_by if not ret["WARNINGS"]: ret.pop("WARNINGS") return ret + + +def base_linked_resources_payload(instance, user, params={}): + lres = base_linked_resources_instances(instance, user, params) + ret = { + "linked_to": LinkedResourceSerializer(lres["linked_to"], embed=True, many=True).data, + "linked_by": LinkedResourceSerializer(instance=lres["linked_by"], serialize_source=True, embed=True, many=True).data + } + if lres.get("WARNINGS", None): + ret["WARNINGS"] = lres["WARNINGS"] + + return ret \ No newline at end of file diff --git a/geonode/inspire/apps.py b/geonode/inspire/apps.py index f32718878cb..650fd8da32d 100644 --- a/geonode/inspire/apps.py +++ b/geonode/inspire/apps.py @@ -27,6 +27,6 @@ class BaseAppConfig(NotificationsAppConfigBase, AppConfig): def ready(self): super().ready() - from geonode.inspire.inspire import init + from geonode.inspire.metadata import init init() diff --git a/geonode/inspire/inspire.py b/geonode/inspire/metadata.py similarity index 95% rename from geonode/inspire/inspire.py rename to geonode/inspire/metadata.py index 4f1af2836db..902180aca4a 100644 --- a/geonode/inspire/inspire.py +++ b/geonode/inspire/metadata.py @@ -25,13 +25,15 @@ # Should be "otherRestrictions", see https://github.com/GeoNode/geonode/issues/12745 FIELDVAL_RESTRICTION_TYPE_DEFAULT = "limitation not listed" +CONTEXT_ID = "inspire" + class INSPIREHandler(MetadataHandler): """ The INSPIRE Handler adds the Regions model options to the schema """ - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): # Schema overriding for INSPIRE # Some Additional Sparse Fields are registered in geonode.inspire.inspire.init() @@ -157,11 +159,11 @@ def update_schema(self, jsonschema, lang=None): return jsonschema def load_serialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): - context["inspire"] = {"schema": jsonschema} + context[CONTEXT_ID] = {"schema": jsonschema} fields = {} for field in SparseField.get_fields(resource, names=(FIELD_RESTRICTION_ACCESS_USE,)): fields[field.name] = field.value - context["inspire"]["fields"] = fields + context[CONTEXT_ID]["fields"] = fields def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): if field_name == FIELD_RESTRICTION_TYPE: @@ -170,7 +172,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No return FIELDVAL_RESTRICTION_TYPE_DEFAULT elif field_name == FIELD_RESTRICTION_PUBLIC_ACCESS: - if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_PUBLIC_ACCESS].get("readOnly", False): + if context[CONTEXT_ID]["schema"]["properties"][FIELD_RESTRICTION_PUBLIC_ACCESS].get("readOnly", False): self._set_error( errors, [FIELD_RESTRICTION_PUBLIC_ACCESS], @@ -184,7 +186,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No return {"id": resource.constraints_other} elif field_name == FIELD_RESTRICTION_ACCESS_USE: - if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): + if context[CONTEXT_ID]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): self._set_error( errors, [FIELD_RESTRICTION_ACCESS_USE], @@ -195,7 +197,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No "Get it from https://inspire.ec.europa.eu/metadata-codelist/ConditionsApplyingToAccessAndUse" ) else: - val = context["inspire"]["fields"].get(FIELD_RESTRICTION_ACCESS_USE, None) + val = context[CONTEXT_ID]["fields"].get(FIELD_RESTRICTION_ACCESS_USE, None) if not val: return {"choice": {"id": None}, "freetext": None} elif val.startswith("http"): @@ -213,7 +215,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No raise Exception(f"The INSPIRE handler does not support field {field_name}") def load_deserialization_context(self, resource: ResourceBase, jsonschema: dict, context: dict): - context["inspire"] = {"schema": jsonschema} + context[CONTEXT_ID] = {"schema": jsonschema} def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): if field_name == FIELD_RESTRICTION_TYPE: @@ -243,7 +245,7 @@ def update_resource(self, resource, field_name, json_instance, context, errors, field_value = json_instance.get(field_name, {}) - if context["inspire"]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): + if context[CONTEXT_ID]["schema"]["properties"][FIELD_RESTRICTION_ACCESS_USE].get("readOnly", False): content = "N/A" self._set_error( errors, diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 47e4645662a..b1c439bc8c3 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -19,12 +19,12 @@ import logging from dal import autocomplete -from django.contrib.auth import get_user_model -from django.core.handlers.wsgi import WSGIRequest from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response +from django.contrib.auth import get_user_model +from django.core.handlers.wsgi import WSGIRequest from django.http import JsonResponse from django.utils.translation.trans_real import get_language_from_request from django.utils.translation import get_language @@ -109,18 +109,13 @@ def schema_instance(self, request, pk=None): def tkeywords_autocomplete(request: WSGIRequest, thesaurusid): - lang = get_language() + lang = remove_country_from_languagecode(get_language()) all_keywords_qs = ThesaurusKeyword.objects.filter(thesaurus_id=thesaurusid) # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( "keyword_id" ) - if not localized_k_ids_qs.exists(): - lang = remove_country_from_languagecode(lang) - localized_k_ids_qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=all_keywords_qs).values( - "keyword_id" - ) # consider all the keywords that do not have a translation in the requested language keywords_not_translated_qs = ( diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 6ceecd24e2c..36d897eb9cf 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -19,6 +19,10 @@ import logging from abc import ABCMeta, abstractmethod +from typing_extensions import deprecated + +from django.utils.translation import gettext as _ + from geonode.base.models import ResourceBase logger = logging.getLogger(__name__) @@ -31,7 +35,7 @@ class MetadataHandler(metaclass=ABCMeta): """ @abstractmethod - def update_schema(self, jsonschema: dict, lang=None): + def update_schema(self, jsonschema: dict, context, lang=None): """ It is called by the MetadataManager when creating the JSON Schema It adds the subschema handled by the handler, and returns the @@ -72,8 +76,28 @@ def load_deserialization_context(self, resource: ResourceBase, jsonschema: dict, """ pass + def _add_subschema(self, jsonschema, property_name, subschema, after_what=None): + after_what = after_what or subschema.get("geonode:after", None) + + if not after_what: + jsonschema["properties"][property_name] = subschema + else: + ret_properties = {} + added = False + for key, val in jsonschema["properties"].items(): + ret_properties[key] = val + if key == after_what: + ret_properties[property_name] = subschema + added = True + + if not added: + logger.warning(f'Could not add "{property_name}" after "{after_what}"') + ret_properties[property_name] = subschema + + jsonschema["properties"] = ret_properties + + @deprecated("Use _add_subschema instead") def _add_after(self, jsonschema, after_what, property_name, subschema): - # add thesauri after category ret_properties = {} added = False for key, val in jsonschema["properties"].items(): @@ -96,3 +120,18 @@ def _set_error(errors: dict, path: list, msg: str): elem = elem.setdefault(step, {}) elem = elem.setdefault("__errors", []) elem.append(msg) + + @staticmethod + def _localize_label(context, lang: str, text: str): + # Try localization via thesaurus: + label = context["labels"].get(text, None) + # fallback: gettext() + if not label: + label = _(text) + + return label + + @staticmethod + def _localize_subschema_label(context, subschema: dict, lang: str, annotation_name: str): + if annotation_name in subschema: + subschema[annotation_name] = MetadataHandler._localize_label(context, lang, subschema[annotation_name]) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index d2e18fe79f2..12bdc9ea4f8 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -171,17 +171,14 @@ def __init__(self): self.json_base_schema = JSONSCHEMA_BASE self.base_schema = None - def update_schema(self, jsonschema, lang=None): - def localize(subschema: dict, annotation_name): - if annotation_name in subschema: - subschema[annotation_name] = _(subschema[annotation_name]) + def update_schema(self, jsonschema, context, lang=None): with open(self.json_base_schema) as f: self.base_schema = json.load(f) # building the full base schema for property_name, subschema in self.base_schema.items(): - localize(subschema, "title") - localize(subschema, "description") + self._localize_subschema_label(context, subschema, lang, "title") + self._localize_subschema_label(context, subschema, lang, "description") jsonschema["properties"][property_name] = subschema diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index e9b00e7e191..38469b93e18 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -34,18 +34,20 @@ class ContactHandler(MetadataHandler): Handles role contacts """ - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): contacts = {} required = [] for role in Roles: - card = f'[{"1" if role.is_required else "0"}..{"N" if role.is_multivalue else "1"}]' + minitems = 1 if role.is_required else 0 + card = f'[{minitems}..{"N" if role.is_multivalue else "1"}]' if role.is_required: required.append(role.name) if role.is_multivalue: contact = { "type": "array", - "title": _(role.label) + " " + card, + "title": self._localize_label(context, lang, role.label) + " " + card, + "minItems": minitems, "items": { "type": "object", "properties": { @@ -64,7 +66,7 @@ def update_schema(self, jsonschema, lang=None): else: contact = { "type": "object", - "title": _(role.label) + " " + card, + "title": self._localize_label(context, lang, role.label) + " " + card, "properties": { "id": { "type": "string", @@ -84,7 +86,7 @@ def update_schema(self, jsonschema, lang=None): jsonschema["properties"]["contacts"] = { "type": "object", - "title": _("Contacts"), + "title": self._localize_label(context, lang, "contacts"), "properties": contacts, "required": required, "geonode:required": bool(required), diff --git a/geonode/metadata/handlers/doi.py b/geonode/metadata/handlers/doi.py index 74183d5857c..5fb565fe47e 100644 --- a/geonode/metadata/handlers/doi.py +++ b/geonode/metadata/handlers/doi.py @@ -28,7 +28,7 @@ class DOIHandler(MetadataHandler): - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): doi_schema = { "type": ["string", "null"], "title": "DOI", @@ -38,7 +38,7 @@ def update_schema(self, jsonschema, lang=None): } # add DOI after edition - self._add_after(jsonschema, "edition", "doi", doi_schema) + self._add_subschema(jsonschema, "doi", doi_schema, after_what="edition") return jsonschema def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py index a78775bc80e..3f7d7f388a3 100644 --- a/geonode/metadata/handlers/group.py +++ b/geonode/metadata/handlers/group.py @@ -35,7 +35,7 @@ class GroupHandler(MetadataHandler): an entry in the resource management panel """ - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): group_schema = { "type": "object", "title": _("group"), @@ -54,7 +54,7 @@ def update_schema(self, jsonschema, lang=None): } # add group after date_type - self._add_after(jsonschema, "date_type", "group", group_schema) + self._add_subschema(jsonschema, "group", group_schema, after_what="date_type") return jsonschema diff --git a/geonode/metadata/handlers/hkeyword.py b/geonode/metadata/handlers/hkeyword.py index 7ca0fcdb73e..24b67c86903 100644 --- a/geonode/metadata/handlers/hkeyword.py +++ b/geonode/metadata/handlers/hkeyword.py @@ -30,7 +30,7 @@ class HKeywordHandler(MetadataHandler): - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): hkeywords = { "type": "array", "title": _("Keywords"), @@ -47,7 +47,7 @@ def update_schema(self, jsonschema, lang=None): "geonode:handler": "hkeyword", } - self._add_after(jsonschema, "tkeywords", "hkeywords", hkeywords) + self._add_subschema(jsonschema, "hkeywords", hkeywords, after_what="tkeywords") return jsonschema def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): diff --git a/geonode/metadata/handlers/linkedresource.py b/geonode/metadata/handlers/linkedresource.py index 77520f568b7..0099e08f15e 100644 --- a/geonode/metadata/handlers/linkedresource.py +++ b/geonode/metadata/handlers/linkedresource.py @@ -30,7 +30,7 @@ class LinkedResourceHandler(MetadataHandler): - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): linked = { "type": "array", "title": _("Related resources"), diff --git a/geonode/metadata/handlers/region.py b/geonode/metadata/handlers/region.py index c33cac429ca..e6690cb7e00 100644 --- a/geonode/metadata/handlers/region.py +++ b/geonode/metadata/handlers/region.py @@ -33,7 +33,7 @@ class RegionHandler(MetadataHandler): The RegionsHandler adds the Regions model options to the schema """ - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): regions = { "type": "array", "title": _("Regions"), @@ -52,7 +52,7 @@ def update_schema(self, jsonschema, lang=None): } # add regions after Attribution - self._add_after(jsonschema, "attribution", "regions", regions) + self._add_subschema(jsonschema, "regions", regions, after_what="attribution") return jsonschema def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 018264634ca..2930febedd3 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # ######################################################################### - +import json import logging from geonode.metadata.handlers.abstract import MetadataHandler @@ -25,6 +25,9 @@ logger = logging.getLogger(__name__) +CONTEXT_ID = "sparse" + + class SparseFieldRegistry: sparse_fields = {} @@ -44,17 +47,17 @@ class SparseHandler(MetadataHandler): Handles sparse in fields in the SparseField table """ - def update_schema(self, jsonschema, lang=None): - # building the full base schema + def update_schema(self, jsonschema, context, lang=None): + # add all registered fields # TODO: manage i18n (thesaurus?) - for field_name, field_info in sparse_field_registry.fields().items(): subschema = field_info["schema"] - if after := field_info["after"]: - self._add_after(jsonschema, after, field_name, subschema) - else: - jsonschema["properties"][field_name] = subschema + + self._localize_subschema_label(context, subschema, lang, "title") + self._localize_subschema_label(context, subschema, lang, "description") + + self._add_subschema(jsonschema, field_name, subschema, after_what=field_info["after"]) # add the handler info to the dictionary if it doesn't exist if "geonode:handler" not in subschema: @@ -69,21 +72,49 @@ def update_schema(self, jsonschema, lang=None): def load_serialization_context(self, resource, jsonschema: dict, context: dict): logger.debug(f"Preloading sparse fields {sparse_field_registry.fields().keys()}") - context["sparse"] = { + context[CONTEXT_ID] = { "fields": { f.name: f.value for f in SparseField.get_fields(resource, names=sparse_field_registry.fields().keys()) - } + }, + "schema": jsonschema, } def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): - return context["sparse"]["fields"].get(field_name, None) + field_type = context[CONTEXT_ID]["schema"]["properties"][field_name]["type"] + match field_type: + case "string": + return context[CONTEXT_ID]["fields"].get(field_name, None) + case "array": + # assuming it's an array of string: TODO implement other cases + try: + arr = context[CONTEXT_ID]["fields"].get(field_name, None) or "[]" + return json.loads(arr) + except Exception as e: + logger.warning(f"Error loading field '{field_name}' with content ({type(arr)}){arr}: {e}") + case _: + logger.warning(f"Unhandled type '{field_type}' for sparse field '{field_name}'") + return None + + def load_deserialization_context(self, resource, jsonschema: dict, context: dict): + context[CONTEXT_ID] = {"schema": jsonschema} def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): - field_value = json_instance.get(field_name, None) + bare_value = json_instance.get(field_name, None) + type = context[CONTEXT_ID]["schema"]["properties"][field_name]["type"] + match type: + case "string": + field_value = bare_value + case "array": + field_value = json.dumps(bare_value) if bare_value else [] + case _: + logger.warning(f"Unhandled type '{type}' for sparse field '{field_name}'") + self._set_error(errors, [field_name], f"Unhandled type {type}. Contact your administrator") + return + try: sf, created = SparseField.objects.update_or_create( defaults={"value": field_value}, resource=resource, name=field_name ) except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") - self._set_error(errors, ["field_name"], f"Error setting value: {e}") + self._set_error(errors, [field_name], f"Error setting value: {e}") diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 7d83fc7178a..0d537ffd0c3 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -82,7 +82,7 @@ def collect_thesauri(filter, lang=None): return collected_thesauri - def update_schema(self, jsonschema, lang=None): + def update_schema(self, jsonschema, context, lang=None): collected_thesauri = self.collect_thesauri(~Q(card_max=0), lang=lang) @@ -130,7 +130,7 @@ def update_schema(self, jsonschema, lang=None): } # add thesauri after category - self._add_after(jsonschema, "category", TKEYWORDS, tkeywords) + self._add_subschema(jsonschema, TKEYWORDS, tkeywords, after_what="category") return jsonschema diff --git a/geonode/metadata/i18n.py b/geonode/metadata/i18n.py new file mode 100644 index 00000000000..2658fa5e9e3 --- /dev/null +++ b/geonode/metadata/i18n.py @@ -0,0 +1,39 @@ +import logging + +from django.db import connection + +logger = logging.getLogger(__name__) + +I18N_THESAURUS_IDENTIFIER = "labels_i18n" + + +def get_localized_tkeywords(lang, thesaurus_identifier: str): + logger.debug("Loading localized tkeyword from DB") + + query = ( + "select " + " tk.id," + " tk.about," + " tk.alt_label," + " tkl.label" + " from" + " base_thesaurus th," + " base_thesauruskeyword tk" + " left outer join " + " (select keyword_id, lang, label from base_thesauruskeywordlabel" + " where lang = %s) as tkl" + " on (tk.id = tkl.keyword_id)" + " where th.identifier = %s" + " and tk.thesaurus_id = th.id" + " order by label, alt_label" + ) + ret = [] + with connection.cursor() as cursor: + cursor.execute(query, [lang, thesaurus_identifier]) + for id, about, alt, label in cursor.fetchall(): + ret.append({"id": id, "about": about, "label": label or alt}) + return sorted(ret, key=lambda i: i["label"].lower()) + + +def get_localized_labels(lang, key="about"): + return {i[key]: i["label"] for i in get_localized_tkeywords(lang, I18N_THESAURUS_IDENTIFIER)} diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 67fc4719d7b..03d08173fab 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -24,6 +24,7 @@ from django.utils.translation import gettext as _ from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.i18n import get_localized_labels from geonode.metadata.settings import MODEL_SCHEMA logger = logging.getLogger(__name__) @@ -48,15 +49,21 @@ def __init__(self): def add_handler(self, handler_id, handler): self.handlers[handler_id] = handler() + def _init_schema_context(self, lang): + # todo: cache localizations + return {"labels": get_localized_labels(lang)} + def build_schema(self, lang=None): logger.debug(f"build_schema {lang}") schema = copy.deepcopy(self.root_schema) schema["title"] = _(schema["title"]) + context = self._init_schema_context(lang) + for key, handler in self.handlers.items(): # logger.debug(f"build_schema: update schema -> {key}") - schema = handler.update_schema(schema, lang) + schema = handler.update_schema(schema, context, lang) # Set required fields. required = [] diff --git a/geonode/metadata/jsonschema_examples/base_schema.json b/geonode/metadata/schemas/base.json similarity index 100% rename from geonode/metadata/jsonschema_examples/base_schema.json rename to geonode/metadata/schemas/base.json diff --git a/geonode/metadata/jsonschema_examples/base_schema_full.json b/geonode/metadata/schemas/full.json similarity index 100% rename from geonode/metadata/jsonschema_examples/base_schema_full.json rename to geonode/metadata/schemas/full.json diff --git a/geonode/metadata/jsonschema_examples/full_schema_V1.json b/geonode/metadata/schemas/full_schema_V1.json similarity index 100% rename from geonode/metadata/jsonschema_examples/full_schema_V1.json rename to geonode/metadata/schemas/full_schema_V1.json diff --git a/geonode/metadata/settings.py b/geonode/metadata/settings.py index bb1d8ca985b..d2a7c786bfe 100644 --- a/geonode/metadata/settings.py +++ b/geonode/metadata/settings.py @@ -10,7 +10,7 @@ } # The base schema is defined as a file in order to be customizable from other GeoNode instances -JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/jsonschema_examples/base_schema.json") +JSONSCHEMA_BASE = os.path.join(PROJECT_ROOT, "metadata/schemas/base.json") METADATA_HANDLERS = { "base": "geonode.metadata.handlers.base.BaseHandler", From 1cfa4464b6a20ce306596ca161cd9d72afbceec0 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 13 Dec 2024 09:59:31 +0200 Subject: [PATCH 66/91] tests for views --- geonode/metadata/api/views.py | 4 +- geonode/metadata/tests.py | 133 ++++++++++++++++++ .../0050_alter_uploadsizelimit_max_size.py | 20 +++ 3 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 geonode/upload/migrations/0050_alter_uploadsizelimit_max_size.py diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b1c439bc8c3..98e0ee1dcc3 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -52,7 +52,7 @@ def list(self, request): # Get the JSON schema # A pk argument is set for futured multiple schemas - @action(detail=False, methods=["get"], url_path=r"schema(?:/(?P\d+))?") + @action(detail=False, methods=["get"], url_path=r"schema(?:/(?P\d+))?", url_name="schema") def schema(self, request, pk=None): """ The user is able to export her/his keys with @@ -70,7 +70,7 @@ def schema(self, request, pk=None): return Response(response) # Get the JSON schema - @action(detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)") + @action(detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)", url_name="schema_instance") def schema_instance(self, request, pk=None): try: diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests.py index e69de29bb2d..c229ed9ca81 100644 --- a/geonode/metadata/tests.py +++ b/geonode/metadata/tests.py @@ -0,0 +1,133 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import logging +import shutil +import io +import json +from unittest.mock import patch +from uuid import uuid4 + +from django.urls import reverse +from django.utils.module_loading import import_string +from django.contrib.auth import get_user_model +from rest_framework import status + +from rest_framework.test import APITestCase +from geonode.metadata.settings import MODEL_SCHEMA +from geonode.settings import PROJECT_ROOT +from geonode.metadata.manager import metadata_manager +from geonode.metadata.settings import METADATA_HANDLERS +from geonode.base.models import ResourceBase + + +class MetadataApiTests(APITestCase): + + def setUp(self): + # set Json schemas + self.model_schema = MODEL_SCHEMA + self.base_schema = os.path.join(PROJECT_ROOT, "metadata/tests/data/base.json") + self.lang = None + + self.context = metadata_manager._init_schema_context(self.lang) + + self.handlers = {} + for handler_id, module_path in METADATA_HANDLERS.items(): + handler_cls = import_string(module_path) + self.handlers[handler_id] = handler_cls() + + self.test_user = get_user_model().objects.create_user( + "someuser", "someuser@fakemail.com", "somepassword", is_active=True + ) + self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user) + + def tearDown(self): + super().tearDown() + + # tests for the encpoint metadata/schema + def test_schema_valid_structure(self): + """ + Ensure the returned basic structure of the schema + """ + + url = reverse('metadata-schema') + + # Make a GET request to the action + response = self.client.get(url, format="json") + + # Assert that the response is in JSON format + self.assertEqual(response['Content-Type'], 'application/json') + + # Check that the response status code is 200 + self.assertEqual(response.status_code, status.HTTP_200_OK) + + # Check the structure of the schema + response_data = response.json() + for field in self.model_schema.keys(): + self.assertIn(field, response_data) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_schema_not_found(self, mock_get_schema): + """ + Test the behaviour of the schema endpoint + if the schema is not found + """ + mock_get_schema.return_value = None + + url = reverse("metadata-schema") + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, {"Message": "Schema not found"}) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_schema_with_lang(self, mock_get_schema): + """ + Test that the view recieves the lang parameter + """ + mock_get_schema.return_value = {"fake_schema": "schema"} + + url = reverse("metadata-schema") + response = self.client.get(url, {"lang": "it"}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, {"fake_schema": "schema"}) + + # Verify that get_schema was called with the correct lang + mock_get_schema.assert_called_once_with("it") + + @patch("geonode.metadata.manager.metadata_manager.build_schema_instance") + def test_schema_instance_get_success(self, mock_build_schema_instance): + + mock_build_schema_instance.return_value = {"fake_schema_instance": "schema_instance"} + + url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertJSONEqual(response.content, {"fake_schema_instance": "schema_instance"}) + + # Ensure the mocked method was called with the correct arguments + mock_build_schema_instance.assert_called_once_with(self.resource, "en") + + + + + \ No newline at end of file diff --git a/geonode/upload/migrations/0050_alter_uploadsizelimit_max_size.py b/geonode/upload/migrations/0050_alter_uploadsizelimit_max_size.py new file mode 100644 index 00000000000..ab612dbd5d5 --- /dev/null +++ b/geonode/upload/migrations/0050_alter_uploadsizelimit_max_size.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.16 on 2024-12-13 07:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("upload", "0049_move_data_from_importer_to_upload"), + ] + + operations = [ + migrations.AlterField( + model_name="uploadsizelimit", + name="max_size", + field=models.PositiveBigIntegerField( + default=104857600, help_text="The maximum file size allowed for upload (bytes)." + ), + ), + ] From b89471e22bf35b817c703c90c5515e59bf2bf054 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 13 Dec 2024 12:29:34 +0200 Subject: [PATCH 67/91] adding more tests for views --- geonode/metadata/api/views.py | 2 +- geonode/metadata/tests.py | 83 +++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 98e0ee1dcc3..426b0d27799 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -104,7 +104,7 @@ def schema_instance(self, request, pk=None): except ResourceBase.DoesNotExist: result = {"message": "The dataset was not found"} - return Response(result) + return Response(result, status=404) def tkeywords_autocomplete(request: WSGIRequest, thesaurusid): diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests.py index c229ed9ca81..03fe46b1427 100644 --- a/geonode/metadata/tests.py +++ b/geonode/metadata/tests.py @@ -43,7 +43,6 @@ class MetadataApiTests(APITestCase): def setUp(self): # set Json schemas self.model_schema = MODEL_SCHEMA - self.base_schema = os.path.join(PROJECT_ROOT, "metadata/tests/data/base.json") self.lang = None self.context = metadata_manager._init_schema_context(self.lang) @@ -114,7 +113,7 @@ def test_schema_with_lang(self, mock_get_schema): mock_get_schema.assert_called_once_with("it") @patch("geonode.metadata.manager.metadata_manager.build_schema_instance") - def test_schema_instance_get_success(self, mock_build_schema_instance): + def test_get_schema_instance_with_default_lang(self, mock_build_schema_instance): mock_build_schema_instance.return_value = {"fake_schema_instance": "schema_instance"} @@ -124,10 +123,86 @@ def test_schema_instance_get_success(self, mock_build_schema_instance): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertJSONEqual(response.content, {"fake_schema_instance": "schema_instance"}) - # Ensure the mocked method was called with the correct arguments - mock_build_schema_instance.assert_called_once_with(self.resource, "en") + # Ensure the mocked method was called + mock_build_schema_instance.assert_called() + @patch("geonode.metadata.manager.metadata_manager.build_schema_instance") + def test_get_schema_instance_with_lang(self, mock_build_schema_instance): + + mock_build_schema_instance.return_value = {"fake_schema_instance": "schema_instance"} + + url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) + response = self.client.get(url, {"lang": "it"}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertJSONEqual(response.content, {"fake_schema_instance": "schema_instance"}) + + # Ensure the mocked method was called with the correct arguments + mock_build_schema_instance.assert_called_once_with(self.resource, "it") + + @patch("geonode.metadata.manager.metadata_manager.update_schema_instance") + def test_put_patch_schema_instance_with_no_errors(self, mock_update_schema_instance): + + url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) + fake_payload = {"field": "value"} + # set the returned value of the mocked update_schema_instance with an empty dict + errors = {} + mock_update_schema_instance.return_value = errors + + methods = [self.client.put, self.client.patch] + + for method in methods: + + response = method(url, data=fake_payload, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertJSONEqual( + response.content, + {"message": "The resource was updated successfully", "extraErrors": errors} + ) + mock_update_schema_instance.assert_called_with(self.resource, fake_payload) + + @patch("geonode.metadata.manager.metadata_manager.update_schema_instance") + def test_put_patch_schema_instance_with_errors(self, mock_update_schema_instance): + + url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) + fake_payload = {"field": "value"} + + # Set fake errors + errors = { + "fake_error_1": "Field 'title' is required", + "fake_error_2": "Invalid value for 'type'" + } + mock_update_schema_instance.return_value = errors + + methods = [self.client.put, self.client.patch] + + for method in methods: + + response = method(url, data=fake_payload, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertJSONEqual( + response.content, + {"message": "Some errors were found while updating the resource", "extraErrors": errors} + ) + mock_update_schema_instance.assert_called_with(self.resource, fake_payload) + + def test_resource_not_found(self): + # Define a fake primary key + fake_pk = 1000 + + # Construct the URL for the action + url = reverse("metadata-schema_instance", kwargs={"pk": fake_pk}) + + # Perform a GET request + response = self.client.get(url) + + # Verify the response + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertJSONEqual( + response.content, + {"message": "The dataset was not found"} + ) \ No newline at end of file From ed76dfd14de1b52e0dc49a6b15b678804d79d6db Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 13 Dec 2024 11:30:30 +0100 Subject: [PATCH 68/91] Tkeywords: hide property if no thesaurus configured --- geonode/metadata/handlers/thesaurus.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/thesaurus.py b/geonode/metadata/handlers/thesaurus.py index 0d537ffd0c3..3bb5f9093d7 100644 --- a/geonode/metadata/handlers/thesaurus.py +++ b/geonode/metadata/handlers/thesaurus.py @@ -124,11 +124,13 @@ def update_schema(self, jsonschema, context, lang=None): "description": _("List of keywords from Thesaurus"), "geonode:handler": "thesaurus", "properties": thesauri, - # "ui:options": { - # 'geonode-ui:group': 'Thesauri grop' - # } } + # We are going to hide the tkeywords property if there's no thesaurus configured + # We can't remove the property altogether, since hkeywords relies on tkeywords for positioning + if not thesauri: + tkeywords["ui:widget"] = "hidden" + # add thesauri after category self._add_subschema(jsonschema, TKEYWORDS, tkeywords, after_what="category") @@ -149,7 +151,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No del tks[tkl.keyword.id] if tks: - logger.info(f"Returning untraslated '{lang}' keywords: {tks}") + logger.info(f"Returning untranslated '{lang}' keywords: {tks}") for tk in tks.values(): keywords = ret.setdefault(tk.thesaurus.identifier, []) keywords.append({"id": tk.about, "label": tk.alt_label}) From 24bf1c8c9cc11af1d2fd6f4efd8b50d7535ff83a Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 13 Dec 2024 13:35:08 +0100 Subject: [PATCH 69/91] Create test errors recursively --- geonode/metadata/manager.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 03d08173fab..33a0daba917 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -106,8 +106,8 @@ def build_schema_instance(self, resource, lang=None): # TESTING ONLY if "error" in resource.title.lower(): - MetadataHandler._set_error(errors, ["title"], "GET: test msg under /title") - MetadataHandler._set_error(errors, ["properties", "title"], "GET: test msg under /properties/title") + for fieldname in schema["properties"]: + MetadataHandler._set_error(errors, [fieldname], f"TEST: test msg for field '{fieldname}' in GET request") instance["extraErrors"] = errors return instance @@ -135,17 +135,21 @@ def update_schema_instance(self, resource, json_instance) -> dict: # TESTING ONLY if "error" in resource.title.lower(): - errors.setdefault("title", {}).setdefault("__errors", []).append("PUT: this is a test error under /title") - errors.setdefault("properties", {}).setdefault("title", {}).setdefault("__errors", []).append( - "PUT: this is a test error under /properties/title" - ) - - MetadataHandler._set_error(errors, ["title"], "PUT: this is another test msg under /title") - MetadataHandler._set_error( - errors, ["properties", "title"], "PUT: this is another test msg under /properties/title" - ) + _create_test_errors(schema, errors, [], "TEST: field <{schema_type}>'{path}' PUT request") return errors +def _create_test_errors(schema, errors, path, msg_template, create_message = True): + if create_message: + stringpath = "/".join(path) + MetadataHandler._set_error(errors, path, msg_template.format(path=stringpath, schema_type=schema['type'])) + + if schema["type"] == "object": + for field, subschema in schema["properties"].items(): + _create_test_errors(subschema, errors, path + [field], msg_template) + elif schema["type"] == "array": + _create_test_errors(schema["items"], errors, path, msg_template, create_message=False) + + metadata_manager = MetadataManager() From 5316dbef4cd6ea23304972616b9dbf7e247e1fcf Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 13 Dec 2024 17:36:42 +0100 Subject: [PATCH 70/91] Recurse localization in complex sparse fields --- geonode/metadata/handlers/sparse.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 2930febedd3..23240654f6e 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -46,17 +46,24 @@ class SparseHandler(MetadataHandler): """ Handles sparse in fields in the SparseField table """ + def _recurse_localization(self, context, schema, lang): + self._localize_subschema_label(context, schema, lang, "title") + self._localize_subschema_label(context, schema, lang, "description") + + match schema["type"]: + case "object": + for _, subschema in schema["properties"].items(): + self._recurse_localization(context, subschema, lang) + case "array": + self._recurse_localization(context, schema["items"], lang) + case _: + pass def update_schema(self, jsonschema, context, lang=None): # add all registered fields - - # TODO: manage i18n (thesaurus?) for field_name, field_info in sparse_field_registry.fields().items(): subschema = field_info["schema"] - - self._localize_subschema_label(context, subschema, lang, "title") - self._localize_subschema_label(context, subschema, lang, "description") - + self._recurse_localization(context, subschema, lang) self._add_subschema(jsonschema, field_name, subschema, after_what=field_info["after"]) # add the handler info to the dictionary if it doesn't exist From 99b1396a155cf7433afcf93b2ba35adb7a9e9835 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 17 Dec 2024 15:36:16 +0100 Subject: [PATCH 71/91] Metadata: fix contact roles --- geonode/metadata/handlers/contact.py | 32 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 38469b93e18..f7b6351d279 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -28,6 +28,23 @@ logger = logging.getLogger(__name__) +# contact roles names are spread in the code, let's map them here: +ROLE_NAMES_MAP = { + Roles.OWNER: "owner", # this is not saved as a contact + Roles.METADATA_AUTHOR: "author", + Roles.PROCESSOR: Roles.PROCESSOR.name, + Roles.PUBLISHER: Roles.PUBLISHER.name, + Roles.CUSTODIAN: Roles.CUSTODIAN.name, + Roles.POC: "pointOfContact", + Roles.DISTRIBUTOR: Roles.DISTRIBUTOR.name, + Roles.RESOURCE_USER: Roles.RESOURCE_USER.name, + Roles.RESOURCE_PROVIDER: Roles.RESOURCE_PROVIDER.name, + Roles.ORIGINATOR: Roles.ORIGINATOR.name, + Roles.PRINCIPAL_INVESTIGATOR: Roles.PRINCIPAL_INVESTIGATOR.name, +} + +NAMES_ROLE_MAP = {v:k for k,v in ROLE_NAMES_MAP.items()} + class ContactHandler(MetadataHandler): """ @@ -82,7 +99,8 @@ def update_schema(self, jsonschema, context, lang=None): "required": ["id"] if role.is_required else [], } - contacts[role.name] = contact + rolename = ROLE_NAMES_MAP[role] + contacts[rolename] = contact jsonschema["properties"]["contacts"] = { "type": "object", @@ -103,15 +121,16 @@ def __create_user_entry(user): contacts = {} for role in Roles: + rolename = ROLE_NAMES_MAP[role] if role.is_multivalue: - content = [__create_user_entry(user) for user in resource.__get_contact_role_elements__(role) or []] + content = [__create_user_entry(user) for user in resource.__get_contact_role_elements__(rolename) or []] else: - users = resource.__get_contact_role_elements__(role) + users = resource.__get_contact_role_elements__(rolename) if not users and role == Roles.OWNER: users = [resource.owner] content = __create_user_entry(users[0]) if users else None - contacts[role.name] = content + contacts[rolename] = content return contacts @@ -119,14 +138,13 @@ def update_resource(self, resource, field_name, json_instance, context, errors, data = json_instance[field_name] logger.debug(f"CONTACTS {data}") for rolename, users in data.items(): - if rolename == Roles.OWNER.OWNER.name: + if rolename == Roles.OWNER.name: if not users: logger.warning(f"User not specified for role '{rolename}'") self._set_error(errors, ["contacts", rolename], f"User not specified for role '{rolename}'") else: resource.owner = get_user_model().objects.get(pk=users["id"]) else: - role = Roles.get_role_by_name(rolename) ids = [u["id"] for u in users] profiles = get_user_model().objects.filter(pk__in=ids) - resource.__set_contact_role_element__(profiles, role) + resource.__set_contact_role_element__(profiles, rolename) From a619d029dd3f1c6e74e3cd6bab2fae4fa8fdd231 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 17 Dec 2024 16:27:17 +0100 Subject: [PATCH 72/91] Metadata: improve handling of None values in sparse fields --- geonode/metadata/handlers/abstract.py | 4 ++ geonode/metadata/handlers/contact.py | 2 +- geonode/metadata/handlers/sparse.py | 91 ++++++++++++++++++--------- geonode/metadata/manager.py | 19 +++--- 4 files changed, 80 insertions(+), 36 deletions(-) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 36d897eb9cf..4f5b9b33bba 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -135,3 +135,7 @@ def _localize_label(context, lang: str, text: str): def _localize_subschema_label(context, subschema: dict, lang: str, annotation_name: str): if annotation_name in subschema: subschema[annotation_name] = MetadataHandler._localize_label(context, lang, subschema[annotation_name]) + + +class UnsetFieldException(Exception): + pass diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index f7b6351d279..7f5ad0a07eb 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -43,7 +43,7 @@ Roles.PRINCIPAL_INVESTIGATOR: Roles.PRINCIPAL_INVESTIGATOR.name, } -NAMES_ROLE_MAP = {v:k for k,v in ROLE_NAMES_MAP.items()} +NAMES_ROLE_MAP = {v: k for k, v in ROLE_NAMES_MAP.items()} class ContactHandler(MetadataHandler): diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 23240654f6e..38f115cf133 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -19,7 +19,7 @@ import json import logging -from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.handlers.abstract import MetadataHandler, UnsetFieldException from geonode.metadata.models import SparseField logger = logging.getLogger(__name__) @@ -46,13 +46,14 @@ class SparseHandler(MetadataHandler): """ Handles sparse in fields in the SparseField table """ + def _recurse_localization(self, context, schema, lang): self._localize_subschema_label(context, schema, lang, "title") self._localize_subschema_label(context, schema, lang, "description") match schema["type"]: case "object": - for _, subschema in schema["properties"].items(): + for subschema in schema["properties"].values(): self._recurse_localization(context, subschema, lang) case "array": self._recurse_localization(context, schema["items"], lang) @@ -86,42 +87,76 @@ def load_serialization_context(self, resource, jsonschema: dict, context: dict): "schema": jsonschema, } + @staticmethod + def _check_type(declared, checked): + return declared == checked or (type(declared) is list and checked in declared) + def get_jsonschema_instance(self, resource, field_name, context, errors, lang=None): field_type = context[CONTEXT_ID]["schema"]["properties"][field_name]["type"] - match field_type: - case "string": - return context[CONTEXT_ID]["fields"].get(field_name, None) - case "array": - # assuming it's an array of string: TODO implement other cases - try: - arr = context[CONTEXT_ID]["fields"].get(field_name, None) or "[]" - return json.loads(arr) - except Exception as e: - logger.warning(f"Error loading field '{field_name}' with content ({type(arr)}){arr}: {e}") - case _: - logger.warning(f"Unhandled type '{field_type}' for sparse field '{field_name}'") - return None + field_value = context[CONTEXT_ID]["fields"].get(field_name, None) + + is_nullable = self._check_type(field_type, "null") + is_string = self._check_type(field_type, "string") + is_number = self._check_type(field_type, "number") + + if field_name not in context[CONTEXT_ID]["fields"] and not is_nullable: + raise UnsetFieldException() + + if is_string or is_number: + return field_value + elif field_type == "array": + # assuming it's a single level array: TODO implement other cases + try: + return json.loads(field_value) if field_value is not None else None + except Exception as e: + logger.warning( + f"Error loading field '{field_name}' with content ({type(field_value)}){field_value}: {e}" + ) + elif field_type == "object": + # assuming it's a single level object: TODO implement other cases + try: + return json.loads(field_value) if field_value is not None else None + except Exception as e: + logger.warning( + f"Error loading field '{field_name}' with content ({type(field_value)}){field_value}: {e}" + ) + else: + logger.warning(f"Unhandled type '{field_type}' for sparse field '{field_name}'") + return None def load_deserialization_context(self, resource, jsonschema: dict, context: dict): context[CONTEXT_ID] = {"schema": jsonschema} def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): + def check_type(declared, checked): + return declared == checked or (type(declared) is list and checked in declared) + bare_value = json_instance.get(field_name, None) - type = context[CONTEXT_ID]["schema"]["properties"][field_name]["type"] - match type: - case "string": - field_value = bare_value - case "array": - field_value = json.dumps(bare_value) if bare_value else [] - case _: - logger.warning(f"Unhandled type '{type}' for sparse field '{field_name}'") - self._set_error(errors, [field_name], f"Unhandled type {type}. Contact your administrator") - return + field_type = context[CONTEXT_ID]["schema"]["properties"][field_name]["type"] + + is_nullable = check_type(field_type, "null") + + if check_type(field_type, "string") or check_type(field_type, "number"): + field_value = bare_value + elif field_type == "array": + field_value = json.dumps(bare_value) if bare_value else "[]" + elif field_type == "object": + field_value = json.dumps(bare_value) if bare_value else "{}" + else: + logger.warning(f"Unhandled type '{field_type}' for sparse field '{field_name}'") + self._set_error(errors, [field_name], f"Unhandled type {field_type}. Contact your administrator") + return try: - sf, created = SparseField.objects.update_or_create( - defaults={"value": field_value}, resource=resource, name=field_name - ) + if field_value is not None: + SparseField.objects.update_or_create( + defaults={"value": field_value}, resource=resource, name=field_name + ) + elif is_nullable: + SparseField.objects.filter(resource=resource, name=field_name).delete() + else: + self._set_error(errors, [field_name], f"Empty value not stored for field '{field_name}'") + logger.debug(f"Not setting null value for {field_name}") except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") self._set_error(errors, [field_name], f"Error setting value: {e}") diff --git a/geonode/metadata/manager.py b/geonode/metadata/manager.py index 33a0daba917..ff3367f30b0 100644 --- a/geonode/metadata/manager.py +++ b/geonode/metadata/manager.py @@ -23,7 +23,7 @@ from django.utils.translation import gettext as _ -from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.handlers.abstract import MetadataHandler, UnsetFieldException from geonode.metadata.i18n import get_localized_labels from geonode.metadata.settings import MODEL_SCHEMA @@ -101,13 +101,18 @@ def build_schema_instance(self, resource, lang=None): logger.warning(f"Missing geonode:handler for schema property {fieldname}. Skipping") continue handler = self.handlers[handler_id] - content = handler.get_jsonschema_instance(resource, fieldname, context, errors, lang) - instance[fieldname] = content + try: + content = handler.get_jsonschema_instance(resource, fieldname, context, errors, lang) + instance[fieldname] = content + except UnsetFieldException: + pass # TESTING ONLY if "error" in resource.title.lower(): for fieldname in schema["properties"]: - MetadataHandler._set_error(errors, [fieldname], f"TEST: test msg for field '{fieldname}' in GET request") + MetadataHandler._set_error( + errors, [fieldname], f"TEST: test msg for field '{fieldname}' in GET request" + ) instance["extraErrors"] = errors return instance @@ -140,10 +145,10 @@ def update_schema_instance(self, resource, json_instance) -> dict: return errors -def _create_test_errors(schema, errors, path, msg_template, create_message = True): +def _create_test_errors(schema, errors, path, msg_template, create_message=True): if create_message: - stringpath = "/".join(path) - MetadataHandler._set_error(errors, path, msg_template.format(path=stringpath, schema_type=schema['type'])) + stringpath = "/".join(path) if path else "ROOT" + MetadataHandler._set_error(errors, path, msg_template.format(path=stringpath, schema_type=schema["type"])) if schema["type"] == "object": for field, subschema in schema["properties"].items(): From 64bd634228f29a1e597bb5088789974941e92ffb Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 17 Dec 2024 18:09:22 +0100 Subject: [PATCH 73/91] Metadata: add authorization to metadata access --- geonode/metadata/api/views.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index b1c439bc8c3..4fcd1f383d1 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -19,6 +19,9 @@ import logging from dal import autocomplete +from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from rest_framework.authentication import BasicAuthentication, SessionAuthentication +from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.viewsets import ViewSet from rest_framework.decorators import action from rest_framework.response import Response @@ -30,6 +33,7 @@ from django.utils.translation import get_language from django.db.models import Q +from geonode.base.api.permissions import UserHasPerms from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel from geonode.base.utils import remove_country_from_languagecode from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete @@ -41,6 +45,9 @@ class MetadataViewSet(ViewSet): + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] + """ Simple viewset that return the metadata JSON schema """ @@ -70,9 +77,22 @@ def schema(self, request, pk=None): return Response(response) # Get the JSON schema - @action(detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)") + @action( + detail=False, + methods=["get", "put", "patch"], + url_path=r"instance/(?P\d+)", + permission_classes=[ + UserHasPerms( + perms_dict={ + "default": { + "GET": ["base.view_resourcebase"], + "POST": ["change_resourcebase_metadata"], + } + } + ) + ], + ) def schema_instance(self, request, pk=None): - try: resource = ResourceBase.objects.get(pk=pk) @@ -83,7 +103,7 @@ def schema_instance(self, request, pk=None): schema_instance, content_type="application/schema-instance+json", json_dumps_params={"indent": 3} ) - elif request.method in ("PUT", "PATCH"): + elif request.method in ("PUT"): logger.debug(f"handling request {request.method}") # try: # logger.debug(f"handling content {json.dumps(request.data, indent=3)}") From 3172f75b69ba80d82c8a176ba8aeb146d7d3d485 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 17 Dec 2024 18:33:13 +0100 Subject: [PATCH 74/91] Metadata: fix required rolenames --- geonode/metadata/handlers/contact.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 7f5ad0a07eb..720519d54e0 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -55,10 +55,11 @@ def update_schema(self, jsonschema, context, lang=None): contacts = {} required = [] for role in Roles: + rolename = ROLE_NAMES_MAP[role] minitems = 1 if role.is_required else 0 card = f'[{minitems}..{"N" if role.is_multivalue else "1"}]' if role.is_required: - required.append(role.name) + required.append(rolename) if role.is_multivalue: contact = { @@ -99,7 +100,7 @@ def update_schema(self, jsonschema, context, lang=None): "required": ["id"] if role.is_required else [], } - rolename = ROLE_NAMES_MAP[role] + contacts[rolename] = contact jsonschema["properties"]["contacts"] = { From d33db5c5fc18ea4fe9d61f46e17895f23b978c02 Mon Sep 17 00:00:00 2001 From: etj Date: Tue, 17 Dec 2024 18:59:59 +0100 Subject: [PATCH 75/91] Metadata: improve type handling in sparse fields --- geonode/metadata/handlers/sparse.py | 37 +++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 38f115cf133..5c5b77ac9f5 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -96,14 +96,31 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No field_value = context[CONTEXT_ID]["fields"].get(field_name, None) is_nullable = self._check_type(field_type, "null") - is_string = self._check_type(field_type, "string") - is_number = self._check_type(field_type, "number") if field_name not in context[CONTEXT_ID]["fields"] and not is_nullable: raise UnsetFieldException() - if is_string or is_number: + if is_nullable and field_value is None: + return None + + if self._check_type(field_type, "string"): return field_value + elif self._check_type(field_type, "number"): + try: + return float(field_value) + except: + logger.warning( + f"Error loading NUMBER field '{field_name}' with content ({type(field_value)}){field_value}" + ) + raise UnsetFieldException() # should be a different exception + elif self._check_type(field_type, "integer"): + try: + return int(field_value) + except: + logger.warning( + f"Error loading INTEGER field '{field_name}' with content ({type(field_value)}){field_value}" + ) + raise UnsetFieldException() # should be a different exception elif field_type == "array": # assuming it's a single level array: TODO implement other cases try: @@ -136,8 +153,20 @@ def check_type(declared, checked): is_nullable = check_type(field_type, "null") - if check_type(field_type, "string") or check_type(field_type, "number"): + if check_type(field_type, "string"): field_value = bare_value + elif check_type(field_type, "number"): + try: + field_value = str(float(bare_value)) + except ValueError as e: + self._set_error(errors, [field_name], f"Error parsing number '{bare_value}'") + return + elif check_type(field_type, "integer"): + try: + field_value = str(int(bare_value)) + except ValueError as e: + self._set_error(errors, [field_name], f"Error parsing integer '{bare_value}'") + return elif field_type == "array": field_value = json.dumps(bare_value) if bare_value else "[]" elif field_type == "object": From ce36d86ff4f5ddf6d67b41ec7b97515e9d7c1c1c Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 18 Dec 2024 09:19:21 +0200 Subject: [PATCH 76/91] adding tests for views and manager --- geonode/metadata/tests.py | 258 +++++++++++++++++-- geonode/metadata/tests/data/fake_schema.json | 26 ++ 2 files changed, 269 insertions(+), 15 deletions(-) create mode 100644 geonode/metadata/tests/data/fake_schema.json diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests.py index 03fe46b1427..4ea97b03b97 100644 --- a/geonode/metadata/tests.py +++ b/geonode/metadata/tests.py @@ -18,24 +18,22 @@ ######################################################################### import os -import logging -import shutil -import io import json -from unittest.mock import patch +from unittest.mock import patch, MagicMock from uuid import uuid4 from django.urls import reverse from django.utils.module_loading import import_string from django.contrib.auth import get_user_model +from django.test import RequestFactory from rest_framework import status from rest_framework.test import APITestCase from geonode.metadata.settings import MODEL_SCHEMA -from geonode.settings import PROJECT_ROOT from geonode.metadata.manager import metadata_manager from geonode.metadata.settings import METADATA_HANDLERS from geonode.base.models import ResourceBase +from geonode.settings import PROJECT_ROOT class MetadataApiTests(APITestCase): @@ -45,17 +43,28 @@ def setUp(self): self.model_schema = MODEL_SCHEMA self.lang = None - self.context = metadata_manager._init_schema_context(self.lang) + self.test_user_1 = get_user_model().objects.create_user( + "user_1", "user_1@fakemail.com", "user_1_password", is_active=True + ) + self.test_user_2 = get_user_model().objects.create_user( + "user_2", "user_2@fakemail.com", "user_2_password", is_active=True + ) + self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user_1) + self.factory = RequestFactory() + + # Setup of the Manager + with open(os.path.join(PROJECT_ROOT, "metadata/tests/data/fake_schema.json")) as f: + self.fake_schema = json.load(f) - self.handlers = {} - for handler_id, module_path in METADATA_HANDLERS.items(): - handler_cls = import_string(module_path) - self.handlers[handler_id] = handler_cls() + self.handler1 = MagicMock() + self.handler2 = MagicMock() + self.handler3 = MagicMock() - self.test_user = get_user_model().objects.create_user( - "someuser", "someuser@fakemail.com", "somepassword", is_active=True - ) - self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user) + self.fake_handlers = { + "fake_handler1": self.handler1, + "fake_handler2": self.handler2, + "fake_handler3": self.handler3, + } def tearDown(self): super().tearDown() @@ -114,6 +123,9 @@ def test_schema_with_lang(self, mock_get_schema): @patch("geonode.metadata.manager.metadata_manager.build_schema_instance") def test_get_schema_instance_with_default_lang(self, mock_build_schema_instance): + """ + Test schema_instance endpoint with the default lang parameter + """ mock_build_schema_instance.return_value = {"fake_schema_instance": "schema_instance"} @@ -128,6 +140,9 @@ def test_get_schema_instance_with_default_lang(self, mock_build_schema_instance) @patch("geonode.metadata.manager.metadata_manager.build_schema_instance") def test_get_schema_instance_with_lang(self, mock_build_schema_instance): + """ + Test schema_instance endpoint with specific lang parameter + """ mock_build_schema_instance.return_value = {"fake_schema_instance": "schema_instance"} @@ -142,6 +157,9 @@ def test_get_schema_instance_with_lang(self, mock_build_schema_instance): @patch("geonode.metadata.manager.metadata_manager.update_schema_instance") def test_put_patch_schema_instance_with_no_errors(self, mock_update_schema_instance): + """ + Test the success case of PATCH and PUT methods of the schema_instance + """ url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) fake_payload = {"field": "value"} @@ -164,6 +182,9 @@ def test_put_patch_schema_instance_with_no_errors(self, mock_update_schema_insta @patch("geonode.metadata.manager.metadata_manager.update_schema_instance") def test_put_patch_schema_instance_with_errors(self, mock_update_schema_instance): + """ + Test the PATCH and PUT methods of the schema_instance in case of errors + """ url = reverse("metadata-schema_instance", kwargs={"pk": self.resource.pk}) fake_payload = {"field": "value"} @@ -188,6 +209,9 @@ def test_put_patch_schema_instance_with_errors(self, mock_update_schema_instance mock_update_schema_instance.assert_called_with(self.resource, fake_payload) def test_resource_not_found(self): + """ + Test case that the resource does not exist + """ # Define a fake primary key fake_pk = 1000 @@ -203,6 +227,210 @@ def test_resource_not_found(self): response.content, {"message": "The dataset was not found"} ) + + + #TODO tests for autocomplete views + + # Manager tests + + def test_registry_and_add_handler(self): + + self.assertEqual(set(metadata_manager.handlers.keys()), set(METADATA_HANDLERS.keys())) + for handler_id in METADATA_HANDLERS.keys(): + self.assertIn(handler_id, metadata_manager.handlers) + + @patch("geonode.metadata.manager.metadata_manager.root_schema", new_callable=lambda: { + "title": "Test Schema", + "properties": { + "field1": {"type": "string"}, + "field2": {"type": "integer", "geonode:required": True}, + }, + }) + @patch("geonode.metadata.manager.metadata_manager._init_schema_context") + def test_build_schema(self, mock_init_schema_context, mock_root_schema): + + mock_init_schema_context.return_value = {} + + with patch.dict(metadata_manager.handlers, self.fake_handlers, clear=True): + + self.handler1.update_schema.side_effect = lambda schema, context, lang: schema + self.handler2.update_schema.side_effect = lambda schema, context, lang: schema + self.handler3.update_schema.side_effect = lambda schema, context, lang: schema + + # Call build_schema + schema = metadata_manager.build_schema(lang="en") + + self.assertEqual(schema["title"], "Test Schema") + self.assertIn("field1", schema["properties"]) + self.assertIn("field2", schema["properties"]) + self.assertIn("required", schema) + self.assertIn("field2", schema["required"]) + self.handler1.update_schema.assert_called() + self.handler2.update_schema.assert_called() + self.handler3.update_schema.assert_called() + + @patch("geonode.metadata.manager.metadata_manager.build_schema") + @patch("cachetools.FIFOCache.get") + # Mock FIFOCache's __setitem__ method (cache setting) + @patch("cachetools.FIFOCache.__setitem__") + def test_get_schema(self, mock_setitem, mock_get, mock_build_schema): + + lang = "en" + expected_schema = self.fake_schema + + # Case when the schema is already in cache + mock_get.return_value = expected_schema + result = metadata_manager.get_schema(lang) + + # Assert that the schema was retrieved from the cache + mock_get.assert_called_once_with(str(lang), None) + mock_build_schema.assert_not_called() + self.assertEqual(result, expected_schema) + + # Reset mock calls to test the second case + mock_get.reset_mock() + mock_build_schema.reset_mock() + mock_setitem.reset_mock() + + # Case when the schema is not in cache + mock_get.return_value = None + mock_build_schema.return_value = expected_schema + result = metadata_manager.get_schema(lang) + + mock_get.assert_called_once_with(str(lang), None) + mock_build_schema.assert_called_once_with(lang) + mock_setitem.assert_called_once_with(str(lang), expected_schema) + self.assertEqual(result, expected_schema) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_build_schema_instance_no_errors(self, mock_get_schema): + + self.lang = "en" + mock_get_schema.return_value = self.fake_schema + + with patch.dict(metadata_manager.handlers, self.fake_handlers, clear=True): + + self.handler1.get_jsonschema_instance.return_value = {"data from fake handler 1"} + self.handler2.get_jsonschema_instance.return_value = {"data from fake handler 2"} + self.handler3.get_jsonschema_instance.return_value = {"data from fake handler 3"} + + # Call the method + instance = metadata_manager.build_schema_instance(self.resource, self.lang) + + # Assert that the handlers were called and instance was built correctly + self.handler1.get_jsonschema_instance.assert_called_once_with(self.resource, "field1", {}, {}, self.lang) + self.handler2.get_jsonschema_instance.assert_called_once_with(self.resource, "field2", {}, {}, self.lang) + self.handler3.get_jsonschema_instance.assert_called_once_with(self.resource, "field3", {}, {}, self.lang) + + self.assertEqual(instance["field1"], {"data from fake handler 1"}) + self.assertEqual(instance["field2"], {"data from fake handler 2"}) + self.assertEqual(instance["field3"], {"data from fake handler 3"}) + self.assertNotIn("extraErrors", instance) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_update_schema_instance_no_errors(self, mock_get_schema): + + # json_instance is the payload from the client. + # In this test is used only to call the update_schema_instance + json_instance = {"field1": "new_value1", "new_field2": "new_value2"} + + mock_get_schema.return_value = self.fake_schema + # Mock the save method + self.resource.save = MagicMock() + + with patch.dict(metadata_manager.handlers, self.fake_handlers, clear=True): + + # Simulate successful handler behavior + self.handler1.update_resource.return_value = None + self.handler2.update_resource.return_value = None + self.handler3.update_resource.return_value = None + + # Call the update_schema_instance method + errors = metadata_manager.update_schema_instance(self.resource, json_instance) + + # Assert that handlers were called to update the resource with the correct data + self.handler1.update_resource.assert_called_once_with(self.resource, "field1", json_instance, {}, {}) + self.handler2.update_resource.assert_called_once_with(self.resource, "field2", json_instance, {}, {}) + self.handler3.update_resource.assert_called_once_with(self.resource, "field3", json_instance, {}, {}) + + # Assert no errors were raised + self.assertEqual(errors, {}) + + # Check that resource.save() is called + self.resource.save.assert_called_once() + + # Assert that there were no extra errors in the response + self.assertNotIn("extraErrors", errors) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_update_schema_instance_with_handler_error(self, mock_get_schema): + + # json_instance is the payload from the client. + # In this test is used only to call the update_schema_instance + json_instance = {} + + mock_get_schema.return_value = self.fake_schema + + # Mock the save method + self.resource.save = MagicMock() + + with patch.dict(metadata_manager.handlers, self.fake_handlers, clear=True): + + # Simulate an error in update_resource for handler2 + self.handler1.update_resource.side_effect = None + self.handler2.update_resource.side_effect = Exception("Error in handler2") + self.handler3.update_resource.side_effect = None + + # Call the method under test + errors = metadata_manager.update_schema_instance(self.resource, json_instance) + + # Assert that update_resource was called for each handler + self.handler1.update_resource.assert_called() + self.handler2.update_resource.assert_called() + self.handler3.update_resource.assert_called() + + # Assert that resource.save() was called + self.resource.save.assert_called_once() + + # Verify that errors are collected for handler2 + self.assertIn("field2", errors) + self.assertEqual( + errors["field2"]["__errors"], + ["Error while processing this field: Error in handler2"] + ) + + # Verify that no other errors were added for handler1 and handler3 + self.assertNotIn("field1", errors) + self.assertNotIn("field3", errors) + + @patch("geonode.metadata.manager.metadata_manager.get_schema") + def test_update_schema_instance_with_db_error(self, mock_get_schema): + + # json_instance is the payload from the client. + # In this test is used only to call the update_schema_instance + json_instance = {} + + mock_get_schema.return_value = self.fake_schema + + # Mock save method with an exception + self.resource.save = MagicMock(side_effect=Exception("Error during the resource save")) + + with patch.dict(metadata_manager.handlers, self.fake_handlers, clear=True): + + self.handler1.update_resource.side_effect = None + self.handler2.update_resource.side_effect = None + self.handler3.update_resource.side_effect = None + + # Call the method under test + errors = metadata_manager.update_schema_instance(self.resource, json_instance) + + # Assert that update_resource was called for each handler + self.handler1.update_resource.assert_called() + self.handler2.update_resource.assert_called() + self.handler3.update_resource.assert_called() - \ No newline at end of file + # Assert that save raised an error and was recorded + self.resource.save.assert_called_once() + self.assertIn("__errors", errors) + self.assertEqual(errors["__errors"], ["Error while saving the resource: Error during the resource save"]) \ No newline at end of file diff --git a/geonode/metadata/tests/data/fake_schema.json b/geonode/metadata/tests/data/fake_schema.json new file mode 100644 index 00000000000..d68cf662b95 --- /dev/null +++ b/geonode/metadata/tests/data/fake_schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "{GEONODE_SITE}/resource.json", + "title": "GeoNode resource", + "type": "object", + "properties": { + "field1": { + "type": "string", + "title": "fake_handler1", + "maxLength": 255, + "geonode:handler": "fake_handler1" + }, + "field2": { + "type": "string", + "title": "fake_handler2", + "maxLength": 255, + "geonode:handler": "fake_handler2" + }, + "field3": { + "type": "string", + "title": "fake_handler3", + "maxLength": 255, + "geonode:handler": "fake_handler3" + } + } + } \ No newline at end of file From 8160a909decb064e277ebb3a50c68f4bfdfdc7c9 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 18 Dec 2024 11:05:17 +0100 Subject: [PATCH 77/91] Metadata: improve handling of None values in sparse fields --- geonode/metadata/handlers/sparse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 5c5b77ac9f5..1e66907045e 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -157,20 +157,20 @@ def check_type(declared, checked): field_value = bare_value elif check_type(field_type, "number"): try: - field_value = str(float(bare_value)) + field_value = str(float(bare_value)) if bare_value is not None else None except ValueError as e: self._set_error(errors, [field_name], f"Error parsing number '{bare_value}'") return elif check_type(field_type, "integer"): try: - field_value = str(int(bare_value)) + field_value = str(int(bare_value)) if bare_value is not None else None except ValueError as e: self._set_error(errors, [field_name], f"Error parsing integer '{bare_value}'") return elif field_type == "array": - field_value = json.dumps(bare_value) if bare_value else "[]" + field_value = json.dumps(bare_value) if bare_value is not None else "[]" elif field_type == "object": - field_value = json.dumps(bare_value) if bare_value else "{}" + field_value = json.dumps(bare_value) if bare_value is not None else "{}" else: logger.warning(f"Unhandled type '{field_type}' for sparse field '{field_name}'") self._set_error(errors, [field_name], f"Unhandled type {field_type}. Contact your administrator") From 73c48954fdb2e930711b7fa99f039be32a77548b Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 18 Dec 2024 13:48:22 +0100 Subject: [PATCH 78/91] Metadata: tentative handling of categories via autocomplete --- geonode/metadata/api/urls.py | 5 +++++ geonode/metadata/api/views.py | 22 ++++++++++++++++++++-- geonode/metadata/handlers/abstract.py | 2 +- geonode/metadata/handlers/base.py | 16 ++++++++++------ geonode/metadata/schemas/base.json | 10 +++++----- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 0b806f4efd4..8782cefd6f1 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -58,5 +58,10 @@ MetadataGroupAutocomplete.as_view(), name="metadata_autocomplete_groups", ), + path( + r"metadata/autocomplete/categories", + views.categories_autocomplete, + name="metadata_autocomplete_categories", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 4fcd1f383d1..063707a1657 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -30,11 +30,11 @@ from django.core.handlers.wsgi import WSGIRequest from django.http import JsonResponse from django.utils.translation.trans_real import get_language_from_request -from django.utils.translation import get_language +from django.utils.translation import get_language, gettext as _ from django.db.models import Q from geonode.base.api.permissions import UserHasPerms -from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel, TopicCategory from geonode.base.utils import remove_country_from_languagecode from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete from geonode.groups.models import GroupProfile @@ -166,6 +166,24 @@ def tkeywords_autocomplete(request: WSGIRequest, thesaurusid): return JsonResponse({"results": ret}) +def categories_autocomplete(request: WSGIRequest): + qs = TopicCategory.objects.order_by("gn_description") + + if q := request.GET.get("q", None): + qs = qs.filter(gn_description__istartswith=q) + + ret = [] + for record in qs.all(): + ret.append( + { + "id": record.identifier, + "label": _(record.gn_description), + } + ) + + return JsonResponse({"results": ret}) + + class ProfileAutocomplete(autocomplete.Select2QuerySetView): def get_queryset(self): if self.request and self.request.user: diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 4f5b9b33bba..13d752a8905 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -114,7 +114,7 @@ def _add_after(self, jsonschema, after_what, property_name, subschema): @staticmethod def _set_error(errors: dict, path: list, msg: str): - logger.warning(f"Reported message: {'/'.join(path)}: {msg} ") + logger.info(f"Reported message: {'/'.join(path)}: {msg} ") elem = errors for step in path: elem = elem.setdefault(step, {}) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 12bdc9ea4f8..8289c84e093 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -21,11 +21,13 @@ import logging from datetime import datetime +from rest_framework.reverse import reverse +from django.utils.translation import gettext as _ + from geonode.base.models import TopicCategory, License, RestrictionCodeType, SpatialRepresentationType from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES -from django.utils.translation import gettext as _ logger = logging.getLogger(__name__) @@ -48,11 +50,13 @@ def deserialize(cls, field_value): class CategorySubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - # subschema["title"] = _("topiccategory") - subschema["oneOf"] = [ - {"const": tc.identifier, "title": _(tc.gn_description), "description": _(tc.description)} - for tc in TopicCategory.objects.order_by("gn_description") - ] + subschema["ui:options"] = { + "geonode-ui:autocomplete": reverse("metadata_autocomplete_categories"), + } + # subschema["oneOf"] = [ + # {"const": tc.identifier, "title": _(tc.gn_description), "description": _(tc.description)} + # for tc in TopicCategory.objects.order_by("gn_description") + # ] @classmethod def serialize(cls, db_value): diff --git a/geonode/metadata/schemas/base.json b/geonode/metadata/schemas/base.json index 354cc5c4608..318918c740b 100644 --- a/geonode/metadata/schemas/base.json +++ b/geonode/metadata/schemas/base.json @@ -39,21 +39,21 @@ "type": "string", "title": "Category", "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.", - "maxLength": 255 + "maxLength": 255, + "geonode:required": true }, "language": { "type": "string", "title": "language", "description": "language used within the dataset", - "maxLength": 255, - "default": "eng" + "maxLength": 16 }, "license": { - "type": ["string", "null"], + "type": "string", "title": "License", "description": "license of the dataset", "maxLength": 255, - "default": "eng" + "geonode:required": true }, "attribution": { "type": ["string", "null"], From 5ad930ab66e5c5f70e421c80dbb5724b666e2fd4 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 18 Dec 2024 16:59:05 +0100 Subject: [PATCH 79/91] Metadata: tentative handling of categories via autocomplete --- geonode/metadata/handlers/abstract.py | 10 +++++++++- geonode/metadata/handlers/base.py | 24 +++++++++++++++--------- geonode/metadata/schemas/base.json | 18 +++++++++++++----- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/geonode/metadata/handlers/abstract.py b/geonode/metadata/handlers/abstract.py index 13d752a8905..637de6d2a74 100644 --- a/geonode/metadata/handlers/abstract.py +++ b/geonode/metadata/handlers/abstract.py @@ -137,5 +137,13 @@ def _localize_subschema_label(context, subschema: dict, lang: str, annotation_na subschema[annotation_name] = MetadataHandler._localize_label(context, lang, subschema[annotation_name]) -class UnsetFieldException(Exception): +class MetadataFieldException(Exception): + pass + + +class UnsetFieldException(MetadataFieldException): + pass + + +class UnparsableFieldException(MetadataFieldException): pass diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 8289c84e093..442d4565a33 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -25,7 +25,7 @@ from django.utils.translation import gettext as _ from geonode.base.models import TopicCategory, License, RestrictionCodeType, SpatialRepresentationType -from geonode.metadata.handlers.abstract import MetadataHandler +from geonode.metadata.handlers.abstract import MetadataHandler, UnparsableFieldException from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES @@ -53,20 +53,24 @@ def update_subschema(cls, subschema, lang=None): subschema["ui:options"] = { "geonode-ui:autocomplete": reverse("metadata_autocomplete_categories"), } - # subschema["oneOf"] = [ - # {"const": tc.identifier, "title": _(tc.gn_description), "description": _(tc.description)} - # for tc in TopicCategory.objects.order_by("gn_description") - # ] @classmethod def serialize(cls, db_value): - if isinstance(db_value, TopicCategory): - return db_value.identifier - return db_value + if db_value is None: + return None + elif isinstance(db_value, TopicCategory): + return {"id": db_value.identifier, "label": _(db_value.gn_description)} + else: + logger.warning(f"Category: can't decode <{type(db_value)}>'{db_value}'") + return None @classmethod def deserialize(cls, field_value): - return TopicCategory.objects.get(identifier=field_value) + if field_value: + obj = json.loads(field_value) + return TopicCategory.objects.get(identifier=obj["id"]) + else: + return None class DateTypeSubHandler(SubHandler): @@ -219,3 +223,5 @@ def update_resource(self, resource, field_name, json_instance, context, errors, setattr(resource, field_name, field_value) except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") + self._set_error(errors, [field_name], f"Error while storing field. Contact your administrator") + diff --git a/geonode/metadata/schemas/base.json b/geonode/metadata/schemas/base.json index 318918c740b..368fb363312 100644 --- a/geonode/metadata/schemas/base.json +++ b/geonode/metadata/schemas/base.json @@ -36,11 +36,19 @@ "maxLength": 255 }, "category": { - "type": "string", - "title": "Category", - "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.", - "maxLength": 255, - "geonode:required": true + "type": "object", + "title": "Category", + "description": "high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "required": ["id"], + "geonode:required": true }, "language": { "type": "string", From 0eb085304efef9a4f3c4f2151e4178fbfa16fb25 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 18 Dec 2024 18:09:22 +0200 Subject: [PATCH 80/91] adding base handlers tests --- .../metadata/tests/data/fake_base_schema.json | 32 ++++ geonode/metadata/tests/test_handlers.py | 142 ++++++++++++++++++ geonode/metadata/{ => tests}/tests.py | 0 geonode/metadata/tests/utils.py | 20 +++ 4 files changed, 194 insertions(+) create mode 100644 geonode/metadata/tests/data/fake_base_schema.json create mode 100644 geonode/metadata/tests/test_handlers.py rename geonode/metadata/{ => tests}/tests.py (100%) create mode 100644 geonode/metadata/tests/utils.py diff --git a/geonode/metadata/tests/data/fake_base_schema.json b/geonode/metadata/tests/data/fake_base_schema.json new file mode 100644 index 00000000000..8ecff0a1d74 --- /dev/null +++ b/geonode/metadata/tests/data/fake_base_schema.json @@ -0,0 +1,32 @@ +{ + "uuid": { + "type": "string", + "title": "UUID", + "maxLength": 36, + "readOnly": true, + "NO_ui:widget": "hidden", + "geonode:handler": "base" + }, + "title": { + "type": "string", + "title": "title", + "description": "name by which the cited resource is known", + "maxLength": 255, + "geonode:handler": "base" + }, + "abstract": { + "type": "string", + "title": "abstract", + "description": "brief narrative summary of the content of the resource(s)", + "maxLength": 2000, + "ui:options": { + "widget": "textarea", + "rows": 5 + } + }, + "date": { + "type": "string", + "format": "date-time", + "title": "Date" + } +} \ No newline at end of file diff --git a/geonode/metadata/tests/test_handlers.py b/geonode/metadata/tests/test_handlers.py new file mode 100644 index 00000000000..8cf1e601325 --- /dev/null +++ b/geonode/metadata/tests/test_handlers.py @@ -0,0 +1,142 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import json +from unittest.mock import patch, MagicMock +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.test import RequestFactory + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.metadata.settings import MODEL_SCHEMA +from geonode.base.models import ResourceBase +from geonode.settings import PROJECT_ROOT +from geonode.metadata.handlers.base import BaseHandler +from geonode.metadata.tests.utils import MockSubHandler + +class HandlersTests(GeoNodeBaseTestSupport): + + def setUp(self): + # set Json schemas + self.model_schema = MODEL_SCHEMA + self.lang = None + self.errors = {} + self.context = MagicMock() + + self.test_user_1 = get_user_model().objects.create_user( + "user_1", "user_1@fakemail.com", "user_1_password", is_active=True + ) + self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user_1) + self.factory = RequestFactory() + + # Fake base schema path + self.fake_base_schema_path = os.path.join(PROJECT_ROOT, "metadata/tests/data/fake_base_schema.json") + + # Load fake base schema + with open(self.fake_base_schema_path) as f: + self.fake_base_schema = json.load(f) + + # Load a mocked schema + # Setup of the Manager + with open(os.path.join(PROJECT_ROOT, "metadata/tests/data/fake_schema.json")) as f: + self.fake_schema = json.load(f) + + self.fake_subhandlers = { + "date": MockSubHandler, + "date_type": MockSubHandler, + "category": MockSubHandler + } + + # Handlers + self.base_handler = BaseHandler() + + def tearDown(self): + super().tearDown() + + # Tests for the Base handler + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + def test_base_handler_update_schema(self, mock_subhandlers): + + # Use of mock_subhandlers as a dictionary + mock_subhandlers.update(self.fake_subhandlers) + + # Only the path is defined since it is loaded inside the base handler + self.base_handler.json_base_schema = self.fake_base_schema_path + + # Input schema and context + jsonschema = self.model_schema + + # Call the method + updated_schema = self.base_handler.update_schema(jsonschema, self.context, self.lang) + + # Check the full base schema + for field in self.fake_base_schema: + self.assertIn(field, updated_schema["properties"]) + + # Check subhandler execution + self.assertEqual(updated_schema["properties"]["date"].get("oneOf"), [{"const": "fake const", "title": "fake title"}]) + self.assertEqual(updated_schema["properties"]["date_type"].get("oneOf"), [{"const": "fake const", "title": "fake title"}]) + self.assertNotIn("oneOf", updated_schema["properties"]["uuid"]) + self.assertNotIn("oneOf", updated_schema["properties"]["title"]) + self.assertNotIn("oneOf", updated_schema["properties"]["abstract"]) + + # Check geonode:handler addition + self.assertEqual(updated_schema["properties"]["date"].get("geonode:handler"), "base") + self.assertEqual(updated_schema["properties"]["date_type"].get("geonode:handler"), "base") + + def test_base_handler_get_jsonschema_instance_without_subhandlers(self): + + fieldname = "title" + self.assertTrue(hasattr(self.resource, fieldname), f"Field '{fieldname}' does not exist.") + expected_field_value = self.resource.title + + # Call the method + field_value = self.base_handler.get_jsonschema_instance(self.resource, fieldname, self.context, self.errors, lang=None) + self.assertEqual(expected_field_value, field_value) + self.assertEqual(self.errors, {}) + + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + def test_base_handler_get_jsonschema_instance_with_subhandlers(self, mock_subhandlers): + + field_name = "category" # A field name which is included in the SUBHANDLERS + + # Create a fake resource + fake_resource = MagicMock() + fake_resource.category = MagicMock() + fake_resource.category.identifier = "mocked_category_value" + expected_field_value = fake_resource.category.identifier + + # Use of mock_subhandlers as a dictionary + mock_subhandlers.update(self.fake_subhandlers) + + # Call the method + field_value = self.base_handler.get_jsonschema_instance( + resource=fake_resource, + field_name=field_name, + context=self.context, + errors=self.errors, + lang=self.lang + ) + + self.assertEqual(expected_field_value, field_value) + + + \ No newline at end of file diff --git a/geonode/metadata/tests.py b/geonode/metadata/tests/tests.py similarity index 100% rename from geonode/metadata/tests.py rename to geonode/metadata/tests/tests.py diff --git a/geonode/metadata/tests/utils.py b/geonode/metadata/tests/utils.py new file mode 100644 index 00000000000..d2e2cd6c206 --- /dev/null +++ b/geonode/metadata/tests/utils.py @@ -0,0 +1,20 @@ +from unittest.mock import MagicMock + +# Mock subhandler class +class MockSubHandler: + + @classmethod + def update_subschema(cls, subschema, lang=None): + subschema["oneOf"] = [ + {"const": "fake const", "title": "fake title"} + ] + + @classmethod + def serialize(cls, db_value): + if isinstance(db_value, MagicMock): + return db_value.identifier + return db_value + + @classmethod + def deserialize(cls, field_value): + return MagicMock.objects.get(identifier=field_value) \ No newline at end of file From e3defc2bb0a987c61e0440588ff65f718dc615e6 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 18 Dec 2024 18:17:56 +0100 Subject: [PATCH 81/91] Metadata: tentative handling of categories via autocomplete --- geonode/metadata/handlers/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 442d4565a33..a8bd832393d 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -66,11 +66,7 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - if field_value: - obj = json.loads(field_value) - return TopicCategory.objects.get(identifier=obj["id"]) - else: - return None + return TopicCategory.objects.get(identifier=field_value["id"]) if field_value else None class DateTypeSubHandler(SubHandler): From f69dddef233d28f2b741d0d86d47a09dcd688d5f Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 19 Dec 2024 10:46:16 +0100 Subject: [PATCH 82/91] Metadata: handling licenses via autocomplete --- geonode/metadata/api/urls.py | 5 +++++ geonode/metadata/api/views.py | 20 +++++++++++++++++++- geonode/metadata/handlers/base.py | 19 +++++++++++-------- geonode/metadata/schemas/base.json | 13 +++++++++++-- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/geonode/metadata/api/urls.py b/geonode/metadata/api/urls.py index 8782cefd6f1..39795f568ff 100644 --- a/geonode/metadata/api/urls.py +++ b/geonode/metadata/api/urls.py @@ -63,5 +63,10 @@ views.categories_autocomplete, name="metadata_autocomplete_categories", ), + path( + r"metadata/autocomplete/licenses", + views.licenses_autocomplete, + name="metadata_autocomplete_licenses", + ), # path(r"metadata/autocomplete/users", login_required(ProfileAutocomplete.as_view()), name="metadata_autocomplete_users"), ] diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index 063707a1657..8fca7094595 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -34,7 +34,7 @@ from django.db.models import Q from geonode.base.api.permissions import UserHasPerms -from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel, TopicCategory +from geonode.base.models import ResourceBase, ThesaurusKeyword, ThesaurusKeywordLabel, TopicCategory, License from geonode.base.utils import remove_country_from_languagecode from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete from geonode.groups.models import GroupProfile @@ -184,6 +184,24 @@ def categories_autocomplete(request: WSGIRequest): return JsonResponse({"results": ret}) +def licenses_autocomplete(request: WSGIRequest): + qs = License.objects.order_by("name") + + if q := request.GET.get("q", None): + qs = qs.filter(name__istartswith=q) + + ret = [] + for record in qs.all(): + ret.append( + { + "id": record.identifier, + "label": _(record.name), + } + ) + + return JsonResponse({"results": ret}) + + class ProfileAutocomplete(autocomplete.Select2QuerySetView): def get_queryset(self): if self.request and self.request.user: diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index a8bd832393d..31e82de0dda 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -99,20 +99,23 @@ def update_subschema(cls, subschema, lang=None): class LicenseSubHandler(SubHandler): @classmethod def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [ - {"const": tc.identifier, "title": tc.name, "description": tc.description} - for tc in License.objects.order_by("name") - ] + subschema["ui:options"] = { + "geonode-ui:autocomplete": reverse("metadata_autocomplete_licenses"), + } @classmethod def serialize(cls, db_value): - if isinstance(db_value, License): - return db_value.identifier - return db_value + if db_value is None: + return None + elif isinstance(db_value, License): + return {"id": db_value.identifier, "label": _(db_value.name)} + else: + logger.warning(f"License: can't decode <{type(db_value)}>'{db_value}'") + return None @classmethod def deserialize(cls, field_value): - return License.objects.get(identifier=field_value) + return License.objects.get(identifier=field_value["id"]) if field_value else None class RestrictionsSubHandler(SubHandler): diff --git a/geonode/metadata/schemas/base.json b/geonode/metadata/schemas/base.json index 368fb363312..7cc2b311313 100644 --- a/geonode/metadata/schemas/base.json +++ b/geonode/metadata/schemas/base.json @@ -57,11 +57,20 @@ "maxLength": 16 }, "license": { - "type": "string", + "type": "object", "title": "License", "description": "license of the dataset", "maxLength": 255, - "geonode:required": true + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "required": ["id"], + "geonode:required": true }, "attribution": { "type": ["string", "null"], From 545a39374aff0d163ed3d8b155329b598befba1c Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 19 Dec 2024 12:42:11 +0100 Subject: [PATCH 83/91] Black/flake --- geonode/metadata/handlers/base.py | 5 ++--- geonode/metadata/handlers/contact.py | 1 - geonode/metadata/handlers/sparse.py | 10 ++++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 31e82de0dda..1b058cb3d55 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -25,7 +25,7 @@ from django.utils.translation import gettext as _ from geonode.base.models import TopicCategory, License, RestrictionCodeType, SpatialRepresentationType -from geonode.metadata.handlers.abstract import MetadataHandler, UnparsableFieldException +from geonode.metadata.handlers.abstract import MetadataHandler from geonode.metadata.settings import JSONSCHEMA_BASE from geonode.base.enumerations import ALL_LANGUAGES, UPDATE_FREQUENCIES @@ -222,5 +222,4 @@ def update_resource(self, resource, field_name, json_instance, context, errors, setattr(resource, field_name, field_value) except Exception as e: logger.warning(f"Error setting field {field_name}={field_value}: {e}") - self._set_error(errors, [field_name], f"Error while storing field. Contact your administrator") - + self._set_error(errors, [field_name], "Error while storing field. Contact your administrator") diff --git a/geonode/metadata/handlers/contact.py b/geonode/metadata/handlers/contact.py index 720519d54e0..5c545b74158 100644 --- a/geonode/metadata/handlers/contact.py +++ b/geonode/metadata/handlers/contact.py @@ -100,7 +100,6 @@ def update_schema(self, jsonschema, context, lang=None): "required": ["id"] if role.is_required else [], } - contacts[rolename] = contact jsonschema["properties"]["contacts"] = { diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index 1e66907045e..f553bf1a9c1 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -108,17 +108,17 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No elif self._check_type(field_type, "number"): try: return float(field_value) - except: + except Exception as e: logger.warning( - f"Error loading NUMBER field '{field_name}' with content ({type(field_value)}){field_value}" + f"Error loading NUMBER field '{field_name}' with content ({type(field_value)}){field_value}: {e}" ) raise UnsetFieldException() # should be a different exception elif self._check_type(field_type, "integer"): try: return int(field_value) - except: + except Exception as e: logger.warning( - f"Error loading INTEGER field '{field_name}' with content ({type(field_value)}){field_value}" + f"Error loading INTEGER field '{field_name}' with content ({type(field_value)}){field_value}: {e}" ) raise UnsetFieldException() # should be a different exception elif field_type == "array": @@ -159,12 +159,14 @@ def check_type(declared, checked): try: field_value = str(float(bare_value)) if bare_value is not None else None except ValueError as e: + logger.warning(f"Error parsing sparse field '{field_name}'::'{field_type}'='{bare_value}': {e}") self._set_error(errors, [field_name], f"Error parsing number '{bare_value}'") return elif check_type(field_type, "integer"): try: field_value = str(int(bare_value)) if bare_value is not None else None except ValueError as e: + logger.warning(f"Error parsing sparse field '{field_name}'::'{field_type}'='{bare_value}': {e}") self._set_error(errors, [field_name], f"Error parsing integer '{bare_value}'") return elif field_type == "array": From bd8d860eaabdbdcf415509595722217005b10265 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 19 Dec 2024 15:38:00 +0100 Subject: [PATCH 84/91] Fix i18n caching --- geonode/metadata/handlers/sparse.py | 3 ++- geonode/metadata/i18n.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/geonode/metadata/handlers/sparse.py b/geonode/metadata/handlers/sparse.py index f553bf1a9c1..4ae1dfd47d0 100644 --- a/geonode/metadata/handlers/sparse.py +++ b/geonode/metadata/handlers/sparse.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import copy import json import logging @@ -63,7 +64,7 @@ def _recurse_localization(self, context, schema, lang): def update_schema(self, jsonschema, context, lang=None): # add all registered fields for field_name, field_info in sparse_field_registry.fields().items(): - subschema = field_info["schema"] + subschema = copy.deepcopy(field_info["schema"]) self._recurse_localization(context, subschema, lang) self._add_subschema(jsonschema, field_name, subschema, after_what=field_info["after"]) diff --git a/geonode/metadata/i18n.py b/geonode/metadata/i18n.py index 2658fa5e9e3..fac3fe970fd 100644 --- a/geonode/metadata/i18n.py +++ b/geonode/metadata/i18n.py @@ -8,7 +8,7 @@ def get_localized_tkeywords(lang, thesaurus_identifier: str): - logger.debug("Loading localized tkeyword from DB") + logger.debug(f"Loading localized tkeyword from DB lang:{lang}") query = ( "select " From 352562d4bab8cc5e5b2ebc7af55d4753651eb001 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 19 Dec 2024 17:37:43 +0200 Subject: [PATCH 85/91] adding more tests for the BaseHandler --- geonode/metadata/tests/test_handlers.py | 621 ++++++++++++++++++++++-- geonode/metadata/tests/utils.py | 20 - 2 files changed, 594 insertions(+), 47 deletions(-) delete mode 100644 geonode/metadata/tests/utils.py diff --git a/geonode/metadata/tests/test_handlers.py b/geonode/metadata/tests/test_handlers.py index 8cf1e601325..b290684fad0 100644 --- a/geonode/metadata/tests/test_handlers.py +++ b/geonode/metadata/tests/test_handlers.py @@ -21,18 +21,35 @@ import json from unittest.mock import patch, MagicMock from uuid import uuid4 +from datetime import datetime from django.contrib.auth import get_user_model from django.test import RequestFactory +from django.utils.translation import gettext as _ -from geonode.tests.base import GeoNodeBaseTestSupport +from django.test.testcases import TestCase from geonode.metadata.settings import MODEL_SCHEMA -from geonode.base.models import ResourceBase +from geonode.base.models import ( + ResourceBase, + TopicCategory, + RestrictionCodeType, + License, + SpatialRepresentationType +) from geonode.settings import PROJECT_ROOT -from geonode.metadata.handlers.base import BaseHandler -from geonode.metadata.tests.utils import MockSubHandler +from geonode.metadata.handlers.base import ( + BaseHandler, + CategorySubHandler, + DateTypeSubHandler, + DateSubHandler, + FrequencySubHandler, + LanguageSubHandler, + LicenseSubHandler, + RestrictionsSubHandler, + SpatialRepresentationTypeSubHandler, +) -class HandlersTests(GeoNodeBaseTestSupport): +class HandlersTests(TestCase): def setUp(self): # set Json schemas @@ -44,7 +61,16 @@ def setUp(self): self.test_user_1 = get_user_model().objects.create_user( "user_1", "user_1@fakemail.com", "user_1_password", is_active=True ) + + # Testing database setup self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user_1) + + # Create two instances for the TopicCategory model + self.category = TopicCategory.objects.create(identifier="fake_category", gn_description="a fake gn description", description="a detailed description") + self.license = License.objects.create(identifier="fake_license", name="a fake name", description="a detailed description") + self.restrictions = RestrictionCodeType.objects.create(identifier="fake_restrictions", description="a detailed description") + self.spatial_repr = SpatialRepresentationType.objects.create(identifier="fake_spatial_repr", description="a detailed description") + self.factory = RequestFactory() # Fake base schema path @@ -59,12 +85,6 @@ def setUp(self): with open(os.path.join(PROJECT_ROOT, "metadata/tests/data/fake_schema.json")) as f: self.fake_schema = json.load(f) - self.fake_subhandlers = { - "date": MockSubHandler, - "date_type": MockSubHandler, - "category": MockSubHandler - } - # Handlers self.base_handler = BaseHandler() @@ -74,16 +94,31 @@ def tearDown(self): # Tests for the Base handler @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) def test_base_handler_update_schema(self, mock_subhandlers): - - # Use of mock_subhandlers as a dictionary - mock_subhandlers.update(self.fake_subhandlers) - # Only the path is defined since it is loaded inside the base handler + """ + Ensure that the update_schema method gets a simple valid schema and + populate it with the base_schema properties accordignly + """ + self.base_handler.json_base_schema = self.fake_base_schema_path - # Input schema and context + # Model schema definition jsonschema = self.model_schema + # Mock subhandlers and update_subschema functionality, which + # will be used for the field "date" + field_name = "date" + mock_subhandlers[field_name] = MagicMock() + mock_subhandlers[field_name].update_subschema = MagicMock() + + def mock_update_subschema(subschema, lang=None): + subschema["oneOf"] = [ + {"const": "fake const", "title": "fake title"} + ] + + # Add the mock behavior for update_subschema + mock_subhandlers[field_name].update_subschema.side_effect = mock_update_subschema + # Call the method updated_schema = self.base_handler.update_schema(jsonschema, self.context, self.lang) @@ -91,41 +126,58 @@ def test_base_handler_update_schema(self, mock_subhandlers): for field in self.fake_base_schema: self.assertIn(field, updated_schema["properties"]) - # Check subhandler execution + # Check subhandler execution for the field name "date" self.assertEqual(updated_schema["properties"]["date"].get("oneOf"), [{"const": "fake const", "title": "fake title"}]) - self.assertEqual(updated_schema["properties"]["date_type"].get("oneOf"), [{"const": "fake const", "title": "fake title"}]) self.assertNotIn("oneOf", updated_schema["properties"]["uuid"]) self.assertNotIn("oneOf", updated_schema["properties"]["title"]) self.assertNotIn("oneOf", updated_schema["properties"]["abstract"]) # Check geonode:handler addition + self.assertEqual(updated_schema["properties"]["abstract"].get("geonode:handler"), "base") self.assertEqual(updated_schema["properties"]["date"].get("geonode:handler"), "base") - self.assertEqual(updated_schema["properties"]["date_type"].get("geonode:handler"), "base") - def test_base_handler_get_jsonschema_instance_without_subhandlers(self): + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + def test_base_handler_get_jsonschema_instance_without_subhandlers(self, mock_subhandlers): + + """ + Ensure that the get_json_schema_instance will get the db value + from a simple field + """ - fieldname = "title" - self.assertTrue(hasattr(self.resource, fieldname), f"Field '{fieldname}' does not exist.") + field_name = "title" + self.assertTrue(hasattr(self.resource, field_name), f"Field '{field_name}' does not exist.") expected_field_value = self.resource.title # Call the method - field_value = self.base_handler.get_jsonschema_instance(self.resource, fieldname, self.context, self.errors, lang=None) + field_value = self.base_handler.get_jsonschema_instance(self.resource, field_name, self.context, self.errors, lang=None) + + # Ensure that the serialize method was not called + mock_subhandlers.get(field_name, MagicMock()).serialize.assert_not_called() self.assertEqual(expected_field_value, field_value) self.assertEqual(self.errors, {}) @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) def test_base_handler_get_jsonschema_instance_with_subhandlers(self, mock_subhandlers): - field_name = "category" # A field name which is included in the SUBHANDLERS + """ + Ensure that when a field name corresponds to a model in th ResourceBase, + the get_jsonschema_instance method gets a field_value which is a model + and assign it to the corresponding value. For testing we use the "category + field" + """ + + field_name = "category" # Create a fake resource fake_resource = MagicMock() + fake_resource.category = MagicMock() fake_resource.category.identifier = "mocked_category_value" - expected_field_value = fake_resource.category.identifier + expected_field_value = fake_resource.category.identifier - # Use of mock_subhandlers as a dictionary - mock_subhandlers.update(self.fake_subhandlers) + # Add a SUBHANDLER for the field that returns the MagicMock model + mock_subhandlers[field_name] = MagicMock() + mock_subhandlers[field_name].serialize.return_value = expected_field_value # Call the method field_value = self.base_handler.get_jsonschema_instance( @@ -136,7 +188,522 @@ def test_base_handler_get_jsonschema_instance_with_subhandlers(self, mock_subhan lang=self.lang ) + # Ensure that the serialize method has been called once + mock_subhandlers[field_name].serialize.assert_called_once_with(fake_resource.category) self.assertEqual(expected_field_value, field_value) + + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + def test_update_resource_success_without_subhandlers(self, mock_subhandlers): + + """ + Ensure that when a simple field name like title is set to the resource + without calling the SUBHANDLERS classes + """ + field_name = "title" + expected_field_value = "new_fake_title_value" + json_instance = {field_name: expected_field_value} + + # Call the method + self.base_handler.update_resource( + resource=self.resource, + field_name=field_name, + json_instance=json_instance, + context=self.context, + errors=self.errors + ) + + # Ensure that the deserialize method was not called + mock_subhandlers.get(field_name, MagicMock()).deserialize.assert_not_called() + self.assertEqual(expected_field_value, self.resource.title) + + + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + def test_update_resource_success_with_subhandlers(self, mock_subhandlers): + + """ + Ensure that when a field name corresponds to a model in th ResourceBase, + the update_resource method receives a field_value and assign it to the + corresponding model. For testing we use the "category field" + """ + field_name = "category" + field_value = "new_category_value" + json_instance = {field_name: field_value} + + # Fake resource object + fake_resource = MagicMock() + + # Simulate a MagicMock model for category + mock_category_model = MagicMock() + mock_category_model.identifier = field_value + + # Add a SUBHANDLER for the field that returns the MagicMock model + mock_subhandlers[field_name] = MagicMock() + mock_subhandlers[field_name].deserialize.return_value = mock_category_model + + # Call the method + self.base_handler.update_resource( + resource=fake_resource, + field_name=field_name, + json_instance=json_instance, + context=self.context, + errors=self.errors + ) + + mock_subhandlers[field_name].deserialize.assert_called_once_with(field_value) + self.assertEqual(fake_resource.category, mock_category_model) + + + @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) + @patch("geonode.metadata.handlers.base.logger") + def test_update_resource_exception_handling(self, mock_logger, mock_subhandlers): + """ + Handling exception + """ + field_name = "category" + field_value = "new_category_value" + json_instance = {field_name: field_value} + + # Fake resource object + fake_resource = MagicMock() + + # Add a SUBHANDLER for the field that raises an exception during deserialization + mock_subhandlers[field_name] = MagicMock() + mock_subhandlers[field_name].deserialize.side_effect = Exception("Deserialization error") + + # Call the method + self.base_handler.update_resource( + resource=fake_resource, + field_name=field_name, + json_instance=json_instance, + context=self.context, + errors=self.errors + ) + + mock_subhandlers[field_name].deserialize.assert_called_once_with(field_value) + + # Ensure that the exception is logged + mock_logger.warning.assert_called_once_with( + f"Error setting field {field_name}={field_value}: Deserialization error" + ) + + # Tests for subhandler classes of the base handler + def test_category_subhandler_update_subschema(self): + + """ + Test for the update_subschema of the CategorySubHandler. + An instance of this model has been created initial setup + """ + + subschema = { + "type": "string", + "title": "Category", + "description": "a fake description", + "maxLength": 255 + } + + # Call the update_subschema method with the real database data + CategorySubHandler.update_subschema(subschema, lang='en') + + # Assertions + self.assertIn("oneOf", subschema) + self.assertEqual(len(subschema["oneOf"]), 1) + + # Check that each entry in "oneOf" contains the expected "const", "title", and "description" + self.assertEqual(subschema["oneOf"][0]["const"], "fake_category") + self.assertEqual(subschema["oneOf"][0]["title"], "a fake gn description") + self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + + + def test_category_subhandler_serialize_with_existed_db_value(self): + + """ + Test the serialize method with existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is a model instance + serialized_value = CategorySubHandler.serialize(self.category) + + # Assert that the serialize method returns the identifier + self.assertEqual(serialized_value, self.category.identifier) + + + def test_category_subhandler_serialize_with_non_existed_db_value(self): + + """ + Test the serialize method without an existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is not a model instance + non_category_value = "nonexistent value" + + serialized_value = CategorySubHandler.serialize(non_category_value) + + # Assert that the serialize method returns the input value unchanged + self.assertEqual(serialized_value, non_category_value) + + def test_category_subhandler_deserialize(self): + + """ + Test the deserialize method. + An instance of this model has been created initial setup + """ + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = CategorySubHandler.deserialize("fake_category") + + # Assert that the deserialized value is the correct model instance + self.assertEqual(deserialized_value, self.category) + self.assertEqual(deserialized_value.identifier, "fake_category") + + + def test_license_subhandler_update_subschema(self): + + """ + Test for the update_subschema of the LicenseSubHandler. + An instance of the License model has been created + """ + + subschema = { + "type": ["string", "null"], + "title": "License", + "description": "license of the dataset", + "maxLength": 255, + "default": "eng" + } + + # Call the update_subschema method with the real database data + LicenseSubHandler.update_subschema(subschema, lang='en') + + # Assertions + self.assertIn("oneOf", subschema) + self.assertEqual(len(subschema["oneOf"]), 1) + + # Check that each entry in "oneOf" contains the expected "const", "title", and "description" + self.assertEqual(subschema["oneOf"][0]["const"], "fake_license") + self.assertEqual(subschema["oneOf"][0]["title"], "a fake name") + self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + + + def test_license_subhandler_serialize_with_existed_db_value(self): + + """ + Test the serialize method with existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is a model instance + serialized_value = LicenseSubHandler.serialize(self.license) + + # Assert that the serialize method returns the identifier + self.assertEqual(serialized_value, self.license.identifier) + + def test_license_subhandler_serialize_with_non_existed_db_value(self): + + """ + Test the serialize method without an existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is not a model instance + non_license_value = "nonexistent value" + + serialized_value = LicenseSubHandler.serialize(non_license_value) + + # Assert that the serialize method returns the input value unchanged + self.assertEqual(serialized_value, non_license_value) + + def test_license_subhandler_deserialize(self): + + """ + Test the deserialize method. + An instance of this model has been created initial setup + """ + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = LicenseSubHandler.deserialize("fake_license") + + # Assert that the deserialized value is the correct model instance + self.assertEqual(deserialized_value, self.license) + self.assertEqual(deserialized_value.identifier, "fake_license") + + + def test_restrictions_subhandler_update_subschema(self): + + """ + Test for the update_subschema of the LicenseSubHandler. + An instance of the RestrictionCodeType model has been created + """ + + subschema = { + "type": "string", + "title": "restrictions", + "description": "limitation(s) placed upon the access or use of the data.", + "maxLength": 255 + } + + # Call the update_subschema method with the real database data + RestrictionsSubHandler.update_subschema(subschema, lang='en') + + # Assertions + self.assertIn("oneOf", subschema) + self.assertEqual(len(subschema["oneOf"]), 1) + + # Check that each entry in "oneOf" contains the expected "const", "title", and "description" + self.assertEqual(subschema["oneOf"][0]["const"], "fake_restrictions") + self.assertEqual(subschema["oneOf"][0]["title"], "fake_restrictions") + self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + + + def test_restrictions_subhandler_serialize_with_existed_db_value(self): + + """ + Test the serialize method with existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is a model instance + serialized_value = RestrictionsSubHandler.serialize(self.restrictions) + + # Assert that the serialize method returns the identifier + self.assertEqual(serialized_value, self.restrictions.identifier) + + def test_restrictions_subhandler_serialize_with_non_existed_db_value(self): + + """ + Test the serialize method without an existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is not a model instance + non_restrictions_value = "nonexistent value" + + serialized_value = RestrictionsSubHandler.serialize(non_restrictions_value) + + # Assert that the serialize method returns the input value unchanged + self.assertEqual(serialized_value, non_restrictions_value) + + def test_restrictions_subhandler_deserialize(self): + + """ + Test the deserialize method. + An instance of this model has been created initial setup + """ + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = RestrictionsSubHandler.deserialize("fake_restrictions") + + # Assert that the deserialized value is the correct model instance + self.assertEqual(deserialized_value, self.restrictions) + self.assertEqual(deserialized_value.identifier, "fake_restrictions") + + + def test_spatial_repr_type_subhandler_update_subschema(self): + + """ + Test for the update_subschema of the SpatialRepresentationTypeSubHandler. + An instance of the SpatialRepresentationType model has been created + """ + + subschema = { + "type": "string", + "title": "spatial representation type", + "description": "method used to represent geographic information in the dataset.", + "maxLength": 255 + } + + # Call the update_subschema method with the real database data + SpatialRepresentationTypeSubHandler.update_subschema(subschema, lang='en') + + # Assertions + self.assertIn("oneOf", subschema) + self.assertEqual(len(subschema["oneOf"]), 1) + + # Check that each entry in "oneOf" contains the expected "const", "title", and "description" + self.assertEqual(subschema["oneOf"][0]["const"], "fake_spatial_repr") + self.assertEqual(subschema["oneOf"][0]["title"], "fake_spatial_repr") + self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + + + def test_spatial_repr_type_subhandler_serialize_with_existed_db_value(self): + + """ + Test the serialize method with existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is a model instance + serialized_value = SpatialRepresentationTypeSubHandler.serialize(self.spatial_repr) + + # Assert that the serialize method returns the identifier + self.assertEqual(serialized_value, self.spatial_repr.identifier) + + def test_spatial_repr_type_subhandler_serialize_with_non_existed_db_value(self): + + """ + Test the serialize method without an existed db value. + An instance of this model has been created initial setup + """ + + # Test the case where the db_value is not a model instance + non_spatial_repr_value = "nonexistent value" + + serialized_value = SpatialRepresentationTypeSubHandler.serialize(non_spatial_repr_value) + + # Assert that the serialize method returns the input value unchanged + self.assertEqual(serialized_value, non_spatial_repr_value) + + def test_spatial_repr_type_subhandler_deserialize(self): + + """ + Test the deserialize method. + An instance of this model has been created initial setup + """ + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = SpatialRepresentationTypeSubHandler.deserialize("fake_spatial_repr") + + # Assert that the deserialized value is the correct model instance + self.assertEqual(deserialized_value, self.spatial_repr) + self.assertEqual(deserialized_value.identifier, "fake_spatial_repr") + + + def test_date_type_subhandler_update_subschema(self): + + """ + SubHandler test for the date type + """ + + # Prepare the initial subschema + subschema = { + "type": "string", + "title": "date type", + "maxLength": 255 + } + + # Expected values for "oneOf" + expected_one_of_values = [ + {"const": "creation", "title": "Creation"}, + {"const": "publication", "title": "Publication"}, + {"const": "revision", "title": "Revision"}, + ] + + # Call the method to update the subschema + DateTypeSubHandler.update_subschema(subschema, lang="en") + + # Assertions + self.assertIn("oneOf", subschema) + self.assertEqual(subschema["oneOf"], expected_one_of_values) + self.assertIn("default", subschema) + self.assertEqual(subschema["default"], "Publication") + + + def test_date_subhandler_serialize_with_valid_datetime(self): + + """ + Subhandler test for the date serialization to the isoformat + """ + + test_datetime = datetime(2024, 12, 19, 15, 30, 45) + + # Call the serialize method + serialized_value = DateSubHandler.serialize(test_datetime) + + # Expected ISO 8601 format + expected_value = "2024-12-19T15:30:45" + + self.assertEqual(serialized_value, expected_value) + + def test_date_subhandler_serialize_without_datetime(self): + + """ + Subhandler test for the date serialization to the isoformat with non + existent datetime object + """ + + test_value = "nonexistent datetime" + + # Call the serialize method + serialized_value = DateSubHandler.serialize(test_value) + + self.assertEqual(serialized_value, test_value) + + + @patch("geonode.metadata.handlers.base.UPDATE_FREQUENCIES", new=[ + ("fake_frequency1", _("Fake frequency 1")), + ("fake_frequency2", _("Fake frequency 2")), + ("fake_frequency3", _("Fake frequency 3")), + ]) + def test_frequency_subhandler_update_subschema(self): + + """ + Subhandler test for the maintenance frequency + """ + + subschema = { + "type": "string", + "title": "maintenance frequency", + "description": "a detailed description", + "maxLength": 255 + } + + # Expected values for "oneOf" + expected_one_of_values = [ + {"const": "fake_frequency1", "title": "Fake frequency 1"}, + {"const": "fake_frequency2", "title": "Fake frequency 2"}, + {"const": "fake_frequency3", "title": "Fake frequency 3"}, + ] + + # Call the method to update the subschema + FrequencySubHandler.update_subschema(subschema, lang="en") + + self.assertIn("oneOf", subschema) + self.assertEqual(subschema["oneOf"], expected_one_of_values) + + + @patch("geonode.metadata.handlers.base.ALL_LANGUAGES", new=[ + ("fake_language1", "Fake language 1"), + ("fake_language2", "Fake language 2"), + ("fake_language3", "Fake language 3"), + ]) + def test_language_subhandler_update_subschema(self): + + """ + Language subhandler test + """ + + subschema = { + "type": "string", + "title": "language", + "description": "language used within the dataset", + "maxLength": 255, + "default": "eng" + } + + # Expected values for "oneOf" + expected_one_of_values = [ + {"const": "fake_language1", "title": "Fake language 1"}, + {"const": "fake_language2", "title": "Fake language 2"}, + {"const": "fake_language3", "title": "Fake language 3"}, + ] + + # Call the method to update the subschema + LanguageSubHandler.update_subschema(subschema, lang="en") + + self.assertIn("oneOf", subschema) + self.assertEqual(subschema["oneOf"], expected_one_of_values) + + + + + + + + + + + \ No newline at end of file diff --git a/geonode/metadata/tests/utils.py b/geonode/metadata/tests/utils.py deleted file mode 100644 index d2e2cd6c206..00000000000 --- a/geonode/metadata/tests/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest.mock import MagicMock - -# Mock subhandler class -class MockSubHandler: - - @classmethod - def update_subschema(cls, subschema, lang=None): - subschema["oneOf"] = [ - {"const": "fake const", "title": "fake title"} - ] - - @classmethod - def serialize(cls, db_value): - if isinstance(db_value, MagicMock): - return db_value.identifier - return db_value - - @classmethod - def deserialize(cls, field_value): - return MagicMock.objects.get(identifier=field_value) \ No newline at end of file From 98cfe081a40fd91b5eaee14e438065561af31a5b Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 19 Dec 2024 17:44:43 +0100 Subject: [PATCH 86/91] Fix flake --- geonode/base/api/views.py | 2 +- geonode/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index d45e4119bc5..262b32e7c98 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -1534,4 +1534,4 @@ def base_linked_resources_payload(instance, user, params={}): if lres.get("WARNINGS", None): ret["WARNINGS"] = lres["WARNINGS"] - return ret \ No newline at end of file + return ret diff --git a/geonode/settings.py b/geonode/settings.py index 58eb27889c0..a42237ae5d6 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2369,4 +2369,4 @@ def get_geonode_catalogue_service(): INSPIRE_ENABLE = ast.literal_eval(os.getenv("INSPIRE_ENABLE", "False")) if INSPIRE_ENABLE: INSTALLED_APPS += ("geonode.inspire",) - GEONODE_APPS += ("geonode.inspire",) \ No newline at end of file + GEONODE_APPS += ("geonode.inspire",) From e7b8045cc7f84376e77300ac0fbac7f40be69c19 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 19 Dec 2024 17:54:50 +0100 Subject: [PATCH 87/91] Black/flake --- geonode/base/api/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 262b32e7c98..8b6063b2f81 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -1529,7 +1529,9 @@ def base_linked_resources_payload(instance, user, params={}): lres = base_linked_resources_instances(instance, user, params) ret = { "linked_to": LinkedResourceSerializer(lres["linked_to"], embed=True, many=True).data, - "linked_by": LinkedResourceSerializer(instance=lres["linked_by"], serialize_source=True, embed=True, many=True).data + "linked_by": LinkedResourceSerializer( + instance=lres["linked_by"], serialize_source=True, embed=True, many=True + ).data, } if lres.get("WARNINGS", None): ret["WARNINGS"] = lres["WARNINGS"] From bacbe516be5231eab72caa2f1c1807d07d530a0d Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 20 Dec 2024 12:50:01 +0100 Subject: [PATCH 88/91] Metadata: fix group handling --- geonode/metadata/handlers/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/metadata/handlers/group.py b/geonode/metadata/handlers/group.py index 3f7d7f388a3..4e089ffdc31 100644 --- a/geonode/metadata/handlers/group.py +++ b/geonode/metadata/handlers/group.py @@ -66,7 +66,7 @@ def get_jsonschema_instance(self, resource, field_name, context, errors, lang=No ) def update_resource(self, resource, field_name, json_instance, context, errors, **kwargs): - data = json_instance[field_name] + data = json_instance.get(field_name, None) id = data.get("id", None) if data else None if id is not None: gp = GroupProfile.objects.get(pk=id) From aef2940e09c52a62e44678dad07f8cc27592ce9b Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 20 Dec 2024 12:53:59 +0100 Subject: [PATCH 89/91] Metadata: fix FK handling --- geonode/metadata/handlers/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geonode/metadata/handlers/base.py b/geonode/metadata/handlers/base.py index 1b058cb3d55..a9ff33017d0 100644 --- a/geonode/metadata/handlers/base.py +++ b/geonode/metadata/handlers/base.py @@ -134,7 +134,7 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - return RestrictionCodeType.objects.get(identifier=field_value) + return RestrictionCodeType.objects.get(identifier=field_value) if field_value else None class SpatialRepresentationTypeSubHandler(SubHandler): @@ -153,7 +153,7 @@ def serialize(cls, db_value): @classmethod def deserialize(cls, field_value): - return SpatialRepresentationType.objects.get(identifier=field_value) + return SpatialRepresentationType.objects.get(identifier=field_value) if field_value else None SUBHANDLERS = { From a7dcf0a3a011ab4c0d5f8fe7829febef39c75713 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Fri, 20 Dec 2024 16:45:38 +0200 Subject: [PATCH 90/91] adding tests for region and linkedrsources handers --- geonode/metadata/tests/test_handlers.py | 336 +++++++++++++++++++++++- 1 file changed, 331 insertions(+), 5 deletions(-) diff --git a/geonode/metadata/tests/test_handlers.py b/geonode/metadata/tests/test_handlers.py index b290684fad0..7fd25a5ad5d 100644 --- a/geonode/metadata/tests/test_handlers.py +++ b/geonode/metadata/tests/test_handlers.py @@ -34,7 +34,9 @@ TopicCategory, RestrictionCodeType, License, - SpatialRepresentationType + SpatialRepresentationType, + Region, + LinkedResource ) from geonode.settings import PROJECT_ROOT from geonode.metadata.handlers.base import ( @@ -48,8 +50,11 @@ RestrictionsSubHandler, SpatialRepresentationTypeSubHandler, ) +from geonode.metadata.handlers.region import RegionHandler +from geonode.metadata.handlers.linkedresource import LinkedResourceHandler +from geonode.tests.base import GeoNodeBaseTestSupport -class HandlersTests(TestCase): +class HandlersTests(GeoNodeBaseTestSupport): def setUp(self): # set Json schemas @@ -64,8 +69,10 @@ def setUp(self): # Testing database setup self.resource = ResourceBase.objects.create(title="Test Resource", uuid=str(uuid4()), owner=self.test_user_1) + self.extra_resource_1 = ResourceBase.objects.create(title="Extra resource 1", uuid=str(uuid4()), owner=self.test_user_1) + self.extra_resource_2 = ResourceBase.objects.create(title="Extra resource 2", uuid=str(uuid4()), owner=self.test_user_1) + self.extra_resource_3 = ResourceBase.objects.create(title="Extra resource 3", uuid=str(uuid4()), owner=self.test_user_1) - # Create two instances for the TopicCategory model self.category = TopicCategory.objects.create(identifier="fake_category", gn_description="a fake gn description", description="a detailed description") self.license = License.objects.create(identifier="fake_license", name="a fake name", description="a detailed description") self.restrictions = RestrictionCodeType.objects.create(identifier="fake_restrictions", description="a detailed description") @@ -87,10 +94,21 @@ def setUp(self): # Handlers self.base_handler = BaseHandler() + self.region_handler = RegionHandler() + self.linkedresource_handler = LinkedResourceHandler() + + # A fake subschema + self.fake_subschema = { + "type": "string", + "title": "new field", + "description": "A new field was added", + "maxLength": 255 + } def tearDown(self): super().tearDown() + # Tests for the Base handler @patch("geonode.metadata.handlers.base.SUBHANDLERS", new_callable=dict) def test_base_handler_update_schema(self, mock_subhandlers): @@ -301,7 +319,11 @@ def test_category_subhandler_update_subschema(self): "description": "a fake description", "maxLength": 255 } - + + # Delete all the TopicCategory models escept the "fake_category" + fake_category = TopicCategory.objects.get(identifier="fake_category") + TopicCategory.objects.exclude(identifier=fake_category.identifier).delete() + # Call the update_subschema method with the real database data CategorySubHandler.update_subschema(subschema, lang='en') @@ -365,7 +387,7 @@ def test_license_subhandler_update_subschema(self): Test for the update_subschema of the LicenseSubHandler. An instance of the License model has been created """ - + subschema = { "type": ["string", "null"], "title": "License", @@ -374,6 +396,10 @@ def test_license_subhandler_update_subschema(self): "default": "eng" } + # Delete all the License models except the "fake_license" + fake_license = License.objects.get(identifier="fake_license") + License.objects.exclude(identifier=fake_license.identifier).delete() + # Call the update_subschema method with the real database data LicenseSubHandler.update_subschema(subschema, lang='en') @@ -443,6 +469,10 @@ def test_restrictions_subhandler_update_subschema(self): "description": "limitation(s) placed upon the access or use of the data.", "maxLength": 255 } + + # Delete all the RestrictionCodeType models except the "fake_license" + fake_restrictions = RestrictionCodeType.objects.get(identifier="fake_restrictions") + RestrictionCodeType.objects.exclude(identifier=fake_restrictions.identifier).delete() # Call the update_subschema method with the real database data RestrictionsSubHandler.update_subschema(subschema, lang='en') @@ -514,6 +544,10 @@ def test_spatial_repr_type_subhandler_update_subschema(self): "maxLength": 255 } + # Delete all the SpatialRepresentationType models except the "fake_spatial_repr" + fake_spatial_repr = SpatialRepresentationType.objects.get(identifier="fake_spatial_repr") + SpatialRepresentationType.objects.exclude(identifier=fake_spatial_repr.identifier).delete() + # Call the update_subschema method with the real database data SpatialRepresentationTypeSubHandler.update_subschema(subschema, lang='en') @@ -695,6 +729,298 @@ def test_language_subhandler_update_subschema(self): self.assertIn("oneOf", subschema) self.assertEqual(subschema["oneOf"], expected_one_of_values) + + def test_add_sub_schema_without_after_what(self): + + """ + This method is used by most of the handlers in the update_schema + method, in order to add the subschema to the desired place. + This test ensures the method's functionality without after_what + """ + + jsonschema = self.fake_schema + subschema = self.fake_subschema + property_name = "new_field" + + self.base_handler._add_subschema(jsonschema, property_name, subschema) + + self.assertIn(property_name, jsonschema["properties"]) + self.assertEqual(jsonschema["properties"][property_name], subschema) + + + def test_add_sub_schema_with_after_what(self): + + """ + This method is used by most of the handlers in the update_schema + method, in order to add the subschema to the desired place. + This test ensures the method's functionality with after_what + """ + + jsonschema = self.fake_schema + subschema = self.fake_subschema + property_name = "new_field" + + # Add the "new_field" after the field "field2" + self.base_handler._add_subschema(jsonschema, property_name, subschema, after_what="field2") + + self.assertIn(property_name, jsonschema["properties"]) + # Check that the new field has been added with the defined order + self.assertEqual(list(jsonschema["properties"].keys()), ["field1", "field2", "new_field", "field3"]) + + + def test_add_subschema_with_nonexistent_after_what(self): + + """ + This method is used by most of the handlers in the update_schema + method, in order to add the subschema to the desired place. + This test ensures the method's functionality with a non-existent + after_what + """ + + jsonschema = self.fake_schema + subschema = self.fake_subschema + property_name = "new_field" + + self.base_handler._add_subschema(jsonschema, property_name, subschema, after_what="nonexistent_property") + + # Check that the new property was added + self.assertIn(property_name, jsonschema["properties"]) + + # Check that the order is maintained as best as possible + self.assertEqual(list(jsonschema["properties"].keys()), ["field1", "field2", "field3", "new_field"]) + + # Check that the subschema was added + self.assertEqual(jsonschema["properties"][property_name], subschema) + + + def test_add_subschema_to_empty_jsonschema(self): + + """ + This method is used by most of the handlers in the update_schema + method, in order to add the subschema to the desired place. + This test ensures the method's functionality with an empty schema + """ + + jsonschema = {"properties": {}} + subschema = self.fake_subschema + property_name = "new_field" + + self.base_handler._add_subschema(jsonschema, property_name, subschema, after_what="nonexistent_property") + + self.assertIn(property_name, jsonschema["properties"]) + self.assertEqual(jsonschema["properties"][property_name], subschema) + + + # Tests for the Region handler + + @patch('geonode.metadata.handlers.region.reverse') + def test_region_handler_update_schema(self, mock_reverse): + + """ + Test for the update_schema of the region_handler. In this + test we don't check if the region subschema was added after + the defined property because the _add_subschema method has + been tested above + """ + + jsonschema = self.fake_schema + mock_reverse.return_value = "/mocked_endpoint" + + # Define the expected regions schema + expected_regions = { + "type": "array", + "title": "Regions", + "description": "keyword identifies a location", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "label": {"type": "string", "title": "title"}, + }, + }, + "geonode:handler": "region", + "ui:options": {"geonode-ui:autocomplete": "/mocked_endpoint"}, + } + + # Call the method + updated_schema = self.region_handler.update_schema(jsonschema, self.context, lang=self.lang) + + self.assertIn("regions", updated_schema["properties"]) + self.assertEqual(updated_schema["properties"]["regions"], expected_regions) + + + def test_region_handler_get_jsonschema_instance(self): + + """ + Test the get_jsonschema_instance of the region handler + using two region examples: Italy and Greece + """ + + # Add two Region instances to the ResourceBase instance + region_1 = Region.objects.get(code="ITA") + region_2 = Region.objects.get(code="GRC") + self.resource.regions.add(region_1, region_2) + + # Call the method to get the JSON schema instance + field_name = "regions" + + region_instance = self.region_handler.get_jsonschema_instance( + self.resource, field_name, self.context, self.errors, self.lang + ) + + # Assert that the JSON schema contains the regions we added + expected_region_subschema = [ + {"id": str(region_1.id), "label": region_1.name}, + {"id": str(region_2.id), "label": region_2.name} + ] + + self.assertEqual( + sorted(region_instance, key=lambda x: x["id"]), + sorted(expected_region_subschema, key=lambda x: x["id"]) + ) + + + def test_region_handler_update_resource(self): + + """ + Test the update resource of the region handler + using two region examples from the testing database + """ + + # Initially we add two Region instances to the ResourceBase instance + region_1 = Region.objects.get(code="GLO") + region_2 = Region.objects.get(code="AFR") + self.resource.regions.add(region_1, region_2) + + # Definition of the new regions which will be used from the tested method + # in order to update the database replacing the above regions with the regions below + updated_region_1 = Region.objects.get(code="ITA") + updated_region_2 = Region.objects.get(code="GRC") + region_3 = Region.objects.get(code="CYP") + + payload_data = { + "regions": [ + {"id": str(updated_region_1.id), "label": updated_region_1.name}, + {"id": str(updated_region_2.id), "label": updated_region_2.name}, + {"id": str(region_3.id), "label": region_3.name}, + ] + } + + # Call the method to get the JSON schema instance + field_name = "regions" + + # Call the method + self.region_handler.update_resource(self.resource, field_name, payload_data, self.context, self.errors) + + # Ensure that only the regions defined in the payload_data are included in the resource model + self.assertEqual( + sorted(self.resource.regions.all(), key=lambda region: region.name), + sorted([updated_region_1, updated_region_2, region_3], key=lambda region: region.name) + ) + + + # Tests for the linkedresource handler + + @patch('geonode.metadata.handlers.linkedresource.reverse') + def test_linkedresource_handler_update_schema(self, mock_reverse): + + """ + Test for the update_schema of the linkedresource + """ + + jsonschema = self.fake_schema + mock_reverse.return_value = "/mocked_endpoint" + + # Define the expected regions schema + expected_linked = { + "type": "array", + "title": _("Related resources"), + "description": _("Resources related to this one"), + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + }, + "label": {"type": "string", "title": _("title")}, + }, + }, + "geonode:handler": "linkedresource", + "ui:options": {"geonode-ui:autocomplete": "/mocked_endpoint"}, + } + + # Call the method + updated_schema = self.linkedresource_handler.update_schema(jsonschema, self.context, lang=self.lang) + + self.assertIn("linkedresources", updated_schema["properties"]) + self.assertEqual(updated_schema["properties"]["linkedresources"], expected_linked) + + + def test_linkedresource_handler_get_jsonschema_instance(self): + + """ + Test the get_jsonschema_instance of the linkedresource handler + """ + + # Add a linked resource to the main resource (self.resource) + linked_resource = LinkedResource.objects.create( + source=self.resource, + target=self.extra_resource_1, + ) + + field_name = "linkedresources" + + linkedresource_instance = self.linkedresource_handler.get_jsonschema_instance( + self.resource, field_name, self.context, self.errors, self.lang + ) + + expected_linkedresource_subschema = [ + {"id": str(linked_resource.target.id), "label": linked_resource.target.title}, + ] + + self.assertEqual(linkedresource_instance, expected_linkedresource_subschema) + + + def test_linkedresource_handler_update_resource(self): + + """ + Test the update resource of the linkedresource handler + """ + + # Add a linked resource just to test if it will be removed + # after the update_resource call + # Add a linked resource to the main resource (self.resource) + LinkedResource.objects.create( + source=self.resource, + target=self.extra_resource_3, + ) + + payload_data = { + "linkedresources": [ + {"id": self.extra_resource_1.id}, + {"id": self.extra_resource_2.id} + ] + } + + # Call the method to get the JSON schema instance + field_name = "linkedresources" + + # Call the method + self.linkedresource_handler.update_resource(self.resource, field_name, payload_data, self.context, self.errors) + + # Verify the new links + linked_resources = LinkedResource.objects.filter(source=self.resource, internal=False) + linked_targets = [link.target for link in linked_resources] + self.assertIn(self.extra_resource_1, linked_targets) + self.assertIn(self.extra_resource_2, linked_targets) + # Ensure that the initial linked resource has been removed + self.assertNotIn(self.extra_resource_3, linked_targets) + + # Ensure that there is only one linked resource + self.assertEqual(len(linked_targets), 2) + + + From 8c1f40f77d158a4aefdac1172b18c0c2c96ef47d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Sat, 21 Dec 2024 20:36:36 +0200 Subject: [PATCH 91/91] fixing tests --- geonode/metadata/api/views.py | 2 +- geonode/metadata/tests/test_handlers.py | 172 +++++++++++++++++------- 2 files changed, 122 insertions(+), 52 deletions(-) diff --git a/geonode/metadata/api/views.py b/geonode/metadata/api/views.py index fda275f6cec..67e80d35e2d 100644 --- a/geonode/metadata/api/views.py +++ b/geonode/metadata/api/views.py @@ -77,11 +77,11 @@ def schema(self, request, pk=None): return Response(response) # Get the JSON schema - @action(detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)", url_name="schema_instance") @action( detail=False, methods=["get", "put", "patch"], url_path=r"instance/(?P\d+)", + url_name="schema_instance", permission_classes=[ UserHasPerms( perms_dict={ diff --git a/geonode/metadata/tests/test_handlers.py b/geonode/metadata/tests/test_handlers.py index 7fd25a5ad5d..b837fe604ea 100644 --- a/geonode/metadata/tests/test_handlers.py +++ b/geonode/metadata/tests/test_handlers.py @@ -306,35 +306,30 @@ def test_update_resource_exception_handling(self, mock_logger, mock_subhandlers) ) # Tests for subhandler classes of the base handler - def test_category_subhandler_update_subschema(self): + @patch('geonode.metadata.handlers.base.reverse') + def test_category_subhandler_update_subschema(self, mocked_endpoint): """ Test for the update_subschema of the CategorySubHandler. An instance of this model has been created initial setup """ + mocked_endpoint.return_value = "/mocked_url" + subschema = { "type": "string", "title": "Category", "description": "a fake description", "maxLength": 255 } - - # Delete all the TopicCategory models escept the "fake_category" - fake_category = TopicCategory.objects.get(identifier="fake_category") - TopicCategory.objects.exclude(identifier=fake_category.identifier).delete() # Call the update_subschema method with the real database data CategorySubHandler.update_subschema(subschema, lang='en') - # Assertions - self.assertIn("oneOf", subschema) - self.assertEqual(len(subschema["oneOf"]), 1) - - # Check that each entry in "oneOf" contains the expected "const", "title", and "description" - self.assertEqual(subschema["oneOf"][0]["const"], "fake_category") - self.assertEqual(subschema["oneOf"][0]["title"], "a fake gn description") - self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + # Assert changes to the subschema + self.assertIn("ui:options", subschema) + self.assertIn("geonode-ui:autocomplete", subschema["ui:options"]) + self.assertEqual(subschema["ui:options"]["geonode-ui:autocomplete"], mocked_endpoint.return_value) def test_category_subhandler_serialize_with_existed_db_value(self): @@ -347,24 +342,31 @@ def test_category_subhandler_serialize_with_existed_db_value(self): # Test the case where the db_value is a model instance serialized_value = CategorySubHandler.serialize(self.category) + expected_value = {"id": self.category.identifier, "label": _(self.category.gn_description)} + + self.assertEqual(serialized_value, expected_value) + # Assert that the serialize method returns the identifier - self.assertEqual(serialized_value, self.category.identifier) + self.assertEqual(serialized_value["id"], self.category.identifier) - def test_category_subhandler_serialize_with_non_existed_db_value(self): + def test_category_subhandler_serialize_invalid_data(self): """ - Test the serialize method without an existed db value. + Test the serialize method with invalid db value. An instance of this model has been created initial setup """ # Test the case where the db_value is not a model instance non_category_value = "nonexistent value" - serialized_value = CategorySubHandler.serialize(non_category_value) + invalid_serialized_value_1 = CategorySubHandler.serialize(non_category_value) + invalid_serialized_value_2 = CategorySubHandler.serialize(None) # Assert that the serialize method returns the input value unchanged - self.assertEqual(serialized_value, non_category_value) + self.assertIsNone(invalid_serialized_value_1) + self.assertIsNone(invalid_serialized_value_2) + def test_category_subhandler_deserialize(self): @@ -373,44 +375,54 @@ def test_category_subhandler_deserialize(self): An instance of this model has been created initial setup """ + field_value = {"id": "fake_category"} + # Call the method using the "fake_category" identifier from the created instance - deserialized_value = CategorySubHandler.deserialize("fake_category") + deserialized_value = CategorySubHandler.deserialize(field_value) # Assert that the deserialized value is the correct model instance self.assertEqual(deserialized_value, self.category) - self.assertEqual(deserialized_value.identifier, "fake_category") + self.assertEqual(deserialized_value.identifier, field_value["id"]) + + def test_category_subhandler_deserialize_with_invalid_data(self): + + """ + Test the deserialize method with invalid data. + """ + + field_value = None + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = CategorySubHandler.deserialize(field_value) + + # Assert that the deserialized value is the correct model instance + self.assertIsNone(deserialized_value) - def test_license_subhandler_update_subschema(self): + @patch('geonode.metadata.handlers.base.reverse') + def test_license_subhandler_update_subschema(self, mocked_endpoint): """ Test for the update_subschema of the LicenseSubHandler. - An instance of the License model has been created + An instance of this model has been created initial setup """ + + mocked_endpoint.return_value = "/mocked_url" subschema = { - "type": ["string", "null"], + "type": "string", "title": "License", - "description": "license of the dataset", - "maxLength": 255, - "default": "eng" + "description": "a fake description", + "maxLength": 255 } - - # Delete all the License models except the "fake_license" - fake_license = License.objects.get(identifier="fake_license") - License.objects.exclude(identifier=fake_license.identifier).delete() - + # Call the update_subschema method with the real database data LicenseSubHandler.update_subschema(subschema, lang='en') - # Assertions - self.assertIn("oneOf", subschema) - self.assertEqual(len(subschema["oneOf"]), 1) - - # Check that each entry in "oneOf" contains the expected "const", "title", and "description" - self.assertEqual(subschema["oneOf"][0]["const"], "fake_license") - self.assertEqual(subschema["oneOf"][0]["title"], "a fake name") - self.assertEqual(subschema["oneOf"][0]["description"], "a detailed description") + # Assert changes to the subschema + self.assertIn("ui:options", subschema) + self.assertIn("geonode-ui:autocomplete", subschema["ui:options"]) + self.assertEqual(subschema["ui:options"]["geonode-ui:autocomplete"], mocked_endpoint.return_value) def test_license_subhandler_serialize_with_existed_db_value(self): @@ -423,23 +435,30 @@ def test_license_subhandler_serialize_with_existed_db_value(self): # Test the case where the db_value is a model instance serialized_value = LicenseSubHandler.serialize(self.license) + expected_value = {"id": self.license.identifier, "label": _(self.license.name)} + + self.assertEqual(serialized_value, expected_value) + # Assert that the serialize method returns the identifier - self.assertEqual(serialized_value, self.license.identifier) + self.assertEqual(serialized_value["id"], self.license.identifier) - def test_license_subhandler_serialize_with_non_existed_db_value(self): + + def test_license_subhandler_serialize_invalid_data(self): """ - Test the serialize method without an existed db value. + Test the serialize method with invalid db value. An instance of this model has been created initial setup """ - + # Test the case where the db_value is not a model instance non_license_value = "nonexistent value" - serialized_value = LicenseSubHandler.serialize(non_license_value) + invalid_serialized_value_1 = LicenseSubHandler.serialize(non_license_value) + invalid_serialized_value_2 = LicenseSubHandler.serialize(None) # Assert that the serialize method returns the input value unchanged - self.assertEqual(serialized_value, non_license_value) + self.assertIsNone(invalid_serialized_value_1) + self.assertIsNone(invalid_serialized_value_2) def test_license_subhandler_deserialize(self): @@ -448,12 +467,29 @@ def test_license_subhandler_deserialize(self): An instance of this model has been created initial setup """ + field_value = {"id": "fake_license"} + # Call the method using the "fake_category" identifier from the created instance - deserialized_value = LicenseSubHandler.deserialize("fake_license") + deserialized_value = LicenseSubHandler.deserialize(field_value) # Assert that the deserialized value is the correct model instance self.assertEqual(deserialized_value, self.license) - self.assertEqual(deserialized_value.identifier, "fake_license") + self.assertEqual(deserialized_value.identifier, field_value["id"]) + + + def test_license_subhandler_deserialize_with_invalid_data(self): + + """ + Test the deserialize method with invalid data. + """ + + field_value = None + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = LicenseSubHandler.deserialize(field_value) + + # Assert that the deserialized value is the correct model instance + self.assertIsNone(deserialized_value) def test_restrictions_subhandler_update_subschema(self): @@ -522,12 +558,29 @@ def test_restrictions_subhandler_deserialize(self): An instance of this model has been created initial setup """ + field_value = "fake_restrictions" + # Call the method using the "fake_category" identifier from the created instance - deserialized_value = RestrictionsSubHandler.deserialize("fake_restrictions") + deserialized_value = RestrictionsSubHandler.deserialize(field_value) # Assert that the deserialized value is the correct model instance self.assertEqual(deserialized_value, self.restrictions) - self.assertEqual(deserialized_value.identifier, "fake_restrictions") + self.assertEqual(deserialized_value.identifier, field_value) + + + def test_restrictions_subhandler_deserialize_with_invalid_data(self): + + """ + Test the deserialize method with invalid data. + """ + + field_value = None + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = RestrictionsSubHandler.deserialize(field_value) + + # Assert that the deserialized value is the correct model instance + self.assertIsNone(deserialized_value) def test_spatial_repr_type_subhandler_update_subschema(self): @@ -596,12 +649,29 @@ def test_spatial_repr_type_subhandler_deserialize(self): An instance of this model has been created initial setup """ + field_value = "fake_spatial_repr" + # Call the method using the "fake_category" identifier from the created instance - deserialized_value = SpatialRepresentationTypeSubHandler.deserialize("fake_spatial_repr") + deserialized_value = SpatialRepresentationTypeSubHandler.deserialize(field_value) # Assert that the deserialized value is the correct model instance self.assertEqual(deserialized_value, self.spatial_repr) - self.assertEqual(deserialized_value.identifier, "fake_spatial_repr") + self.assertEqual(deserialized_value.identifier, field_value) + + + def test_spatial_repr_type_subhandler_deserialize_with_invalid_data(self): + + """ + Test the deserialize method with invalid data. + """ + + field_value = None + + # Call the method using the "fake_category" identifier from the created instance + deserialized_value = SpatialRepresentationTypeSubHandler.deserialize(field_value) + + # Assert that the deserialized value is the correct model instance + self.assertIsNone(deserialized_value) def test_date_type_subhandler_update_subschema(self):