From b34aa9ec69f0b83ee22334dbbc3d3891337fe304 Mon Sep 17 00:00:00 2001 From: Marcel Wallschlaeger Date: Wed, 5 Jul 2023 11:49:38 +0200 Subject: [PATCH 01/11] added executionrequest endpoint --- geonodectl | 23 ++++++++++++++++++++++- src/executionrequest.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 src/executionrequest.py diff --git a/geonodectl b/geonodectl index a03a492..19957d1 100755 --- a/geonodectl +++ b/geonodectl @@ -17,6 +17,7 @@ from src.maps import GeonodeMapsHandler from src.people import GeonodePeopleHandler from src.geoapps import GeonodeGeoappsHandler from src.uploads import GeonodeUploadsHandler +from src.executionrequest import GeonodeExecutionRequestHandler GEONODECTL_URL_ENV_VAR: str = "GEONODECTL_URL" GEONODECTL_BASIC_ENV_VAR: str = "GEONODECTL_BASIC" @@ -337,7 +338,6 @@ To use this tool you have to set the following environment variables before star help="patch metadata by providing a json string like: \'{\"category\":\"{\"identifier\": \"farming\"}}\'", ) - # DESCRIBE maps_describe = maps_subparsers.add_parser("describe", help="get map details") maps_describe.add_argument( @@ -460,6 +460,25 @@ To use this tool you have to set the following environment variables before star # LIST uploads_subparsers.add_parser("list", help="list uploads") + ##################################### + # EXECUTIONREQUEST ARGUMENT PARSING # + ##################################### + executionrequest = subparsers.add_parser("executionrequest", help="executionrequest commands") + executionrequest_subparsers = executionrequest.add_subparsers( + help="geonodectl executionrequest commands", dest="subcommand", required=True + ) + + # LIST + executionrequest_subparsers.add_parser("list", help="list executionrequests") + + # DESCRIBE + executionrequest_describe = executionrequest_subparsers.add_parser( + "describe", help="get executionrequest details" + ) + executionrequest_describe.add_argument( + type=str, dest="exec_id", help="exec_id of executionrequest to describe ..." + ) + ##################### # END OF ARGPARSING # ##################### @@ -496,6 +515,8 @@ To use this tool you have to set the following environment variables before star g_obj = GeonodeGeoappsHandler(env=geonode_env) case "uploads": g_obj = GeonodeUploadsHandler(env=geonode_env) + case "executionrequest" | "execrequest": + g_obj = GeonodeExecutionRequestHandler(env=geonode_env) case _: raise NotImplemented diff --git a/src/executionrequest.py b/src/executionrequest.py new file mode 100644 index 0000000..69d7f8b --- /dev/null +++ b/src/executionrequest.py @@ -0,0 +1,39 @@ +from typing import Dict + +from src.resources import GeonodeResourceHandler +from src.geonodetypes import GeonodeCmdOutListKey + + +class GeonodeExecutionRequestHandler(GeonodeResourceHandler): + ENDPOINT_NAME = "executionrequest" + JSON_OBJECT_NAME = "requests" + SINGULAR_RESOURCE_NAME = "request" + + # TODO + LIST_CMDOUT_HEADER = [ + GeonodeCmdOutListKey(key="exec_id"), + GeonodeCmdOutListKey(key="name"), + GeonodeCmdOutListKey(key="status"), + GeonodeCmdOutListKey(key="user"), + GeonodeCmdOutListKey(key="source"), + GeonodeCmdOutListKey(key="created"), + GeonodeCmdOutListKey(key="log"), + ] + + def cmd_describe(self, exec_id: str, **kwargs): + obj = self.get(exec_id=exec_id, **kwargs) + self.print_json(obj) + + def get(self, exec_id: str, **kwargs) -> Dict: + """get details for a given exec_id + + Args: + exec_id (int): exec_id of the object + + Returns: + Dict: obj details + """ + r = self.http_get( + endpoint=f"{self.ENDPOINT_NAME}/{exec_id}?page_size={kwargs['page_size']}" + ) + return r[self.SINGULAR_RESOURCE_NAME] From 8675850d6d045262088912ab526aeae991ed695a Mon Sep 17 00:00:00 2001 From: Marcel Wallschlaeger Date: Wed, 5 Jul 2023 11:49:49 +0200 Subject: [PATCH 02/11] issue#18_UPLOAD_support_new_importer_design_in_verison_4.1 --- src/datasets.py | 3 +-- src/documents.py | 2 +- src/geoapps.py | 2 +- src/geonodeobject.py | 15 ++++++++------- src/maps.py | 2 +- src/people.py | 2 +- src/resources.py | 2 +- src/uploads.py | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/datasets.py b/src/datasets.py index 92d0632..8551cf5 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -9,7 +9,7 @@ class GeonodeDatasetsHandler(GeonodeResourceHandler): - RESOURCE_TYPE = "datasets" + ENDPOINT_NAME = JSON_OBJECT_NAME = "datasets" SINGULAR_RESOURCE_NAME = "dataset" LIST_CMDOUT_HEADER = [ @@ -51,7 +51,6 @@ def cmd_upload( ) if kwargs["json"] is True: self.print_json(r) - else: list_items = [ ["title", title], diff --git a/src/documents.py b/src/documents.py index 8e6799b..9e7401b 100644 --- a/src/documents.py +++ b/src/documents.py @@ -10,7 +10,7 @@ class GeonodeDocumentsHandler(GeonodeResourceHandler): - RESOURCE_TYPE = "documents" + ENDPOINT_NAME = JSON_OBJECT_NAME = "documents" SINGULAR_RESOURCE_NAME = "document" LIST_CMDOUT_HEADER = [ diff --git a/src/geoapps.py b/src/geoapps.py index 5d5f7d8..8dfbe9a 100644 --- a/src/geoapps.py +++ b/src/geoapps.py @@ -3,7 +3,7 @@ class GeonodeGeoappsHandler(GeonodeResourceHandler): - RESOURCE_TYPE = "geoapps" + ENDPOINT_NAME = JSON_OBJECT_NAME = "geoapps" SINGULAR_RESOURCE_NAME = "geoapp" LIST_CMDOUT_HEADER = [ diff --git a/src/geonodeobject.py b/src/geonodeobject.py index 7cae31d..70d52d0 100644 --- a/src/geonodeobject.py +++ b/src/geonodeobject.py @@ -14,7 +14,8 @@ class GeonodeObjectHandler(GeonodeRest): GeonodeCmdOutListKey(type=list, key="pk") ] DEFAULT_UPLOAD_KEYS: List[str] = ["key", "value"] - RESOURCE_TYPE: str = "" + JSON_OBJECT_NAME: str = "" + ENDPOINT_NAME: str = "" SINGULAR_RESOURCE_NAME: str = "" @classmethod @@ -36,17 +37,17 @@ def list(self, **kwargs) -> Dict: Dict: request response """ r = self.http_get( - endpoint=f"{self.RESOURCE_TYPE}/?page_size={kwargs['page_size']}" + endpoint=f"{self.ENDPOINT_NAME}/?page_size={kwargs['page_size']}" ) - return r[self.RESOURCE_TYPE] + return r[self.JSON_OBJECT_NAME] def cmd_delete(self, pk: int, **kwargs): self.delete(pk=pk, **kwargs) - print(f"{self.RESOURCE_TYPE}: {pk} deleted ...") + print(f"{self.JSON_OBJECT_NAME}: {pk} deleted ...") def delete(self, pk: int, **kwargs): """delete geonode resource object""" - self.http_get(endpoint=f"{self.RESOURCE_TYPE}/{pk}") + self.http_get(endpoint=f"{self.ENDPOINT_NAME}/{pk}") self.http_delete(endpoint=f"resources/{pk}/delete") def cmd_patch(self, pk: int, fields: str, **kwargs): @@ -75,7 +76,7 @@ def patch(self, pk: int, fields: str, **kwargs) -> Dict: ) ) - return self.http_patch(endpoint=f"{self.RESOURCE_TYPE}/{pk}/", params=json_data) + return self.http_patch(endpoint=f"{self.ENDPOINT_NAME}/{pk}/", params=json_data) def cmd_describe(self, pk: int, **kwargs): obj = self.get(pk=pk, **kwargs) @@ -91,7 +92,7 @@ def get(self, pk: int, **kwargs) -> Dict: Dict: obj details """ r = self.http_get( - endpoint=f"{self.RESOURCE_TYPE}/{pk}?page_size={kwargs['page_size']}" + endpoint=f"{self.ENDPOINT_NAME}/{pk}?page_size={kwargs['page_size']}" ) return r[self.SINGULAR_RESOURCE_NAME] diff --git a/src/maps.py b/src/maps.py index 2ef49a4..75be43a 100644 --- a/src/maps.py +++ b/src/maps.py @@ -3,7 +3,7 @@ class GeonodeMapsHandler(GeonodeResourceHandler): - RESOURCE_TYPE = "maps" + ENDPOINT_NAME = JSON_OBJECT_NAME = "maps" SINGULAR_RESOURCE_NAME = "map" LIST_CMDOUT_HEADER = [ diff --git a/src/people.py b/src/people.py index 7d375e6..fa41673 100644 --- a/src/people.py +++ b/src/people.py @@ -6,7 +6,7 @@ class GeonodePeopleHandler(GeonodeObjectHandler): - RESOURCE_TYPE = "users" + ENDPOINT_NAME = JSON_OBJECT_NAME = "users" SINGULAR_RESOURCE_NAME = "user" LIST_CMDOUT_HEADER = [ diff --git a/src/resources.py b/src/resources.py index 314dda9..6a87bb5 100644 --- a/src/resources.py +++ b/src/resources.py @@ -15,7 +15,7 @@ class GeonodeResourceHandler(GeonodeObjectHandler): - RESOURCE_TYPE = "resources" + ENDPOINT_NAME = JSON_OBJECT_NAME = "resources" SINGULAR_RESOURCE_NAME = "resource" LIST_CMDOUT_HEADER = [ diff --git a/src/uploads.py b/src/uploads.py index cfdd39d..0ae9ab6 100644 --- a/src/uploads.py +++ b/src/uploads.py @@ -5,7 +5,7 @@ class GeonodeUploadsHandler(GeonodeObjectHandler): - RESOURCE_TYPE = "uploads" + ENDPOINT_NAME = JSON_OBJECT_NAME = "uploads" SINGULAR_RESOURCE_NAME = "upload" LIST_CMDOUT_HEADER: List[GeonodeCmdOutObjectKey] = [ From 46f4bd30fbeea14faf465676d43f173c6dfeffda Mon Sep 17 00:00:00 2001 From: Marcel Wallschlaeger Date: Thu, 10 Aug 2023 14:49:12 +0200 Subject: [PATCH 03/11] issue #18_UPLOAD_support_new_importer_design_in_verison_4.1 --- src/datasets.py | 2 ++ src/executionrequest.py | 35 ++++++++++++++++++++++++++++------- src/geonodeobject.py | 17 +++++++++++++---- src/geonodetypes.py | 39 +++++++++++++++++++++++++++++++++++++++ src/people.py | 6 +++--- src/rest.py | 4 +++- 6 files changed, 88 insertions(+), 15 deletions(-) diff --git a/src/datasets.py b/src/datasets.py index 8551cf5..feb9be0 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -9,6 +9,8 @@ class GeonodeDatasetsHandler(GeonodeResourceHandler): + """docstring for GeonodeDatasetsHandler""" + ENDPOINT_NAME = JSON_OBJECT_NAME = "datasets" SINGULAR_RESOURCE_NAME = "dataset" diff --git a/src/executionrequest.py b/src/executionrequest.py index 69d7f8b..e666ecb 100644 --- a/src/executionrequest.py +++ b/src/executionrequest.py @@ -1,10 +1,11 @@ -from typing import Dict - -from src.resources import GeonodeResourceHandler from src.geonodetypes import GeonodeCmdOutListKey +from src.geonodeobject import GeonodeObjectHandler +from src.rest import GeonodeRest + +from typing import Dict -class GeonodeExecutionRequestHandler(GeonodeResourceHandler): +class GeonodeExecutionRequestHandler(GeonodeRest): ENDPOINT_NAME = "executionrequest" JSON_OBJECT_NAME = "requests" SINGULAR_RESOURCE_NAME = "request" @@ -22,13 +23,14 @@ class GeonodeExecutionRequestHandler(GeonodeResourceHandler): def cmd_describe(self, exec_id: str, **kwargs): obj = self.get(exec_id=exec_id, **kwargs) - self.print_json(obj) + GeonodeObjectHandler.print_json(obj) def get(self, exec_id: str, **kwargs) -> Dict: - """get details for a given exec_id + """ + get details for a given exec_id Args: - exec_id (int): exec_id of the object + exec_id (str): exec_id of the object Returns: Dict: obj details @@ -37,3 +39,22 @@ def get(self, exec_id: str, **kwargs) -> Dict: endpoint=f"{self.ENDPOINT_NAME}/{exec_id}?page_size={kwargs['page_size']}" ) return r[self.SINGULAR_RESOURCE_NAME] + + def cmd_list(self, **kwargs): + """show list of geonode obj on the cmdline""" + obj = self.list(**kwargs) + if kwargs["json"]: + self.print_json(obj) + else: + GeonodeObjectHandler.print_list_on_cmd(obj) + + def list(self, **kwargs) -> Dict: + """returns dict of datasets from geonode + + Returns: + Dict: request response + """ + r = self.http_get( + endpoint=f"{self.ENDPOINT_NAME}/?page_size={kwargs['page_size']}" + ) + return r[self.JSON_OBJECT_NAME] diff --git a/src/geonodeobject.py b/src/geonodeobject.py index 70d52d0..eea0e70 100644 --- a/src/geonodeobject.py +++ b/src/geonodeobject.py @@ -18,10 +18,6 @@ class GeonodeObjectHandler(GeonodeRest): ENDPOINT_NAME: str = "" SINGULAR_RESOURCE_NAME: str = "" - @classmethod - def print_json(cls, json_str: Union[str, dict]): - print(json.dumps(json_str, indent=2)) - def cmd_list(self, **kwargs): """show list of geonode obj on the cmdline""" obj = self.list(**kwargs) @@ -120,3 +116,16 @@ def generate_line(i, obj: Dict, headers: List[GeonodeCmdOutObjectKey]) -> List: generate_line(i, obj, cls.LIST_CMDOUT_HEADER) for i in range(len(obj)) ] show_list(headers=cls.cmd_list_header(), values=values) + + @classmethod + def print_json(cls, json_str: Union[str, dict]): + """ + Print the given JSON string or dictionary with an indentation of 2 spaces. + + Args: + json_str (Union[str, dict]): The JSON string or dictionary to be printed. + + Returns: + None + """ + print(json.dumps(json_str, indent=2)) diff --git a/src/geonodetypes.py b/src/geonodetypes.py index 8e6d381..23af689 100644 --- a/src/geonodetypes.py +++ b/src/geonodetypes.py @@ -8,6 +8,18 @@ class GeonodeCmdOutObjectKey: @abstractmethod def get_key(self, ds: dict): + """ + Get the key from the given dictionary. + + Parameters: + ds (dict): The dictionary to extract the key from. + + Raises: + NotImplementedError: This method is meant to be overridden by subclasses. + + Returns: + None + """ raise NotImplementedError @@ -17,9 +29,21 @@ class GeonodeCmdOutListKey(GeonodeCmdOutObjectKey): type: Type = list def __str__(self) -> str: + """ + Return a string representation of the object. + """ return self.key def get_key(self, ds: dict): + """ + Get the value associated with the given key from the given dictionary. + + Args: + ds (dict): The dictionary to retrieve the value from. + + Returns: + The value associated with the key in the dictionary. + """ return ds[self.key] @@ -32,9 +56,24 @@ class GeonodeCmdOutDictKey(GeonodeCmdOutObjectKey): type: Type = dict def __str__(self) -> str: + """ + Returns a string representation of the object. + + :return: The string representation. + :rtype: str + """ return ".".join(self.key) def get_key(self, ds: dict): + """ + Get the value from a nested dictionary using a list of keys. + + Args: + ds (dict): The nested dictionary. + + Returns: + The value obtained by accessing the nested dictionary using the list of keys. + """ for k in self.key: ds = ds[k] return ds diff --git a/src/people.py b/src/people.py index fa41673..cb7c993 100644 --- a/src/people.py +++ b/src/people.py @@ -70,16 +70,16 @@ def get( r: Dict if user_groups is True: r = self.http_get( - endpoint=f"{self.RESOURCE_TYPE}/{pk}/groups?page_size={kwargs['page_size']}" + endpoint=f"{self.ENDPOINT_NAME}/{pk}/groups?page_size={kwargs['page_size']}" ) return r elif user_resources is True: r = self.http_get( - endpoint=f"{self.RESOURCE_TYPE}/{pk}/resources?page_size={kwargs['page_size']}" + endpoint=f"{self.ENDPOINT_NAME}/{pk}/resources?page_size={kwargs['page_size']}" ) return r else: r = self.http_get( - endpoint=f"{self.RESOURCE_TYPE}/{pk}?page_size={kwargs['page_size']}" + endpoint=f"{self.ENDPOINT_NAME}/{pk}?page_size={kwargs['page_size']}" ) return r[self.SINGULAR_RESOURCE_NAME] diff --git a/src/rest.py b/src/rest.py index 768e610..b711e70 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,6 +1,8 @@ -from typing import List, Dict, Optional +from typing import List, Dict, Optional, Union import requests import urllib3 +import json + from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile urllib3.disable_warnings() From ef7d6019e3f42afdcb665c626f90ef318d3eeb5e Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 10 Aug 2023 16:12:36 +0200 Subject: [PATCH 04/11] issue#18_UPLOAD_support_new_importer_design_in_verison_4.1 --- geonodectl | 4 ++-- src/executionrequest.py | 2 +- src/rest.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/geonodectl b/geonodectl index 19957d1..19f6111 100755 --- a/geonodectl +++ b/geonodectl @@ -4,7 +4,7 @@ import logging import os import sys import argparse -from typing import List +from typing import List, Union from argparse import RawTextHelpFormatter from pathlib import Path @@ -499,7 +499,7 @@ To use this tool you have to set the following environment variables before star ) geonode_env = GeonodeEnv(url=url, auth_basic=basic, verify=args.ssl_verify) - g_obj: GeonodeObjectHandler + g_obj: Union[GeonodeObjectHandler, GeonodeExecutionRequestHandler] match args.command: case "resources" | "resource": g_obj = GeonodeResourceHandler(env=geonode_env) diff --git a/src/executionrequest.py b/src/executionrequest.py index e666ecb..4cdba00 100644 --- a/src/executionrequest.py +++ b/src/executionrequest.py @@ -44,7 +44,7 @@ def cmd_list(self, **kwargs): """show list of geonode obj on the cmdline""" obj = self.list(**kwargs) if kwargs["json"]: - self.print_json(obj) + GeonodeObjectHandler.print_json(obj) else: GeonodeObjectHandler.print_list_on_cmd(obj) diff --git a/src/rest.py b/src/rest.py index b711e70..5ae7d07 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,7 +1,6 @@ -from typing import List, Dict, Optional, Union +from typing import List, Dict, Optional import requests import urllib3 -import json from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile From 0ebc445dfb09f4b78cd261064eb1437c618f98b3 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 10 Aug 2023 17:02:19 +0200 Subject: [PATCH 05/11] issue#18_UPLOAD_support_new_importer_design_in_verison_4.1 --- src/datasets.py | 7 +------ src/rest.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/datasets.py b/src/datasets.py index feb9be0..f4d2726 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -55,12 +55,7 @@ def cmd_upload( self.print_json(r) else: list_items = [ - ["title", title], - ["success", str(r["success"])], - ["status", r["status"]], - ["bbox", r["bbox"] if "bbox" in r else ""], - ["crs", r["crs"] if "crs" in r else ""], - ["url", r["url"] if "url" in r else ""], + ["execution_id", str(r["execution_id"])], ] show_list(values=list_items, headers=["key", "value"]) diff --git a/src/rest.py b/src/rest.py index 5ae7d07..9159617 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,7 +1,7 @@ from typing import List, Dict, Optional import requests import urllib3 - +import sys from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile urllib3.disable_warnings() @@ -11,6 +11,19 @@ class GeonodeRest(object): def __init__(self, env: GeonodeEnv): self.gn_credentials = env + def network_exception_handling(func): + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except requests.exceptions.ConnectionError: + raise SystemExit(f"connection error: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") + except urllib3.exceptions.MaxRetryError: + raise SystemExit("max retries exceeded: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") + except ConnectionRefusedError: + raise SystemExit("connection refused: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") + return inner + + @property def url(self): return str(self.gn_credentials.url) @@ -23,6 +36,7 @@ def header(self): def verify(self): return self.gn_credentials.verify + @network_exception_handling def http_post( self, endpoint: str, @@ -58,6 +72,7 @@ def http_post( raise SystemExit(err) return r.json() + @network_exception_handling def http_get_download(self, url: str) -> requests.models.Response: """raw get url @@ -77,6 +92,7 @@ def http_get_download(self, url: str) -> requests.models.Response: raise SystemExit(err) return r + @network_exception_handling def http_get(self, endpoint: str, params: Dict = {}) -> Dict: """execute http delete on endpoint with params @@ -98,6 +114,7 @@ def http_get(self, endpoint: str, params: Dict = {}) -> Dict: raise SystemExit(err) return r.json() + @network_exception_handling def http_patch(self, endpoint: str, params: Dict = {}) -> Dict: """execute http patch on endpoint with params @@ -121,6 +138,7 @@ def http_patch(self, endpoint: str, params: Dict = {}) -> Dict: raise SystemExit(err) return r.json() + @network_exception_handling def http_delete(self, endpoint: str, params: Dict = {}) -> Dict: """execute http delete on endpoint with params From 40ac249d6ffe02af2d9885d96879337d76f21298 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Fri, 11 Aug 2023 16:39:36 +0200 Subject: [PATCH 06/11] issue#18_UPLOAD_support_new_importer_design_in_verison_4.1 --- src/datasets.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/datasets.py b/src/datasets.py index f4d2726..903b5de 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -6,6 +6,7 @@ from src.geonodetypes import GeonodeHTTPFile from src.cmdprint import show_list from src.geonodetypes import GeonodeCmdOutListKey, GeonodeCmdOutDictKey +from src.executionrequest import GeonodeExecutionRequestHandler class GeonodeDatasetsHandler(GeonodeResourceHandler): @@ -32,7 +33,7 @@ def cmd_upload( charset: str = "UTF-8", time: bool = False, mosaic: bool = False, - **kwargs + **kwargs, ): """upload data and show them on the cmdline @@ -49,13 +50,25 @@ def cmd_upload( charset=charset, time=time, mosaic=mosaic, - **kwargs + **kwargs, ) + if not "execution_id" in r: + raise SystemExit(f"unexpected API response ...\n{r}") + + execution_request_handler = GeonodeExecutionRequestHandler( + env=self.gn_credentials + ) + er = execution_request_handler.get(exec_id=str(r["execution_id"]), **kwargs) + if kwargs["json"] is True: - self.print_json(r) + self.print_json(er) else: list_items = [ - ["execution_id", str(r["execution_id"])], + ["exec_id", str(er["exec_id"])], + ["status", str(er["status"])], + ["created", str(er["created"])], + ["name", str(er["name"])], + ["link", str(er["link"])], ] show_list(values=list_items, headers=["key", "value"]) @@ -66,7 +79,7 @@ def upload( charset: str = "UTF-8", time: bool = False, mosaic: bool = False, - **kwargs + **kwargs, ) -> Dict: """Upload dataset to geonode. From 62f550d2d46c56d3d4956c7c597e352170a9aa83 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Fri, 11 Aug 2023 16:39:52 +0200 Subject: [PATCH 07/11] issue#18_UPLOAD_support_new_importer_design_in_verison_4.1 --- json-examples/attributes.json | 19 +++++++++++++++++++ json-examples/funders.json | 12 ++++++++++++ json-examples/related_identifier.json | 23 +++++++++++++++++++++++ json-examples/relatedprojects.json | 1 + 4 files changed, 55 insertions(+) create mode 100644 json-examples/attributes.json create mode 100644 json-examples/funders.json create mode 100644 json-examples/related_identifier.json create mode 100644 json-examples/relatedprojects.json diff --git a/json-examples/attributes.json b/json-examples/attributes.json new file mode 100644 index 0000000..a389c9d --- /dev/null +++ b/json-examples/attributes.json @@ -0,0 +1,19 @@ +"attribute_set": { + "pk": 6, + "attribute": "location", + "description": null, + "attribute_label": "location_label", + "attribute_type": "gml:PointPropertyType", + "visible": false, + "display_order": 6, + "featureinfo_type": "type_property", + "count": 1, + "min": "NA", + "max": "NA", + "average": "NA", + "median": "NA", + "stddev": "NA", + "sum": "NA", + "unique_values": "NA", + "last_stats_updated": "2023-03-30T07:10:40.476920Z" + }, \ No newline at end of file diff --git a/json-examples/funders.json b/json-examples/funders.json new file mode 100644 index 0000000..db8bf65 --- /dev/null +++ b/json-examples/funders.json @@ -0,0 +1,12 @@ +{ + "funders": [ + { + "award_title": "MOTivational strength of ecosystem services", + "award_uri": "http://cordis.europa.eu/project/rcn/100180_en.html", + "award_number": "282625", + "funding_reference": { + "funder_identifier_type": "BMBF" + } + } + ] +} diff --git a/json-examples/related_identifier.json b/json-examples/related_identifier.json new file mode 100644 index 0000000..169ff67 --- /dev/null +++ b/json-examples/related_identifier.json @@ -0,0 +1,23 @@ +{ + "related_identifier": [ + { + "related_identifier": "test10", + "related_identifier_type": { + "label": "ARK" + }, + "relation_type": { + "label": "IsNewVersionOf" + } + }, + { + "related_identifier": "test2", + "related_identifier_type": { + "label": "bibcode" + }, + "relation_type": { + "label": "Documents" + } + } + ] +} + diff --git a/json-examples/relatedprojects.json b/json-examples/relatedprojects.json new file mode 100644 index 0000000..a4d1e56 --- /dev/null +++ b/json-examples/relatedprojects.json @@ -0,0 +1 @@ +{"related_projects": [{"label": "signal"},{"label": "rhizo_bread_and_beer"}]} \ No newline at end of file From 076fe60c9dcd86a364ce88aa946058d45d12651b Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Tue, 22 Aug 2023 15:08:24 +0200 Subject: [PATCH 08/11] improve error handling issue#6 --- src/datasets.py | 2 +- src/geonodetypes.py | 9 ++++++++- src/rest.py | 31 ++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/datasets.py b/src/datasets.py index 903b5de..55ab47a 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -52,7 +52,7 @@ def cmd_upload( mosaic=mosaic, **kwargs, ) - if not "execution_id" in r: + if "execution_id" not in r: raise SystemExit(f"unexpected API response ...\n{r}") execution_request_handler = GeonodeExecutionRequestHandler( diff --git a/src/geonodetypes.py b/src/geonodetypes.py index 23af689..99a8106 100644 --- a/src/geonodetypes.py +++ b/src/geonodetypes.py @@ -1,9 +1,11 @@ -from typing import TypeAlias, Type, Tuple, Union, List +from typing import TypeAlias, Type, Tuple, Union, List, Callable, Dict, Any from dataclasses import dataclass from abc import abstractmethod import io +from src.rest import GeonodeRest + class GeonodeCmdOutObjectKey: @abstractmethod @@ -83,6 +85,11 @@ def get_key(self, ds: dict): str, Union[Tuple[str, io.BufferedReader], Tuple[str, io.BufferedReader, str]] ] +NetworkExceptionHandlingTypes: TypeAlias = ( + Callable[[GeonodeRest, str, Dict[Any, Any]], Dict[Any, Any]] + | Callable[[GeonodeRest, str], Dict[Any, Any]] +) + @dataclass class GeonodeEnv: diff --git a/src/rest.py b/src/rest.py index 9159617..63b2fb5 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,8 +1,7 @@ from typing import List, Dict, Optional -import requests import urllib3 -import sys -from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile +import requests +from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile, NetworkExceptionHandlingTypes urllib3.disable_warnings() @@ -11,18 +10,28 @@ class GeonodeRest(object): def __init__(self, env: GeonodeEnv): self.gn_credentials = env - def network_exception_handling(func): + @staticmethod + def network_exception_handling(func: NetworkExceptionHandlingTypes): def inner(*args, **kwargs): try: - return func(*args, **kwargs) + return func(*args, **kwargs) except requests.exceptions.ConnectionError: - raise SystemExit(f"connection error: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") + raise SystemExit( + "connection error: Could not reach geonode api. please check if the endpoint up and available, " + "check also the env variable: GEONODECTL_URL ..." + ) except urllib3.exceptions.MaxRetryError: - raise SystemExit("max retries exceeded: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") + raise SystemExit( + "max retries exceeded: Could not reach geonode api. please check if the endpoint up and available, " + "check also the env variable: GEONODECTL_URL ..." + ) except ConnectionRefusedError: - raise SystemExit("connection refused: Could not reach geonode api. please check if the endpoint up and available, check also the env variable: GEONODECTL_URL ...") - return inner + raise SystemExit( + "connection refused: Could not reach geonode api. please check if the endpoint up and available, " + "check also the env variable: GEONODECTL_URL ..." + ) + return inner @property def url(self): @@ -73,7 +82,7 @@ def http_post( return r.json() @network_exception_handling - def http_get_download(self, url: str) -> requests.models.Response: + def http_get_download(self, url: str) -> Dict: """raw get url Args: @@ -90,7 +99,7 @@ def http_get_download(self, url: str) -> requests.models.Response: r.raise_for_status() except requests.exceptions.HTTPError as err: raise SystemExit(err) - return r + return r.json() @network_exception_handling def http_get(self, endpoint: str, params: Dict = {}) -> Dict: From 49fd5998e9059354cd23027f0a181e3727ccf33c Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Tue, 22 Aug 2023 16:01:42 +0200 Subject: [PATCH 09/11] removed, title and abstract, fixed typing --- geonodectl | 6 ------ src/datasets.py | 3 --- src/geonodetypes.py | 10 +--------- src/rest.py | 11 +++++++++-- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/geonodectl b/geonodectl index 19f6111..61db340 100755 --- a/geonodectl +++ b/geonodectl @@ -187,12 +187,6 @@ To use this tool you have to set the following environment variables before star required=True, help="file to upload", ) - datasets_upload.add_argument( - "-t", "--title", type=str, dest="title", required=True, help="dataset title" - ) - datasets_upload.add_argument( - "-a", "--abstract", type=str, dest="abstract", help="dataset abstract" - ) datasets_upload.add_argument( "--time", action="store_true", diff --git a/src/datasets.py b/src/datasets.py index 55ab47a..8bb1816 100644 --- a/src/datasets.py +++ b/src/datasets.py @@ -74,7 +74,6 @@ def cmd_upload( def upload( self, - title: str, file_path: Path, charset: str = "UTF-8", time: bool = False, @@ -153,8 +152,6 @@ def upload( params = { # layer permissions "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}', - "dataset_title": title, - "abstract": kwargs["abstract"] if "abstract" in kwargs else "", "mosaic": mosaic, "time": str(time), "charset": charset, diff --git a/src/geonodetypes.py b/src/geonodetypes.py index 99a8106..b1b5bde 100644 --- a/src/geonodetypes.py +++ b/src/geonodetypes.py @@ -1,11 +1,8 @@ -from typing import TypeAlias, Type, Tuple, Union, List, Callable, Dict, Any - +from typing import TypeAlias, Type, Tuple, Union, List from dataclasses import dataclass from abc import abstractmethod import io -from src.rest import GeonodeRest - class GeonodeCmdOutObjectKey: @abstractmethod @@ -85,11 +82,6 @@ def get_key(self, ds: dict): str, Union[Tuple[str, io.BufferedReader], Tuple[str, io.BufferedReader, str]] ] -NetworkExceptionHandlingTypes: TypeAlias = ( - Callable[[GeonodeRest, str, Dict[Any, Any]], Dict[Any, Any]] - | Callable[[GeonodeRest, str], Dict[Any, Any]] -) - @dataclass class GeonodeEnv: diff --git a/src/rest.py b/src/rest.py index 63b2fb5..ae28f6b 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,11 +1,18 @@ -from typing import List, Dict, Optional +from typing import List, Dict, Optional, TypeAlias, Callable, Dict, Any + import urllib3 import requests -from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile, NetworkExceptionHandlingTypes +from src.geonodetypes import GeonodeEnv, GeonodeHTTPFile urllib3.disable_warnings() +NetworkExceptionHandlingTypes: TypeAlias = ( + Callable[["GeonodeRest", str, Dict[Any, Any]], Dict[Any, Any]] + | Callable[["GeonodeRest", str], Dict[Any, Any]] +) + + class GeonodeRest(object): def __init__(self, env: GeonodeEnv): self.gn_credentials = env From 51d09f47c8d1e87022daa256596a31e86a8db50b Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Wed, 23 Aug 2023 11:28:16 +0200 Subject: [PATCH 10/11] added contact_role example --- json-examples/attributes.json | 19 ------------------- json-examples/contact_roles.json | 11 +++++++++++ 2 files changed, 11 insertions(+), 19 deletions(-) delete mode 100644 json-examples/attributes.json create mode 100644 json-examples/contact_roles.json diff --git a/json-examples/attributes.json b/json-examples/attributes.json deleted file mode 100644 index a389c9d..0000000 --- a/json-examples/attributes.json +++ /dev/null @@ -1,19 +0,0 @@ -"attribute_set": { - "pk": 6, - "attribute": "location", - "description": null, - "attribute_label": "location_label", - "attribute_type": "gml:PointPropertyType", - "visible": false, - "display_order": 6, - "featureinfo_type": "type_property", - "count": 1, - "min": "NA", - "max": "NA", - "average": "NA", - "median": "NA", - "stddev": "NA", - "sum": "NA", - "unique_values": "NA", - "last_stats_updated": "2023-03-30T07:10:40.476920Z" - }, \ No newline at end of file diff --git a/json-examples/contact_roles.json b/json-examples/contact_roles.json new file mode 100644 index 0000000..946af43 --- /dev/null +++ b/json-examples/contact_roles.json @@ -0,0 +1,11 @@ +{ + "processor": [ + { + "pk": 1000, + "username": "admin" + }, + { + "pk": 1001 + } + ] +} \ No newline at end of file From 3c9f0886ed653ead34a073b160d5f15452168548 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Wed, 23 Aug 2023 11:32:53 +0200 Subject: [PATCH 11/11] fixing import --- src/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rest.py b/src/rest.py index ae28f6b..98ac000 100644 --- a/src/rest.py +++ b/src/rest.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Optional, TypeAlias, Callable, Dict, Any +from typing import List, Dict, Optional, TypeAlias, Callable, Any import urllib3 import requests