Merge branch 'master' into ISSUE_9862

docker-compose-test.yml

@@ -135,7 +135,7 @@ services:
       - dbbackups:/pg_backups
     restart: on-failure
     healthcheck:
-      test: "pg_isready -d gis"
+      test: "pg_isready -d postgres -U postgres"
     # uncomment to enable remote connections to postgres
     #ports:
     #  - "5432:5432"

docker-compose.yml

@@ -135,7 +135,7 @@ services:
       - dbbackups:/pg_backups
     restart: on-failure
     healthcheck:
-      test: "pg_isready -d gis"
+      test: "pg_isready -d postgres -U postgres"
     # uncomment to enable remote connections to postgres
     #ports:
     #  - "5432:5432"

geonode/storage/data_retriever.py

@@ -191,6 +191,8 @@ def items(self):
         return self.data_items.items()
 
     def _unzip(self, zip_name: str) -> Mapping:
+        from geonode.utils import get_allowed_extensions
+
         '''
         Function to unzip the file. If is a shp or a tiff
         is assigned as base_file otherwise will create the expected payloads
@@ -199,8 +201,11 @@ def _unzip(self, zip_name: str) -> Mapping:
         zip_file = self.file_paths['base_file']
         the_zip = zipfile.ZipFile(zip_file, allowZip64=True)
         the_zip.extractall(self.temporary_folder)
+        available_choices = get_allowed_extensions()
+        not_main_files = ['xml', 'sld', 'zip']
+        base_file_choices = [x for x in available_choices if x not in not_main_files]
         for _file in Path(self.temporary_folder).iterdir():
-            if _file.name.endswith('.shp') or _file.name.endswith('.tif'):
+            if any([_file.name.endswith(_ext) for _ext in base_file_choices]):
                 self.file_paths['base_file'] = Path(str(_file))
             elif not zipfile.is_zipfile(str(_file)):
                 ext = _file.name.split(".")[-1]

geonode/storage/tests.py

@@ -18,8 +18,8 @@
 #########################################################################
 import io
 import os
+import shutil
 import gisdata
-
 from unittest.mock import patch
 
 from django.test.testcases import SimpleTestCase, TestCase
@@ -401,6 +401,11 @@ def test_storage_manager_replace_single_file(self, path, strg):
 
 
 class TestDataRetriever(TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+        cls.project_root = os.path.abspath(os.path.dirname(__file__))
+
     def setUp(self):
         self.sut = StorageManager
         self.local_files_paths = {
@@ -569,3 +574,26 @@ def test_storage_manager_rmtree(self):
 
         self.sut().rmtree(_tmpdir)
         self.assertFalse(os.path.exists(_tmpdir))
+
+    def test_zip_file_should_correctly_recognize_main_extension_with_csv(self):
+        # reinitiate the storage manager with the zip file
+        storage_manager = self.sut(remote_files={"base_file": os.path.join(f"{self.project_root}", "tests/data/example.zip")})
+        storage_manager.clone_remote_files()
+
+        self.assertIsNotNone(storage_manager.data_retriever.temporary_folder)
+        _files = storage_manager.get_retrieved_paths()
+        self.assertTrue("example.csv" in _files.get("base_file"))
+
+    def test_zip_file_should_correctly_recognize_main_extension_with_shp(self):
+        # zipping files
+        storage_manager = self.sut(remote_files=self.local_files_paths)
+        storage_manager.clone_remote_files()
+        storage_manager.data_retriever.temporary_folder
+        output = shutil.make_archive(f"{storage_manager.data_retriever.temporary_folder}/output", 'zip', storage_manager.data_retriever.temporary_folder)
+        # reinitiate the storage manager with the zip file
+        storage_manager = self.sut(remote_files={"base_file": output})
+        storage_manager.clone_remote_files()
+
+        self.assertIsNotNone(storage_manager.data_retriever.temporary_folder)
+        _files = storage_manager.get_retrieved_paths()
+        self.assertTrue("single_point.shp" in _files.get("base_file"))

requirements.txt

@@ -20,11 +20,11 @@ vine==5.0.0
 tqdm==4.64.0
 Deprecated==1.2.13
 wrapt==1.14.1
-jsonschema==4.9.0
+jsonschema==4.14.0
 zipstream-new==1.1.8
 schema==0.7.5
 rdflib==6.1.1
-smart_open==6.0.0
+smart_open==6.1.0
 
 # Django Apps
 django-allauth==0.51.0
@@ -53,8 +53,8 @@ pyopenssl==22.0.0
 pyproj<3.3.0
 OWSLib==0.26.0
 pycsw==2.6.1
-SQLAlchemy==1.4.39 # required by PyCSW
-Shapely==1.8.2
+SQLAlchemy==1.4.40 # required by PyCSW
+Shapely==1.8.4
 mercantile==1.2.1
 geoip2==4.6.0
 numpy==1.23.*
@@ -76,7 +76,7 @@ djangorestframework<3.12.0,>=3.8.0
 djangorestframework-gis==1.0
 djangorestframework-guardian==0.3.0
 drf-extensions==0.7.1
-drf-writable-nested==0.6.4
+drf-writable-nested==0.7.0
 drf-spectacular==0.23.1
 dynamic-rest==2.1.2
 Markdown==3.4.1
@@ -93,7 +93,7 @@ geonode-user-messages==2.0.2
 geonode-announcements==2.0.2
 geonode-django-activity-stream==0.10.0
 gn-arcrest==10.5.5
-geoserver-restconfig==2.0.7
+geoserver-restconfig==2.0.8
 gn-gsimporter==2.0.4
 gisdata==0.5.4
 
@@ -102,14 +102,14 @@ django-haystack==3.2.1
 elasticsearch>=2.0.0,<9.0.0
 
 # datetimepicker widget
-django-bootstrap3-datetimepicker-2==2.8.2
+django-bootstrap3-datetimepicker-2==2.8.3
 
 # storage manager dependencies
-django-storages==1.12.3
+django-storages==1.13.1
 dropbox==11.33.0
-google-cloud-storage==2.4.0
+google-cloud-storage==2.5.0
 google-cloud-core==2.3.2
-boto3==1.24.42
+boto3==1.24.56
 
 # Django Caches
 python-memcached<=1.59
@@ -122,10 +122,10 @@ inflection>=0.4.0
 jdcal==1.4.1
 mock<5.0.0
 python-dateutil==2.8.2
-pytz==2022.1
+pytz==2022.2.1
 requests==2.28.1
 timeout-decorator==0.5.0
-pylibmc==1.6.1
+pylibmc==1.6.2
 sherlock==0.3.2
 
 # required by monitoring
@@ -142,20 +142,20 @@ pycountry
 uWSGI==2.0.20
 gunicorn==20.1.0
 ipython==8.4.0
-docker==5.0.3
+docker==6.0.0
 invoke==1.7.1
 
 # tests
-coverage==6.4.2
+coverage==6.4.4
 requests-toolbelt==0.9.1
-flake8==5.0.1
+flake8==5.0.4
 pytest==7.1.2
 pytest-bdd==6.0.1
 splinter==0.18.1
 pytest-splinter==3.3.1
 pytest-django==4.5.2
-setuptools>=59.1.1,<63.3.0
-pip==22.2.1
+setuptools>=59.1.1,<65.3.0
+pip==22.2.2
 Twisted==22.4.0
 pixelmatch==0.3.0
 factory-boy==3.2.1

setup.cfg

@@ -46,11 +46,11 @@ install_requires =
     tqdm==4.64.0
     Deprecated==1.2.13
     wrapt==1.14.1
-    jsonschema==4.9.0
+    jsonschema==4.14.0
     zipstream-new==1.1.8
     schema==0.7.5
     rdflib==6.1.1
-    smart_open==6.0.0
+    smart_open==6.1.0
 
     # Django Apps
     django-allauth==0.51.0
@@ -79,8 +79,8 @@ install_requires =
     pyproj<3.3.0
     OWSLib==0.26.0
     pycsw==2.6.1
-    SQLAlchemy==1.4.39 # required by PyCSW
-    Shapely==1.8.2
+    SQLAlchemy==1.4.40 # required by PyCSW
+    Shapely==1.8.4
     mercantile==1.2.1
     geoip2==4.6.0
     numpy==1.23.*
@@ -102,7 +102,7 @@ install_requires =
     djangorestframework-gis==1.0
     djangorestframework-guardian==0.3.0
     drf-extensions==0.7.1
-    drf-writable-nested==0.6.4
+    drf-writable-nested==0.7.0
     drf-spectacular==0.23.1
     dynamic-rest==2.1.2
     Markdown==3.4.1
@@ -118,7 +118,7 @@ install_requires =
     geonode-announcements==2.0.2
     geonode-django-activity-stream==0.10.0
     gn-arcrest==10.5.5
-    geoserver-restconfig==2.0.7
+    geoserver-restconfig==2.0.8
     gn-gsimporter==2.0.4
     gisdata==0.5.4
 
@@ -127,14 +127,14 @@ install_requires =
     elasticsearch>=2.0.0,<9.0.0
 
     # datetimepicker widget
-    django-bootstrap3-datetimepicker-2==2.8.2
+    django-bootstrap3-datetimepicker-2==2.8.3
 
     # storage manager dependencies
-    django-storages==1.12.3
+    django-storages==1.13.1
     dropbox==11.33.0
-    google-cloud-storage==2.4.0
+    google-cloud-storage==2.5.0
     google-cloud-core==2.3.2
-    boto3==1.24.42
+    boto3==1.24.56
 
     # Django Caches
     python-memcached<=1.59
@@ -147,10 +147,10 @@ install_requires =
     jdcal==1.4.1
     mock<5.0.0
     python-dateutil==2.8.2
-    pytz==2022.1
+    pytz==2022.2.1
     requests==2.28.1
     timeout-decorator==0.5.0
-    pylibmc==1.6.1
+    pylibmc==1.6.2
     sherlock==0.3.2
 
     # required by monitoring
@@ -167,20 +167,20 @@ install_requires =
     uWSGI==2.0.20
     gunicorn==20.1.0
     ipython==8.4.0
-    docker==5.0.3
+    docker==6.0.0
     invoke==1.7.1
 
     # tests
-    coverage==6.4.2
+    coverage==6.4.4
     requests-toolbelt==0.9.1
-    flake8==5.0.1
+    flake8==5.0.4
     pytest==7.1.2
     pytest-bdd==6.0.1
     splinter==0.18.1
     pytest-splinter==3.3.1
     pytest-django==4.5.2
-    setuptools>=59.1.1,<63.3.0
-    pip==22.2.1
+    setuptools>=59.1.1,<65.3.0
+    pip==22.2.2
     Twisted==22.4.0
     pixelmatch==0.3.0
     factory-boy==3.2.1