From 09e45c03684e995b2c6f6bad6cc0b776b29c5c75 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 14:36:10 -0400 Subject: [PATCH 1/8] stach commit of ci/* --- ci/310.yaml | 1 + ci/311-dev.yaml | 1 + ci/311.yaml | 1 + ci/38-minimal.yaml | 1 + ci/39.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/ci/310.yaml b/ci/310.yaml index 3e0a60ccd..2fe7f9254 100644 --- a/ci/310.yaml +++ b/ci/310.yaml @@ -16,6 +16,7 @@ dependencies: - pytest-cov - pytest-xdist # optional + - sqlalchemy - geopandas - joblib - networkx diff --git a/ci/311-dev.yaml b/ci/311-dev.yaml index e3e10d272..054b69a68 100644 --- a/ci/311-dev.yaml +++ b/ci/311-dev.yaml @@ -13,6 +13,7 @@ dependencies: - pytest-cov - pytest-xdist # optional + - sqlalchemy - geos - pyproj - fiona diff --git a/ci/311.yaml b/ci/311.yaml index 1798e9929..72ee5bf98 100644 --- a/ci/311.yaml +++ b/ci/311.yaml @@ -16,6 +16,7 @@ dependencies: - pytest-cov - pytest-xdist # optional + - sqlalchemy - geopandas>=0.12.0 - joblib - networkx diff --git a/ci/38-minimal.yaml b/ci/38-minimal.yaml index b0ecd5711..af4f651f8 100644 --- a/ci/38-minimal.yaml +++ b/ci/38-minimal.yaml @@ -15,6 +15,7 @@ dependencies: - pytest-cov - pytest-xdist # optional + - sqlalchemy=2.0 - geopandas>=0.10.0,<0.13 - shapely==2.0.1 - joblib diff --git a/ci/39.yaml b/ci/39.yaml index d9e0bb04f..063d72d9b 100644 --- a/ci/39.yaml +++ b/ci/39.yaml @@ -16,6 +16,7 @@ dependencies: - pytest-cov - pytest-xdist # optional + - sqlalchemy - geopandas - joblib - networkx From 952a4d90889873a62eb73fbf21c2779338743448 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 16:44:03 -0400 Subject: [PATCH 2/8] proper handling of sqlalchemy & removal of geomet --- libpysal/io/iohandlers/__init__.py | 2 +- libpysal/io/iohandlers/db.py | 27 ++++++++----------------- libpysal/io/iohandlers/tests/test_db.py | 27 +++++-------------------- pyproject.toml | 1 + 4 files changed, 15 insertions(+), 42 deletions(-) diff --git a/libpysal/io/iohandlers/__init__.py b/libpysal/io/iohandlers/__init__.py index 83f71eae2..ba4fcc68b 100644 --- a/libpysal/io/iohandlers/__init__.py +++ b/libpysal/io/iohandlers/__init__.py @@ -22,4 +22,4 @@ try: from . import db except: - warnings.warn("SQLAlchemy and Geomet not installed, database I/O disabled") + warnings.warn("SQLAlchemy not installed, database I/O disabled") diff --git a/libpysal/io/iohandlers/db.py b/libpysal/io/iohandlers/db.py index b64eb82c4..232d87109 100644 --- a/libpysal/io/iohandlers/db.py +++ b/libpysal/io/iohandlers/db.py @@ -1,17 +1,8 @@ from .. import fileio +from shapely import wkb errmsg = "" -try: - try: - from geomet import wkb - except ImportError: - from shapely import wkb -except ImportError: - wkb = None - errmsg += "No WKB parser found. Please install one of the following packages " - errmsg += "to enable this functionality: [geomet, shapely].\n" - try: from sqlalchemy.ext.automap import automap_base from sqlalchemy import create_engine @@ -20,13 +11,14 @@ nosql_mode = False except ImportError: nosql_mode = True - errmsg += "No module named sqlalchemy. Please install" - errmsg += " sqlalchemy to enable this functionality." + errmsg += ( + "No module named sqlalchemy. Please install" + " sqlalchemy to enable this functionality." + ) class SQLConnection(fileio.FileIO): - """Reads an SQL mappable. - """ + """Reads an SQL mappable.""" FORMATS = ["sqlite", "db"] MODES = ["r"] @@ -58,11 +50,9 @@ def close(self): fileio.FileIO.close(self) def _get_gjson(self, tablename: str, geom_column="GEOMETRY"): - gjson = {"type": "FeatureCollection", "features": []} for row in self.session.query(self.metadata.tables[tablename]): - feat = {"type": "Feature", "geometry": {}, "properties": {}} feat["GEOMETRY"] = wkb.loads(getattr(row, geom_column)) @@ -76,7 +66,6 @@ def _get_gjson(self, tablename: str, geom_column="GEOMETRY"): @property def tables(self) -> list: - if not hasattr(self, "_tables"): self._tables = list(self.metadata.tables.keys()) @@ -85,12 +74,12 @@ def tables(self) -> list: @property def session(self): """Create an ``sqlalchemy.orm.Session`` instance. - + Returns ------- self._session : sqlalchemy.orm.Session An ``sqlalchemy.orm.Session`` instance. - + """ # What happens if the session is externally closed? Check for None? diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index cdb219d57..a58e9c988 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -4,6 +4,9 @@ import unittest as ut from .... import examples as pysal_examples +from shapely.geometry import Point +from shapely import wkb + try: import sqlalchemy @@ -11,28 +14,8 @@ except ImportError: missing_sql = True -try: - import geomet - - missing_geomet = False -except ImportError: - missing_geomet = True - - -def to_wkb_point(c): - """Super quick hack that does not actually belong in here.""" - - point = {"type": "Point", "coordinates": [c[0], c[1]]} - - return geomet.wkb.dumps(point) - -@ut.skipIf( - missing_sql or missing_geomet, - "Missing dependencies: Geomet ({}) & SQLAlchemy ({}).".format( - missing_geomet, missing_sql - ), -) +@ut.skipIf(missing_sql, f"Missing dependency: SQLAlchemy ({missing_sql}).") class Test_sqlite_reader(ut.TestCase): def setUp(self): path = pysal_examples.get_path("new_haven_merged.dbf") @@ -40,7 +23,7 @@ def setUp(self): pysal_examples.load_example("newHaven") path = pysal_examples.get_path("new_haven_merged.dbf") df = pdio.read_files(path) - df["GEOMETRY"] = df["geometry"].apply(to_wkb_point) + df["GEOMETRY"] = df["geometry"].apply(lambda p: wkb.dumps(Point(p))) # This is a hack to not have to worry about a custom point type in the DB del df["geometry"] engine = sqlalchemy.create_engine("sqlite:///test.db") diff --git a/pyproject.toml b/pyproject.toml index bc98a3667..1dc1dd810 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ plus = [ "numba>=0.55", "pyarrow>=7.0", "scikit-learn>=1.1", + "sqlalchemy>=2.0", "zstd", ] dev = [ From 61fea4615a8ff9592f7be572a869f2fc0b9b950b Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 17:32:00 -0400 Subject: [PATCH 3/8] ensure testing results are deleted --- libpysal/io/iohandlers/tests/test_db.py | 1 - libpysal/weights/tests/test_user.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index a58e9c988..c8f31a041 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -49,7 +49,6 @@ def test_deserialize(self): gj = db._get_gjson("newhaven") self.assertEqual(gj["type"], "FeatureCollection") - def tearDown(self): os.remove("test.db") diff --git a/libpysal/weights/tests/test_user.py b/libpysal/weights/tests/test_user.py index 7d6ed1aaa..4a840030c 100644 --- a/libpysal/weights/tests/test_user.py +++ b/libpysal/weights/tests/test_user.py @@ -17,6 +17,7 @@ def test_build_lattice_shapefile(self): user.build_lattice_shapefile(20, 20, of) w = Rook.from_shapefile(of) self.assertEqual(w.n, 400) + os.remove("lattice.dbf") os.remove("lattice.shp") os.remove("lattice.shx") From 1c36da0e6fc2b55e308d57de4222607c55527b04 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 17:40:31 -0400 Subject: [PATCH 4/8] Update libpysal/io/iohandlers/tests/test_db.py Co-authored-by: Martin Fleischmann --- libpysal/io/iohandlers/tests/test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index c8f31a041..059d7d732 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -5,7 +5,7 @@ from .... import examples as pysal_examples from shapely.geometry import Point -from shapely import wkb +import shapely try: import sqlalchemy From 34fb5b3498f9b0c6b2d2f6a432562976a3976a36 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 17:40:47 -0400 Subject: [PATCH 5/8] Update libpysal/io/iohandlers/tests/test_db.py Co-authored-by: Martin Fleischmann --- libpysal/io/iohandlers/tests/test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index 059d7d732..8185f85f2 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -23,7 +23,7 @@ def setUp(self): pysal_examples.load_example("newHaven") path = pysal_examples.get_path("new_haven_merged.dbf") df = pdio.read_files(path) - df["GEOMETRY"] = df["geometry"].apply(lambda p: wkb.dumps(Point(p))) + df["GEOMETRY"] = shapely.to_wkb(shapely.points(df["geometry"])) # This is a hack to not have to worry about a custom point type in the DB del df["geometry"] engine = sqlalchemy.create_engine("sqlite:///test.db") From 0bd4958697a9dc6f42af179d815390e94c4b8437 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 17:54:30 -0400 Subject: [PATCH 6/8] vectorized point in db testing --- libpysal/io/iohandlers/tests/test_db.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index 8185f85f2..68986d74e 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -4,7 +4,6 @@ import unittest as ut from .... import examples as pysal_examples -from shapely.geometry import Point import shapely try: @@ -23,7 +22,7 @@ def setUp(self): pysal_examples.load_example("newHaven") path = pysal_examples.get_path("new_haven_merged.dbf") df = pdio.read_files(path) - df["GEOMETRY"] = shapely.to_wkb(shapely.points(df["geometry"])) + df["GEOMETRY"] = shapely.to_wkb(shapely.points(df["geometry"].values.tolist())) # This is a hack to not have to worry about a custom point type in the DB del df["geometry"] engine = sqlalchemy.create_engine("sqlite:///test.db") From 8b2219fdfb94baa2ac819417708af204ac01e095 Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sat, 21 Oct 2023 19:57:35 -0400 Subject: [PATCH 7/8] properly close db in test & update reflect param --- libpysal/io/iohandlers/db.py | 2 +- libpysal/io/iohandlers/tests/test_db.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libpysal/io/iohandlers/db.py b/libpysal/io/iohandlers/db.py index 232d87109..a252455fe 100644 --- a/libpysal/io/iohandlers/db.py +++ b/libpysal/io/iohandlers/db.py @@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs): self.dbname = args[0] self.Base = automap_base() self._engine = create_engine(self.dbname) - self.Base.prepare(self._engine, reflect=True) + self.Base.prepare(autoload_with=self._engine) self.metadata = self.Base.metadata def read(self, *args, **kwargs): diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index 68986d74e..2892b630f 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -25,11 +25,12 @@ def setUp(self): df["GEOMETRY"] = shapely.to_wkb(shapely.points(df["geometry"].values.tolist())) # This is a hack to not have to worry about a custom point type in the DB del df["geometry"] - engine = sqlalchemy.create_engine("sqlite:///test.db") - conn = engine.connect() + self.dbf = "iohandlers_test_db.db" + engine = sqlalchemy.create_engine(f"sqlite:///{self.dbf}") + self.conn = engine.connect() df.to_sql( "newhaven", - conn, + self.conn, index=True, dtype={ "date": sqlalchemy.types.UnicodeText, # Should convert the df date into a true date object, just a hack again @@ -42,13 +43,15 @@ def setUp(self): ) # This is converted to TEXT as lowest type common sqlite def test_deserialize(self): - db = psopen("sqlite:///test.db") + db = psopen(f"sqlite:///{self.dbf}") self.assertEqual(db.tables, ["newhaven"]) gj = db._get_gjson("newhaven") self.assertEqual(gj["type"], "FeatureCollection") - os.remove("test.db") + self.conn.close() + + os.remove(self.dbf) if __name__ == "__main__": From 433920ab3638dece7b9246e76e335b3b718270de Mon Sep 17 00:00:00 2001 From: James Gaboardi Date: Sun, 22 Oct 2023 13:55:54 -0400 Subject: [PATCH 8/8] skip test_db.py on windows --- libpysal/io/iohandlers/tests/test_db.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libpysal/io/iohandlers/tests/test_db.py b/libpysal/io/iohandlers/tests/test_db.py index 2892b630f..bc6530ffb 100644 --- a/libpysal/io/iohandlers/tests/test_db.py +++ b/libpysal/io/iohandlers/tests/test_db.py @@ -4,6 +4,7 @@ import unittest as ut from .... import examples as pysal_examples +import platform import shapely try: @@ -14,6 +15,10 @@ missing_sql = True +windows = platform.system() == "Windows" + + +@ut.skipIf(windows, "Skipping Windows due to `PermissionError`.") @ut.skipIf(missing_sql, f"Missing dependency: SQLAlchemy ({missing_sql}).") class Test_sqlite_reader(ut.TestCase): def setUp(self):