Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MAINT: dealing with sqlalchemy & geomet #592

Merged
merged 11 commits into from
Oct 22, 2023
1 change: 1 addition & 0 deletions ci/310-oldest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ dependencies:
- numba=0.55
- pyarrow>=7.0
- scikit-learn=1.1
- sqlalchemy=2.0
- zstd
- pip
- pip:
Expand Down
1 change: 1 addition & 0 deletions ci/310.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,5 @@ dependencies:
- numba
- pyarrow
- scikit-learn
- sqlalchemy
- zstd
3 changes: 2 additions & 1 deletion ci/311.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ dependencies:
- pytest-cov
- pytest-mpl
- pytest-xdist
# optional
# optional
- geodatasets
- joblib
- networkx
- numba
- pyarrow
- scikit-learn
- sqlalchemy
- zstd
1 change: 1 addition & 0 deletions ci/312-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ dependencies:
- packaging
- pyarrow
- pyproj
- sqlalchemy
- zstd
- pip
- pip:
Expand Down
1 change: 1 addition & 0 deletions ci/312.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ dependencies:
# - numba # follow up when numba is ready for 3.12
- pyarrow
- scikit-learn
- sqlalchemy
- zstd
# for docs build action (this env only)
- mkdocs-jupyter
Expand Down
2 changes: 1 addition & 1 deletion libpysal/io/iohandlers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@
try:
from . import db
except:
warnings.warn("SQLAlchemy and Geomet not installed, database I/O disabled")
warnings.warn("SQLAlchemy not installed, database I/O disabled")
27 changes: 8 additions & 19 deletions libpysal/io/iohandlers/db.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,8 @@
from .. import fileio
from shapely import wkb

errmsg = ""

try:
try:
from geomet import wkb
except ImportError:
from shapely import wkb
except ImportError:
wkb = None
errmsg += "No WKB parser found. Please install one of the following packages "
errmsg += "to enable this functionality: [geomet, shapely].\n"

try:
from sqlalchemy.ext.automap import automap_base
from sqlalchemy import create_engine
Expand All @@ -20,13 +11,14 @@
nosql_mode = False
except ImportError:
nosql_mode = True
errmsg += "No module named sqlalchemy. Please install"
errmsg += " sqlalchemy to enable this functionality."
errmsg += (
"No module named sqlalchemy. Please install"
" sqlalchemy to enable this functionality."
)


class SQLConnection(fileio.FileIO):
"""Reads an SQL mappable.
"""
"""Reads an SQL mappable."""

FORMATS = ["sqlite", "db"]
MODES = ["r"]
Expand Down Expand Up @@ -58,11 +50,9 @@ def close(self):
fileio.FileIO.close(self)

def _get_gjson(self, tablename: str, geom_column="GEOMETRY"):

gjson = {"type": "FeatureCollection", "features": []}

for row in self.session.query(self.metadata.tables[tablename]):

feat = {"type": "Feature", "geometry": {}, "properties": {}}
feat["GEOMETRY"] = wkb.loads(getattr(row, geom_column))

Expand All @@ -76,7 +66,6 @@ def _get_gjson(self, tablename: str, geom_column="GEOMETRY"):

@property
def tables(self) -> list:

if not hasattr(self, "_tables"):
self._tables = list(self.metadata.tables.keys())

Expand All @@ -85,12 +74,12 @@ def tables(self) -> list:
@property
def session(self):
"""Create an ``sqlalchemy.orm.Session`` instance.

Returns
-------
self._session : sqlalchemy.orm.Session
An ``sqlalchemy.orm.Session`` instance.

"""

# What happens if the session is externally closed? Check for None?
Expand Down
28 changes: 5 additions & 23 deletions libpysal/io/iohandlers/tests/test_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,43 +4,26 @@
import unittest as ut
from .... import examples as pysal_examples

from shapely.geometry import Point
from shapely import wkb
jGaboardi marked this conversation as resolved.
Show resolved Hide resolved

try:
import sqlalchemy

missing_sql = False
except ImportError:
missing_sql = True

try:
import geomet

missing_geomet = False
except ImportError:
missing_geomet = True


def to_wkb_point(c):
"""Super quick hack that does not actually belong in here."""

point = {"type": "Point", "coordinates": [c[0], c[1]]}

return geomet.wkb.dumps(point)


@ut.skipIf(
missing_sql or missing_geomet,
"Missing dependencies: Geomet ({}) & SQLAlchemy ({}).".format(
missing_geomet, missing_sql
),
)
@ut.skipIf(missing_sql, f"Missing dependency: SQLAlchemy ({missing_sql}).")
class Test_sqlite_reader(ut.TestCase):
def setUp(self):
path = pysal_examples.get_path("new_haven_merged.dbf")
if path is None:
pysal_examples.load_example("newHaven")
path = pysal_examples.get_path("new_haven_merged.dbf")
df = pdio.read_files(path)
df["GEOMETRY"] = df["geometry"].apply(to_wkb_point)
df["GEOMETRY"] = df["geometry"].apply(lambda p: wkb.dumps(Point(p)))
jGaboardi marked this conversation as resolved.
Show resolved Hide resolved
# This is a hack to not have to worry about a custom point type in the DB
del df["geometry"]
engine = sqlalchemy.create_engine("sqlite:///test.db")
Expand All @@ -66,7 +49,6 @@ def test_deserialize(self):
gj = db._get_gjson("newhaven")
self.assertEqual(gj["type"], "FeatureCollection")

def tearDown(self):
os.remove("test.db")


Expand Down
1 change: 1 addition & 0 deletions libpysal/weights/tests/test_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ def test_build_lattice_shapefile(self):
user.build_lattice_shapefile(20, 20, of)
w = Rook.from_shapefile(of)
self.assertEqual(w.n, 400)
os.remove("lattice.dbf")
os.remove("lattice.shp")
os.remove("lattice.shx")

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ plus = [
"numba>=0.55",
"pyarrow>=7.0",
"scikit-learn>=1.1",
"sqlalchemy>=2.0",
"zstd",
]
dev = [
Expand Down
Loading