-
Notifications
You must be signed in to change notification settings - Fork 202
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add python opener from rasterio 1.4 #1331
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
ARG GDAL=ubuntu-small-3.6.4 | ||
FROM ghcr.io/osgeo/gdal:${GDAL} AS gdal | ||
ARG PYTHON_VERSION=3.9 | ||
ENV LANG="C.UTF-8" LC_ALL="C.UTF-8" | ||
RUN apt-get update && apt-get install -y software-properties-common | ||
RUN add-apt-repository -y ppa:deadsnakes/ppa | ||
RUN apt-get update && \ | ||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ | ||
g++ \ | ||
gdb \ | ||
make \ | ||
python3-pip \ | ||
python${PYTHON_VERSION} \ | ||
python${PYTHON_VERSION}-dev \ | ||
python${PYTHON_VERSION}-venv \ | ||
&& rm -rf /var/lib/apt/lists/* | ||
|
||
WORKDIR /app | ||
COPY requirements*.txt ./ | ||
RUN python${PYTHON_VERSION} -m venv /venv && \ | ||
/venv/bin/python -m pip install -U build pip && \ | ||
/venv/bin/python -m pip install -r requirements-dev.txt && \ | ||
/venv/bin/python -m pip list | ||
|
||
FROM gdal | ||
COPY . . | ||
RUN /venv/bin/python -m build -o wheels | ||
RUN /venv/bin/python -m pip install --no-index -f wheels fiona[test] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's been good to have a chance to revisit the Dockerfile. This is the way to build and install packages now. |
||
ENTRYPOINT ["/venv/bin/fio"] | ||
CMD ["--help"] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
PYTHON_VERSION ?= 3.9 | ||
GDAL ?= ubuntu-small-3.6.4 | ||
all: deps clean install test | ||
|
||
.PHONY: docs | ||
|
||
install: | ||
python setup.py build_ext | ||
pip install -e .[all] | ||
|
||
deps: | ||
pip install -r requirements-dev.txt | ||
|
||
clean: | ||
pip uninstall -y fiona || echo "no need to uninstall" | ||
python setup.py clean --all | ||
find . -name '__pycache__' -delete -print -o -name '*.pyc' -delete -print | ||
touch fiona/*.pyx | ||
|
||
sdist: | ||
python setup.py sdist | ||
|
||
test: | ||
python -m pytest --maxfail 1 -v --cov fiona --cov-report html --pdb tests | ||
|
||
docs: | ||
cd docs && make apidocs && make html | ||
|
||
doctest: | ||
py.test --doctest-modules fiona --doctest-glob='*.rst' docs/*.rst | ||
|
||
dockertestimage: | ||
docker build --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona:$(GDAL)-py$(PYTHON_VERSION) . | ||
|
||
dockertest: dockertestimage | ||
docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' | ||
|
||
dockershell: dockertestimage | ||
docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /bin/bash' | ||
|
||
dockersdist: dockertestimage | ||
docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m build --sdist' | ||
|
||
dockergdb: dockertestimage | ||
docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && gdb -ex=r --args /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' | ||
|
||
dockerdocs: dockertestimage | ||
docker run -it -v $(shell pwd):/app --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c 'source /venv/bin/activate && cd docs && make clean && make html' | ||
|
||
dockertestimage-amd64: | ||
docker build --platform linux/amd64 --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) . | ||
|
||
dockertest-amd64: dockertestimage-amd64 | ||
docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,6 +20,7 @@ | |
|
||
""" | ||
|
||
from contextlib import ExitStack | ||
import glob | ||
import logging | ||
import os | ||
|
@@ -46,6 +47,7 @@ | |
) | ||
from fiona._env import driver_count | ||
from fiona._show_versions import show_versions | ||
from fiona._vsiopener import _opener_registration | ||
from fiona.collection import BytesCollection, Collection | ||
from fiona.drvsupport import supported_drivers | ||
from fiona.env import ensure_env_with_credentials, Env | ||
|
@@ -60,7 +62,7 @@ | |
_remove, | ||
_remove_layer, | ||
) | ||
from fiona.path import ParsedPath, parse_path, vsi_path | ||
from fiona._path import _ParsedPath, _UnparsedPath, _parse_path, _vsi_path | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FWIW, I see you added a deprecation warning inside the We are running into some issues with this in geopandas: geopandas/geopandas#3207 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I commented at geopandas/geopandas#3207. Fiona's path module and methods are only public by accident. Other projects shouldn't be coupled to fiona's internals, which are full of weird details. Removing this coupling between geopandas and fiona shouldn't be hard and I'm happy to help. |
||
from fiona.vfs import parse_paths as vfs_parse_paths | ||
|
||
# These modules are imported by fiona.ogrext, but are also import here to | ||
|
@@ -82,7 +84,7 @@ | |
"remove", | ||
] | ||
|
||
__version__ = "2.0dev" | ||
__version__ = "1.10dev" | ||
__gdal_version__ = get_gdal_release_name() | ||
|
||
gdal_version = get_gdal_version_tuple() | ||
|
@@ -104,6 +106,7 @@ def open( | |
enabled_drivers=None, | ||
crs_wkt=None, | ||
allow_unsupported_drivers=False, | ||
opener=None, | ||
**kwargs | ||
): | ||
"""Open a collection for read, append, or write | ||
|
@@ -191,6 +194,19 @@ def open( | |
Defaults to GDAL's default (WKT1_GDAL for GDAL 3). | ||
allow_unsupported_drivers : bool | ||
If set to true do not limit GDAL drivers to set set of known working. | ||
opener : callable or obj, optional | ||
A custom dataset opener which can serve GDAL's virtual | ||
filesystem machinery via Python file-like objects. The | ||
underlying file-like object is obtained by calling *opener* with | ||
(*fp*, *mode*) or (*fp*, *mode* + "b") depending on the format | ||
driver's native mode. *opener* must return a Python file-like | ||
object that provides read, seek, tell, and close methods. Note: | ||
only one opener at a time per fp, mode pair is allowed. | ||
|
||
Alternatively, opener may be a filesystem object from a package | ||
like fsspec that provides the following methods: isdir(), | ||
isfile(), ls(), mtime(), open(), and size(). The exact interface | ||
is defined in the fiona._vsiopener._AbstractOpener class. | ||
kwargs : mapping | ||
Other driver-specific parameters that will be interpreted by | ||
the OGR library as layer creation or opening options. | ||
|
@@ -273,49 +289,67 @@ def func(*args, **kwds): | |
# At this point, the fp argument is a string or path-like object | ||
# which can be converted to a string. | ||
else: | ||
# If a pathlib.Path instance is given, convert it to a string path. | ||
if isinstance(fp, Path): | ||
fp = str(fp) | ||
stack = ExitStack() | ||
|
||
if vfs: | ||
warnings.warn( | ||
"The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.", | ||
FionaDeprecationWarning, | ||
stacklevel=2, | ||
) | ||
path, scheme, archive = vfs_parse_paths(fp, vfs=vfs) | ||
path = ParsedPath(path, archive, scheme) | ||
if hasattr(fp, "path") and hasattr(fp, "fs"): | ||
log.debug("Detected fp is an OpenFile: fp=%r", fp) | ||
raw_dataset_path = fp.path | ||
opener = fp.fs.open | ||
else: | ||
path = parse_path(fp) | ||
|
||
if mode in ("a", "r"): | ||
colxn = Collection( | ||
path, | ||
mode, | ||
driver=driver, | ||
encoding=encoding, | ||
layer=layer, | ||
enabled_drivers=enabled_drivers, | ||
allow_unsupported_drivers=allow_unsupported_drivers, | ||
**kwargs | ||
) | ||
elif mode == "w": | ||
colxn = Collection( | ||
path, | ||
mode, | ||
crs=crs, | ||
driver=driver, | ||
schema=schema, | ||
encoding=encoding, | ||
layer=layer, | ||
enabled_drivers=enabled_drivers, | ||
crs_wkt=crs_wkt, | ||
allow_unsupported_drivers=allow_unsupported_drivers, | ||
**kwargs | ||
) | ||
else: | ||
raise ValueError("mode string must be one of {'r', 'w', 'a'}") | ||
|
||
raw_dataset_path = os.fspath(fp) | ||
|
||
try: | ||
if opener: | ||
log.debug("Registering opener: raw_dataset_path=%r, mode=%r, opener=%r", raw_dataset_path, mode, opener) | ||
vsi_path_ctx = _opener_registration(raw_dataset_path, mode[0], opener) | ||
registered_vsi_path = stack.enter_context(vsi_path_ctx) | ||
log.debug("Registered vsi path: registered_vsi_path%r", registered_vsi_path) | ||
path = _UnparsedPath(registered_vsi_path) | ||
else: | ||
if vfs: | ||
warnings.warn( | ||
"The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.", | ||
FionaDeprecationWarning, | ||
stacklevel=2, | ||
) | ||
path, scheme, archive = vfs_parse_paths(fp, vfs=vfs) | ||
path = _ParsedPath(path, archive, scheme) | ||
else: | ||
path = _parse_path(fp) | ||
|
||
if mode in ("a", "r"): | ||
colxn = Collection( | ||
path, | ||
mode, | ||
driver=driver, | ||
encoding=encoding, | ||
layer=layer, | ||
enabled_drivers=enabled_drivers, | ||
allow_unsupported_drivers=allow_unsupported_drivers, | ||
**kwargs | ||
) | ||
elif mode == "w": | ||
colxn = Collection( | ||
path, | ||
mode, | ||
crs=crs, | ||
driver=driver, | ||
schema=schema, | ||
encoding=encoding, | ||
layer=layer, | ||
enabled_drivers=enabled_drivers, | ||
crs_wkt=crs_wkt, | ||
allow_unsupported_drivers=allow_unsupported_drivers, | ||
**kwargs | ||
) | ||
else: | ||
raise ValueError("mode string must be one of {'r', 'w', 'a'}") | ||
|
||
except Exception: | ||
stack.close() | ||
raise | ||
|
||
colxn._env = stack | ||
return colxn | ||
|
||
|
||
|
@@ -392,8 +426,8 @@ def listdir(fp): | |
if not isinstance(fp, str): | ||
raise TypeError("invalid path: %r" % fp) | ||
|
||
pobj = parse_path(fp) | ||
return _listdir(vsi_path(pobj)) | ||
pobj = _parse_path(fp) | ||
return _listdir(_vsi_path(pobj)) | ||
|
||
|
||
@ensure_env_with_credentials | ||
|
@@ -442,13 +476,13 @@ def listlayers(fp, vfs=None, **kwargs): | |
FionaDeprecationWarning, | ||
stacklevel=2, | ||
) | ||
pobj_vfs = parse_path(vfs) | ||
pobj_path = parse_path(fp) | ||
pobj = ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme) | ||
pobj_vfs = _parse_path(vfs) | ||
pobj_path = _parse_path(fp) | ||
pobj = _ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme) | ||
else: | ||
pobj = parse_path(fp) | ||
pobj = _parse_path(fp) | ||
|
||
return _listlayers(vsi_path(pobj), **kwargs) | ||
return _listlayers(_vsi_path(pobj), **kwargs) | ||
|
||
|
||
def prop_width(val): | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
New default.