Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wrap GMT's standard data type GMT_IMAGE for images #3338

Merged
merged 23 commits into from
Jul 27, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
921674c
Wrap GMT's standard data type GMT_IMAGE for images
seisman Mar 18, 2024
c01d6ab
Use GMT_OUT|GMT_IS_REFERENCE for images and GMT_OUT for others
seisman Jul 19, 2024
36430c8
Add image support to Session.read_data
seisman Jul 19, 2024
a918cfd
Add tests for reading images
seisman Jul 8, 2024
1983b7e
Improve image doctest
seisman Jul 19, 2024
84cbd11
Fix one test
seisman Jul 19, 2024
f887a23
Merge branch 'main' into wrapper/gmtimage
seisman Jul 20, 2024
95275b0
Fix tests
seisman Jul 20, 2024
d43f2e9
Revert "Fix tests"
seisman Jul 20, 2024
47d4e4a
Update pygmt/tests/test_clib_read_data.py
seisman Jul 20, 2024
bb0efe3
Temporarily add libgdal-hdf5
seisman Jul 20, 2024
994de3a
Fix one failure
seisman Jul 20, 2024
8d9c2ae
Merge branch 'main' into wrapper/gmtimage
seisman Jul 21, 2024
75c0f7e
Merge branch 'main' into wrapper/gmtimage
seisman Jul 21, 2024
dc1a649
Revert "Temporarily add libgdal-hdf5"
seisman Jul 21, 2024
0638dab
Remove the test_clib_read_data_image_actual_grid test
seisman Jul 21, 2024
7080d8b
Revert "Remove the test_clib_read_data_image_actual_grid test"
seisman Jul 21, 2024
8692161
Reapply "Temporarily add libgdal-hdf5"
seisman Jul 21, 2024
aeb2ea3
Do not check header.wesn
seisman Jul 21, 2024
dad88ce
Merge branch 'main' into wrapper/gmtimage
seisman Jul 22, 2024
3f2f6bf
Merge branch 'main' into wrapper/gmtimage
seisman Jul 23, 2024
74b889f
Remove the test for reading grid as image
seisman Jul 23, 2024
bf7f0b0
Merge branch 'main' into wrapper/gmtimage
seisman Jul 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions pygmt/clib/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
vectors_to_arrays,
)
from pygmt.clib.loading import load_libgmt
from pygmt.datatypes import _GMT_DATASET, _GMT_GRID
from pygmt.datatypes import _GMT_DATASET, _GMT_GRID, _GMT_IMAGE
from pygmt.exceptions import (
GMTCLibError,
GMTCLibNoSessionError,
Expand Down Expand Up @@ -1070,7 +1070,7 @@ def put_matrix(self, dataset, matrix, pad=0):
def read_data(
self,
infile: str,
kind: Literal["dataset", "grid"],
kind: Literal["dataset", "grid", "image"],
family: str | None = None,
geometry: str | None = None,
mode: str = "GMT_READ_NORMAL",
Expand All @@ -1088,8 +1088,8 @@ def read_data(
infile
The input file name.
kind
The data kind of the input file. Valid values are ``"dataset"`` and
``"grid"``.
The data kind of the input file. Valid values are ``"dataset"``, ``"grid"``
and ``"image"``.
family
A valid GMT data family name (e.g., ``"GMT_IS_DATASET"``). See the
``FAMILIES`` attribute for valid names. If ``None``, will determine the data
Expand Down Expand Up @@ -1140,6 +1140,7 @@ def read_data(
_family, _geometry, dtype = {
"dataset": ("GMT_IS_DATASET", "GMT_IS_PLP", _GMT_DATASET),
"grid": ("GMT_IS_GRID", "GMT_IS_SURFACE", _GMT_GRID),
"image": ("GMT_IS_IMAGE", "GMT_IS_SURFACE", _GMT_IMAGE),
}[kind]
if family is None:
family = _family
Expand Down Expand Up @@ -1789,7 +1790,9 @@ def virtualfile_from_data(

@contextlib.contextmanager
def virtualfile_out(
self, kind: Literal["dataset", "grid"] = "dataset", fname: str | None = None
self,
kind: Literal["dataset", "grid", "image"] = "dataset",
fname: str | None = None,
) -> Generator[str, None, None]:
r"""
Create a virtual file or an actual file for storing output data.
Expand All @@ -1802,8 +1805,8 @@ def virtualfile_out(
Parameters
----------
kind
The data kind of the virtual file to create. Valid values are ``"dataset"``
and ``"grid"``. Ignored if ``fname`` is specified.
The data kind of the virtual file to create. Valid values are ``"dataset"``,
``"grid"``, and ``"image"``. Ignored if ``fname`` is specified.
fname
The name of the actual file to write the output data. No virtual file will
be created.
Expand Down Expand Up @@ -1846,8 +1849,10 @@ def virtualfile_out(
family, geometry = {
"dataset": ("GMT_IS_DATASET", "GMT_IS_PLP"),
"grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"),
"image": ("GMT_IS_IMAGE", "GMT_IS_SURFACE"),
}[kind]
with self.open_virtualfile(family, geometry, "GMT_OUT", None) as vfile:
direction = "GMT_OUT|GMT_IS_REFERENCE" if kind == "image" else "GMT_OUT"
with self.open_virtualfile(family, geometry, direction, None) as vfile:
yield vfile

def inquire_virtualfile(self, vfname: str) -> int:
Expand Down Expand Up @@ -1893,7 +1898,8 @@ def read_virtualfile(
Name of the virtual file to read.
kind
Cast the data into a GMT data container. Valid values are ``"dataset"``,
``"grid"`` and ``None``. If ``None``, will return a ctypes void pointer.
``"grid"``, ``"image"`` and ``None``. If ``None``, will return a ctypes void
pointer.

Returns
-------
Expand Down Expand Up @@ -1943,9 +1949,9 @@ def read_virtualfile(
# _GMT_DATASET).
if kind is None: # Return the ctypes void pointer
return pointer
if kind in {"image", "cube"}:
if kind == "cube":
raise NotImplementedError(f"kind={kind} is not supported yet.")
dtype = {"dataset": _GMT_DATASET, "grid": _GMT_GRID}[kind]
dtype = {"dataset": _GMT_DATASET, "grid": _GMT_GRID, "image": _GMT_IMAGE}[kind]
return ctp.cast(pointer, ctp.POINTER(dtype))

def virtualfile_to_dataset(
Expand Down
1 change: 1 addition & 0 deletions pygmt/datatypes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@

from pygmt.datatypes.dataset import _GMT_DATASET
from pygmt.datatypes.grid import _GMT_GRID
from pygmt.datatypes.image import _GMT_IMAGE
94 changes: 94 additions & 0 deletions pygmt/datatypes/image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""
Wrapper for the GMT_IMAGE data type.
"""

import ctypes as ctp
from typing import ClassVar

from pygmt.datatypes.grid import _GMT_GRID_HEADER


class _GMT_IMAGE(ctp.Structure): # noqa: N801
"""
GMT image data structure.

Examples
--------
>>> import numpy as np
>>> from pygmt.clib import Session
>>> with Session() as lib:
... with lib.virtualfile_out(kind="image") as voutimg:
... lib.call_module("read", ["@earth_day_01d", voutimg, "-Ti"])
... # Read the image from the virtual file
... image = lib.read_virtualfile(vfname=voutimg, kind="image").contents
... # The image header
... header = image.header.contents
... # Access the header properties
... print(header.n_rows, header.n_columns, header.registration)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

header.z_min and header.z_max are not checked here, because they have invalid values like 1.79769313486231570000e+308 (i.e., DBL_MAX).

$ gmt grdinfo @earth_day_01d_p                               
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Title: Grid imported via GDAL
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Command:
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Remark:
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Pixel node registration used [Geographic grid]
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Grid file format: gd = Import/export through GDAL
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: x_min: -180 x_max: 180 x_inc: 1 name: x n_columns: 360
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: y_min: -90 y_max: 90 y_inc: 1 name: y n_rows: 180
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: v_min: 1.79769313486e+308 v_max: -1.79769313486e+308 name: z
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: scale_factor: 1 add_offset: 0
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Default CPT:
+proj=longlat +R=6378137 +no_defs

... print(header.wesn[:], header.inc[:])
... print(header.z_scale_factor, header.z_add_offset)
... print(header.x_units, header.y_units, header.z_units)
... print(header.title)
... print(header.command)
... print(header.remark)
... print(header.nm, header.size, header.complex_mode)
... print(header.type, header.n_bands, header.mx, header.my)
... print(header.pad[:])
... print(header.mem_layout, header.nan_value, header.xy_off)
... # Image-specific attributes.
... print(image.type, image.n_indexed_colors)
... # The x and y coordinates
... x = image.x[: header.n_columns]
... y = image.y[: header.n_rows]
... # The data array (with paddings)
... data = np.reshape(
... image.data[: header.n_bands * header.mx * header.my],
... (header.my, header.mx, header.n_bands),
... )
... # The data array (without paddings)
... pad = header.pad[:]
... data = data[pad[2] : header.my - pad[3], pad[0] : header.mx - pad[1], :]
180 360 1
[-180.0, 180.0, -90.0, 90.0] [1.0, 1.0]
1.0 0.0
b'x' b'y' b'z'
b''
b''
b''
64800 66976 0
0 3 364 184
[2, 2, 2, 2]
b'BRPa' 0.0 0.5
1 0
>>> x
[-179.5, -178.5, ..., 178.5, 179.5]
>>> y
[89.5, 88.5, ..., -88.5, -89.5]
>>> data.shape
(180, 360, 3)
>>> data.min(), data.max()
(10, 255)
"""

_fields_: ClassVar = [
# Data type, e.g. GMT_FLOAT
("type", ctp.c_int),
# Array with color lookup values
("colormap", ctp.POINTER(ctp.c_int)),
# Number of colors in a paletted image
("n_indexed_colors", ctp.c_int),
# Pointer to full GMT header for the image
("header", ctp.POINTER(_GMT_GRID_HEADER)),
# Pointer to actual image
("data", ctp.POINTER(ctp.c_ubyte)),
# Pointer to an optional transparency layer stored in a separate variable
("alpha", ctp.POINTER(ctp.c_ubyte)),
# Color interpolation
("color_interp", ctp.c_char_p),
# Pointer to the x-coordinate vector
("x", ctp.POINTER(ctp.c_double)),
# Pointer to the y-coordinate vector
("y", ctp.POINTER(ctp.c_double)),
# Book-keeping variables "hidden" from the API
("hidden", ctp.c_void_p),
]
54 changes: 54 additions & 0 deletions pygmt/tests/test_clib_read_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,60 @@ def test_clib_read_data_grid_actual_image():
)


# Note: Simplify the tests for images after GMT_IMAGE.to_dataarray() is implemented.
def test_clib_read_data_image():
"""
Test the Session.read_data method for images.
"""
with Session() as lib:
image = lib.read_data("@earth_day_01d_p", kind="image").contents
header = image.header.contents
assert header.n_rows == 180
assert header.n_columns == 360
assert header.n_bands == 3
assert header.wesn[:] == [-180.0, 180.0, -90.0, 90.0]
assert image.data


def test_clib_read_data_image_two_steps():
"""
Test the Session.read_data method for images in two steps, first reading the header
and then the data.
"""
infile = "@earth_day_01d_p"
with Session() as lib:
# Read the header first
data_ptr = lib.read_data(infile, kind="image", mode="GMT_CONTAINER_ONLY")
grid = data_ptr.contents
header = grid.header.contents
assert header.n_rows == 180
assert header.n_columns == 360
assert header.wesn[:] == [-180.0, 180.0, -90.0, 90.0]
assert header.n_bands == 3 # Explicitly check n_bands
assert not grid.data # The data is not read yet

# Read the data
lib.read_data(infile, kind="image", mode="GMT_DATA_ONLY", data=data_ptr)
assert grid.data
seisman marked this conversation as resolved.
Show resolved Hide resolved


def test_clib_read_data_image_actual_grid():
"""
Test the Session.read_data method for image, but actually the file is a grid.
"""
with Session() as lib:
data_ptr = lib.read_data(
"@earth_relief_01d_p", kind="image", mode="GMT_CONTAINER_ONLY"
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Getting this error at https://github.com/GenericMappingTools/pygmt/actions/runs/10017032324/job/27690729251?pr=3338#step:8:1556:

ERROR 4: `/home/runner/.gmt/server/earth/earth_relief/earth_relief_01d_p.grd' not recognized as being in a supported file format. It could have been recognized by driver HDF5, but plugin gdal_HDF5.so is not available in your installation. You may install it with 'conda install -c conda-forge libgdal-hdf5'
Error: ession [ERROR]: Unable to open earth_relief_01d_p.grd.
Error: ession [ERROR]: ERROR reading image with gdalread.
pygmt-session (gmtapi_import_image): Could not read from file [earth_relief_01d_p.grd]
[Session pygmt-session (198)]: Error returned from GMT API: GMT_IMAGE_READ_ERROR (22)
[Session pygmt-session (198)]: Error returned from GMT API: GMT_IMAGE_READ_ERROR (22)

So do we need to add libgdal-hdf5 to gmt-feedstock, similar to conda-forge/gmt-feedstock#291?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think yes

Copy link
Member Author

@seisman seisman Jul 20, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Working in conda-forge/gmt-feedstock#293.

Edit:

Tried to run gmt grdcut @earth_relief_01d -R-10/-9/3/5 -Greliefcut.nc in conda-forge/gmt-feedstock@ff8d995 and it worked. It means reading the netCDF file usually doesn't need the HDF5 library, but here we're actually reading the netCDF file as an image (kind="image" which reads the file into a GMT_IMAGE data container), which calls the GDAL library to read rather than the netCDF library.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test still fails even if libgdal-hdf5 is installed, see https://github.com/GenericMappingTools/pygmt/actions/runs/10018688146/job/27694518606?pr=3338.

The expected header.wesn is [-180, 180, -90, 90], but the actual results are platform-dependent. For Python 3.12 + Ubuntu, the result is [-179.5, 179.5, -89.5, 89.5] but for Python 3.10 + Ubuntu, the result is [0.5, 359.5, 0.5, 179.5].

    def test_clib_read_data_image_actual_grid():
        """
        Test the Session.read_data method for image, but actually the file is a grid.
        """
        with Session() as lib:
            data_ptr = lib.read_data(
                "@earth_relief_01d_p", kind="image", mode="GMT_CONTAINER_ONLY"
            )
            image = data_ptr.contents
            header = image.header.contents
            assert header.n_rows == 180
            assert header.n_columns == 360
>           assert header.wesn[:] == [-179.5, 179.5, -89.5, 89.5]
E           assert [0.5, 359.5, 0.5, 179.5] == [-179.5, 179.5, -89.5, 89.5]

The test added in conda-forge/gmt-feedstock#293 also fails (see https://github.com/conda-forge/gmt-feedstock/pull/293/checks?check_run_id=27696860313, https://dev.azure.com/conda-forge/feedstock-builds/_build/results?buildId=983672&view=logs&jobId=656edd35-690f-5c53-9ba3-09c10d0bea97&j=656edd35-690f-5c53-9ba3-09c10d0bea97&t=986b1512-c876-5f92-0d81-ba851554a0a3).

Not interested in digging down the rabbit hole, so I prefer to removing this test.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, or we can keep the test and mark it with xfail? Might have something to do with different GDAL versions or something. I almost think we should include GDAL in pygmt.show_versions() to more easily check things.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added the test back but the macOS + Python 3.12 CI job keeps crashing (https://github.com/GenericMappingTools/pygmt/actions/runs/10024733960/job/27707337991?pr=3338).

image = data_ptr.contents
header = image.header.contents
assert header.n_rows == 180
assert header.n_columns == 360
assert header.wesn[:] == [-180.0, 180.0, -90.0, 90.0]
# Explicitly check n_bands. Grid has only one band.
assert header.n_bands == 1


def test_clib_read_data_fails():
"""
Test that the Session.read_data method raises an exception if there are errors.
Expand Down
Loading