Fix wheel builds (#577)

### Change list - Update with a fix to not implement Fd support on Windows. From omerbenamram/pyo3-file#19. - Have to wait for redhat to get their shit together: https://status.redhat.com/incidents/qh68rjfg6xs6 - The aarch64 build is failing with ``error: failed to run custom build command for `ring v0.17.8` `` which looks to be briansmith/ring#1414.
geoarrow · Mar 22, 2024 · ee42b3b · ee42b3b
1 parent e019256
commit ee42b3b
Show file tree

Hide file tree

Showing 5 changed files with 118 additions and 94 deletions.
diff --git a/.github/workflows/python-core-wheels.yml b/.github/workflows/python-core-wheels.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: [x86_64, i686]
+        target: [x86_64]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -22,8 +22,11 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          manylinux: auto
+          manylinux: manylinux_2_28
           args: --release --out dist -m python/core/Cargo.toml
+          before-script-linux: |
+            yum update -y
+            yum install openssl openssl-devel perl-IPC-Cmd -y
 
       - name: Install built wheel
         if: matrix.target == 'x86_64'
@@ -39,79 +42,82 @@ jobs:
           name: wheels
           path: dist/*.whl
 
-  linux-cross:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target: [aarch64, armv7, s390x, ppc64le, ppc64]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          manylinux: auto
-          args: --release --out dist -m python/core/Cargo.toml
+  # linux-cross:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       target: [aarch64, armv7, s390x, ppc64le, ppc64]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.8
 
-      # This is currently failing with
-      # python: command not found
-
-      # - uses: uraimo/run-on-arch-action@v2.5.1
-      #   if: matrix.target == 'aarch64'
-      #   name: Install built wheel
-      #   with:
-      #     arch: ${{ matrix.target }}
-      #     distro: ubuntu20.04
-      #     githubToken: ${{ github.token }}
-      #     install: |
-      #       apt-get update
-      #       apt-get install -y --no-install-recommends python3 python3-pip
-      #       pip3 install -U pip
-      #     run: |
-      #       pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall
-      #       python -c "import geoarrow.rust.core"
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@v1
+  #       with:
+  #         target: ${{ matrix.target }}
+  #         manylinux: auto
+  #         args: --release --out dist -m python/core/Cargo.toml
 
-      # Have to set path from root
-      # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360
-      - name: Upload wheels
-        uses: actions/upload-artifact@v3
-        with:
-          name: wheels
-          path: dist/*.whl
+  #     # This is currently failing with
+  #     # python: command not found
+
+  #     # - uses: uraimo/run-on-arch-action@v2.5.1
+  #     #   if: matrix.target == 'aarch64'
+  #     #   name: Install built wheel
+  #     #   with:
+  #     #     arch: ${{ matrix.target }}
+  #     #     distro: ubuntu20.04
+  #     #     githubToken: ${{ github.token }}
+  #     #     install: |
+  #     #       apt-get update
+  #     #       apt-get install -y --no-install-recommends python3 python3-pip
+  #     #       pip3 install -U pip
+  #     #     run: |
+  #     #       pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall
+  #     #       python -c "import geoarrow.rust.core"
+
+  #     # Have to set path from root
+  #     # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360
+  #     - name: Upload wheels
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: wheels
+  #         path: dist/*.whl
 
   macos:
-    runs-on: macos-latest
+    name: Build ${{ matrix.arch }} wheels on ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-11
+            arch: x86_64
+            target: x86_64
+            python-version: 3.8
+          - os: macos-14
+            arch: arm64
+            target: aarch64
+            # Earliest python version on Github for macos arm64
+            # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
+            python-version: 3.11.3
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-          architecture: x64
+          python-version: ${{ matrix.python-version }}
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - name: Build wheels - x86_64
+      - name: Build wheels - ${{ matrix.target }}
         uses: PyO3/maturin-action@v1
         with:
-          target: x86_64
+          target: ${{ matrix.target }}
           args: --release --out dist --sdist -m python/core/Cargo.toml
 
-      - name: Install built wheel - x86_64
-        run: |
-          pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall
-          python -c "import geoarrow.rust.core"
-
-      - name: Build wheels - universal2
-        uses: PyO3/maturin-action@v1
-        with:
-          target: universal2-apple-darwin
-          args: --release --out dist -m python/core/Cargo.toml
-
-      - name: Install built wheel - universal2
+      - name: Install built wheel - ${{ matrix.target }}
         run: |
           pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall
           python -c "import geoarrow.rust.core"
@@ -251,7 +257,7 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-    needs: [macos, windows, linux, linux-cross]
+    needs: [macos, windows, linux]
     steps:
       - uses: actions/download-artifact@v3
         with:

diff --git a/python/core/CHANGELOG.md b/python/core/CHANGELOG.md
@@ -4,37 +4,46 @@
 
 ### New! :sparkles:
 
-- New I/O support for reading and writing to geospatial formats:
-  - [Asynchronous FlatGeobuf reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_flatgeobuf_async). This also supports passing a spatial filter.
-  - [Initial support for reading from PostGIS](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_postgis). Note that not all Postgres data types are supported yet. Create an issue if your query fails.
-  - [Reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_geojson_lines) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_geojson_lines) [newline-delimited GeoJSON](https://stevage.github.io/ndgeojson/).
-  - [Pyogrio integration](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_pyogrio) for reading from OGR/GDAL.
-  - [Asynchronous GeoParquet reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_parquet_async) for reading from remote files. By @weiji14 in https://github.com/geoarrow/geoarrow-rs/pull/493
-  - Also new support for writing GeoParquet files.
-  - Most I/O readers and writers support Python file-like objects (in binary mode).
-  - Support for [reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_ipc) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_ipc) Arrow IPC files.
-- Better interoperability with the Python geospatial ecosystem.
-  - [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_geopandas) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_geopandas) GeoPandas GeoDataFrames. Refer to the [GeoPandas interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/geopandas/).
-  - [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_shapely) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_shapely) Shapely arrays. Refer to the [Shapely interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/shapely/).
-  - [Better integration with Lonboard](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/lonboard/) for fast visualization in Jupyter.
-  - All scalars, arrays, chunked arrays, and table objects implement [`__geo_interface__`](https://gist.github.com/sgillies/2217756) for interoperability with existing tools.
-  - Numpy interoperability for float array output from algorithms. Pass any `Float64Array` or `BooleanArray` to `numpy.asarray`. You can also pass a numpy array as vectorized input into a function like [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point).
-- New algorithms!
-  - [Explode a Table](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.explode) where each multi-geometry expands into multiple rows of single geometries.
-  - [`total_bounds`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.total_bounds)
-  - Unified [`area` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.area) for planar and geodesic area.
-  - Unified [`simplify` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.simplify) for multiple simplification methods. Also new support for [topology-preserving simplification](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/enums/#geoarrow.rust.core.enums.SimplifyMethod.VW_Preserve).
-  - Unified [`length` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.length) for euclidean and geodesic length calculations.
-  - [`frechet_distance`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.frechet_distance) for LineString similarity.
-  - [`affine_transform`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.affine_transform), which integrates with the Python [`affine`](https://github.com/rasterio/affine) library.
-  - [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point) and [`line_locate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_locate_point) for linear referencing.
-  - [`polylabel`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.polylabel) for polygon labeling.
-- Improved display of Python objects:
-  - Scalars now display as SVG geometries inside Jupyter environments.
-  - Tables, arrays, and chunked arrays implement `__repr__` so that you can inspect data easily.
-- Improved usability:
-  - [`PointArray.from_xy`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/geometry/array/#geoarrow.rust.core.PointArray.from_xy) to simplify creating a point array from numpy arrays of coordinates.
-  - Index into arrays and chunked arrays with square brackets. E.g. `point_array[0]` will access the first point in the array. Negative indexing is also supported, so `point_array[-1]` will get the last item in the array.
+#### New I/O support for reading and writing to geospatial formats
+
+- [Asynchronous FlatGeobuf reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_flatgeobuf_async). This also supports passing a spatial filter.
+- [Initial support for reading from PostGIS](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_postgis). Note that not all Postgres data types are supported yet. Create an issue if your query fails.
+- [Reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_geojson_lines) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_geojson_lines) [newline-delimited GeoJSON](https://stevage.github.io/ndgeojson/).
+- [Pyogrio integration](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_pyogrio) for reading from OGR/GDAL.
+- [Asynchronous GeoParquet reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_parquet_async) for reading from remote files. By @weiji14 in https://github.com/geoarrow/geoarrow-rs/pull/493
+- Also new support for writing GeoParquet files.
+- Most I/O readers and writers support Python file-like objects (in binary mode).
+- Support for [reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_ipc) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_ipc) Arrow IPC files.
+
+#### Better interoperability with the Python geospatial ecosystem.
+
+- [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_geopandas) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_geopandas) GeoPandas GeoDataFrames. Refer to the [GeoPandas interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/geopandas/).
+- [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_shapely) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_shapely) Shapely arrays. Refer to the [Shapely interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/shapely/).
+- [Better integration with Lonboard](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/lonboard/) for fast visualization in Jupyter.
+- All scalars, arrays, chunked arrays, and table objects implement [`__geo_interface__`](https://gist.github.com/sgillies/2217756) for interoperability with existing tools.
+- Numpy interoperability for float array output from algorithms. Pass any `Float64Array` or `BooleanArray` to `numpy.asarray`. You can also pass a numpy array as vectorized input into a function like [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point).
+
+#### New algorithms!
+
+- [Explode a Table](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.explode) where each multi-geometry expands into multiple rows of single geometries.
+- [`total_bounds`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.total_bounds)
+- Unified [`area` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.area) for planar and geodesic area.
+- Unified [`simplify` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.simplify) for multiple simplification methods. Also new support for [topology-preserving simplification](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/enums/#geoarrow.rust.core.enums.SimplifyMethod.VW_Preserve).
+- Unified [`length` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.length) for euclidean and geodesic length calculations.
+- [`frechet_distance`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.frechet_distance) for LineString similarity.
+- [`affine_transform`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.affine_transform), which integrates with the Python [`affine`](https://github.com/rasterio/affine) library.
+- [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point) and [`line_locate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_locate_point) for linear referencing.
+- [`polylabel`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.polylabel) for polygon labeling.
+
+#### Improved display of Python objects:
+
+- Scalars now display as SVG geometries inside Jupyter environments.
+- Tables, arrays, and chunked arrays implement `__repr__` so that you can inspect data easily.
+
+#### Improved usability:
+
+- [`PointArray.from_xy`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/geometry/array/#geoarrow.rust.core.PointArray.from_xy) to simplify creating a point array from numpy arrays of coordinates.
+- Index into arrays and chunked arrays with square brackets. E.g. `point_array[0]` will access the first point in the array. Negative indexing is also supported, so `point_array[-1]` will get the last item in the array.
 - New [top-level docs website](https://geoarrow.org/geoarrow-rs/).
 
 

diff --git a/python/core/Cargo.lock b/python/core/Cargo.lock
diff --git a/python/core/Cargo.toml b/python/core/Cargo.toml
@@ -55,5 +55,11 @@ tokio = { version = "1.9", features = ["rt"] }
 url = "2.5"
 
 # reqwest is pulled in by object store, but not used by python binding itself
-# for binary wheel best practice, statically link openssl
-reqwest = { version = "*", features = ["native-tls-vendored"] }
+# We statically link the rust-native rustls. For a binary wheel we want a
+# static-compiled openssl anyways, and having it be native rust seems to make
+# things simpler.
+# See native roots discussion here:
+# https://github.com/rustls/rustls-native-certs/blob/0ebeee01ed61ebeacd5ebb98e7f5694b0ee70a08/README.md#should-i-use-this-or-webpki-roots
+reqwest = { version = "*", features = ["rustls-tls-native-roots"] }
+# Compile openssl from source
+openssl = { version = "0.10", features = ["vendored"] }
diff --git a/python/core/src/io/input/sync.rs b/python/core/src/io/input/sync.rs
@@ -11,6 +11,7 @@ use pyo3::types::{PyBytes, PyString, PyType};
 use std::fs::File;
 use std::io::{self, BufRead, BufReader, BufWriter};
 use std::io::{Read, Seek, SeekFrom, Write};
+#[cfg(not(target_os = "windows"))]
 use std::os::fd::{AsRawFd, RawFd};
 use std::path::Path;
 
@@ -203,6 +204,7 @@ impl Seek for PyFileLikeObject {
     }
 }
 
+#[cfg(not(target_os = "windows"))]
 impl AsRawFd for PyFileLikeObject {
     fn as_raw_fd(&self) -> RawFd {
         Python::with_gil(|py| {