Python wrapper, API

.github/workflows/release.yml

@@ -9,7 +9,6 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  HUMANE_VERSION: "0.9.0"
   WASM_PACK_VERSION: "v0.10.3"
 
 jobs:
@@ -87,6 +86,77 @@ jobs:
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
+  publish-python-packages:
+    name: Publish python packages
+    runs-on: ubuntu-latest # ok since none of the scripts depend on version-specific features
+    defaults:
+      run:
+        shell: bash
+    needs: publish-github-release
+    permissions: # required for trusted publishing to pypi/test-pypi
+      id-token: write
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+      - name: Download CLI binaries
+        uses: actions/download-artifact@v4
+        with:
+          pattern: release-*
+          merge-multiple: true
+          path: ./wrappers/python/vendor
+      - name: Set up python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Set up poetry
+        run: ./wrappers/python/scripts/ci/github/setup_poetry.sh
+
+      - name: cache venv
+        uses: actions/cache@v4
+        with:
+          path: wrappers/python/.venv
+          key: ${{ runner.os }}-poetry-3.12-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dev dependencies
+        run: ./wrappers/python/scripts/ci/github/install_dev_dependencies.sh
+
+      - name: activate venv
+        run: ./wrappers/python/scripts/ci/github/activate_venv.sh
+
+      - name: debug python paths
+        run: ./wrappers/python/scripts/ci/github/debug_python_paths.sh
+
+      - name: package binaries
+        working-directory: ./wrappers/python
+        run: | # should take ~30s; writes wheels to wrappers/python/dist
+          export PAGEFIND_PYTHON_LOG_LEVEL=DEBUG
+          python -m scripts.build.all_binary_only_wheels \
+            --tag "${{ github.ref_name }}" \
+            --bin-dir ./vendor
+      - name: package python api
+        working-directory: ./wrappers/python
+        run: | # writes stdist + wheel to wrappers/python/dist
+          export PAGEFIND_PYTHON_LOG_LEVEL=DEBUG
+          python -m scripts.build.api_package --tag "${{ github.ref_name }}"
+      - name: Archive dist
+        uses: actions/upload-artifact@v4
+        with:
+          path: wrappers/python/dist
+          name: python-packages
+          if-no-files-found: error
+
+      - name: Publish to pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # Note: this action requires test-pypi / pypi trusted publishing to be
+        # configured in the target repository. For instructions, see
+        # - https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi
+        # - https://docs.pypi.org/trusted-publishers/
+        with: # see https://github.com/pypa/gh-action-pypi-publish/tree/release/v1/?tab=readme-ov-file#customizing-target-package-dists-directory
+          packages-dir: wrappers/python/dist
+          # repository-url: https://test.pypi.org/legacy/ # Use to publish test packages
+          verbose: true # can be commented out once the action is working as expected
+
   publish-npm-package:
     name: Publish NPM packages
     runs-on: ubuntu-20.04
@@ -321,35 +391,30 @@ jobs:
             os: windows-latest
             rust: stable
             target: x86_64-pc-windows-msvc
-            humane_target: x86_64-pc-windows-msvc
             cross: false
             run_tests: true
           - build: linux
             os: ubuntu-latest
             rust: stable
             target: x86_64-unknown-linux-musl
-            humane_target: x86_64-unknown-linux-musl
             cross: false
             run_tests: true
           - build: linux
             os: ubuntu-latest
             rust: stable
             target: aarch64-unknown-linux-musl
-            humane_target: x86_64-unknown-linux-musl
             cross: false
             run_tests: false
           - build: macos
             os: macos-latest
             rust: stable
             target: x86_64-apple-darwin
-            humane_target: x86_64-apple-darwin
             cross: false
             run_tests: true
           - build: macos-m1
             os: macos-latest
             rust: stable
             target: aarch64-apple-darwin
-            humane_target: x86_64-apple-darwin
             cross: false
             run_tests: false
     steps:
@@ -417,13 +482,10 @@ jobs:
         with:
           version: ${{env.WASM_PACK_VERSION}}
 
-      - name: Install humane
-        if: matrix.run_tests == true
-        uses: supplypike/setup-bin@v3
+      - name: Set up python 3.12
+        uses: actions/setup-python@v5
         with:
-          uri: "https://github.com/CloudCannon/humane/releases/download/v${{env.HUMANE_VERSION}}/humane-v${{env.HUMANE_VERSION}}-${{matrix.humane_target}}.tar.gz"
-          name: "humane"
-          version: ${{env.HUMANE_VERSION}}
+          python-version: "3.12"
 
       - name: Prepare Git
         run: |

.github/workflows/test.yml

@@ -65,6 +65,13 @@ jobs:
           default: true
           components: rustfmt, clippy
 
+      - name: Set up python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          # NOTE: ^this strategy leaves older python versions intentionally
+          # without test coverage to keep CI fast.
+
       - name: Check versions
         run: |
           cargo --version
@@ -111,3 +118,46 @@ jobs:
 
       - name: Test CLI
         run: ./test_ci.sh "release"
+
+      - name: Set up poetry
+        run: ./wrappers/python/scripts/ci/github/setup_poetry.sh
+
+      - name: cache venv
+        uses: actions/cache@v4
+        with:
+          path: wrappers/python/.venv
+          key: ${{ runner.os }}-poetry-3.12-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dev dependencies
+        run: ./wrappers/python/scripts/ci/github/install_dev_dependencies.sh
+
+      - name: activate venv
+        run: ./wrappers/python/scripts/ci/github/activate_venv.sh
+
+      - name: debug python paths
+        run: ./wrappers/python/scripts/ci/github/debug_python_paths.sh
+
+      - name: Lint python
+        # avoid duplicating linting work on different OSes
+        if: runner.os == 'Linux'
+        working-directory: ./wrappers/python
+        run: ./scripts/ci/python_lints.sh
+
+      - name: ensure cog up-to-date
+        # avoid duplicating linting work on different OSes
+        if: runner.os == 'Linux'
+        working-directory: ./wrappers/python
+        run: ./scripts/ci/cog/check.sh
+
+      - name: set PYTHONPATH
+        shell: bash
+        working-directory: ./wrappers/python
+        run: python ./scripts/ci/github/add_src_to_pythonpath.py
+      - name: add target/release to windows PATH
+        shell: bash
+        run: echo $PWD/target/release >> "$GITHUB_PATH"
+      - name: Test python API
+        timeout-minutes: 1
+        # ^ guard against the tests getting deadlock if the subprocess pipe
+        # gets clogged
+        run: ./wrappers/python/scripts/ci/github/integration_tests.sh

.gitignore

@@ -34,3 +34,11 @@ vendor
 
 # Node
 node_modules
+
+# Python
+__pycache__/
+*.pyc
+dist
+*.whl
+*.egg-info
+*.log

.shellcheckrc

@@ -0,0 +1,4 @@
+external-sources=true
+source-path=SCRIPTDIR
+disable=SC2002
+# SC2002: ignore "useless cat" warning: starting pipes with `cat` improves composability

.vscode/settings.json

@@ -1,3 +1,8 @@
 {
-    "rust-analyzer.showUnlinkedFileNotification": false
-}
+  "rust-analyzer.showUnlinkedFileNotification": false,
+  "python.analysis.typeCheckingMode": "standard",
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  }
+}

docs/content/docs/_index.md

@@ -30,15 +30,22 @@ Now build your site to an output directory — this guide assumes that you're ru
 
 ## Indexing your site
 
-The easiest way to run pagefind is through npx. If you don't have Node and npm installed, or want to install Pagefind another way, see the [Installing Pagefind](/docs/installation/) guide.
+The easiest way to run Pagefind is through one of the official wrapper packages. If you don't have Node or Python installed, or want to install Pagefind another way, see the [Installing Pagefind](/docs/installation/) guide.
 
-Run the following command from your terminal, where `--site` points to the output directory of your static site generator. We'll also add `--serve` so that we can view our final site right away.
+To use the Node wrapper, run the following command from your terminal, where `--site` points to the output directory of your static site generator. We'll also add `--serve` so that we can view our final site right away.
 
 ```bash
 npx -y pagefind --site public --serve
 ```
 
-You should see some output along the lines of:
+Using the Python wrapper is similar, but requires an initial install:
+
+```bash
+python3 -m pip install 'pagefind[extended]'
+python3 -m pagefind --site public --serve
+```
+
+Regardless of the command you choose, after Pagefind has downloaded you should see some output along the lines of:
 ```
 Indexed 2496 pages
 Indexed 22852 words

docs/content/docs/installation.md

@@ -5,25 +5,55 @@ nav_section: References
 weight: 49
 ---
 
-Pagefind is a static binary with no dynamic dependencies, so in most cases will be simple to install and run. Pagefind is currently supported on Windows, macOS, and x86-64 Linux distributions.
+Pagefind is a static binary with no dynamic dependencies, so in most cases will be simple to install and run. Pagefind is currently supported on Windows, macOS, and Linux distributions.
 
 ## Running via npx
 
+For users with a NodeJS toolchain already installed, Pagefind publishes a [wrapper package through npm](https://www.npmjs.com/package/pagefind):
+
 ```bash
 npx pagefind --site "public"
 ```
 
-Pagefind publishes a [wrapper package through npm](https://www.npmjs.com/package/pagefind), which is the easiest way to get started. This package will download the correct [binary of the latest release](https://github.com/CloudCannon/pagefind/releases) as an npm dependency for your platform and run it.
+This package includes the correct [binary of the relevant release](https://github.com/CloudCannon/pagefind/releases) as a dependency for your platform.
 
 Specific versions can be run by passing a version tag:
 
 ```bash
 npx pagefind@latest --site "public"
 
-npx pagefind@v0.2.0 --site "public"
+npx pagefind@v1.1.1 --site "public"
+```
+
+Running Pagefind via npx will always download the `pagefind_extended` release, which includes specialized support for indexing Chinese and Japanese pages.
+
+> Pagefind's npm package can also be imported and controlled from a script. See the [Node API documentation](/docs/node-api/) for details.
+
+## Running via Python
+
+For users with a Python toolchain already installed, Pagefind publishes a [wrapper package through pypi](https://pypi.org/project/pagefind/):
+
+```bash
+python3 -m pip install 'pagefind[extended]'
+python3 -m pagefind --site "public"
+```
+
+This package includes the correct [binary of the relevant release](https://github.com/CloudCannon/pagefind/releases) as a dependency for your platform.
+
+Specific versions can be installed by passing a version:
+
+```bash
+python3 -m pip install 'pagefind[extended]==1.1.1'
+```
+
+The above example shows installing the `pagefind_extended` release, which includes specialized support for indexing Chinese and Japanese pages.
+To install the smaller standard release, run:
+
+```bash
+python3 -m pip install 'pagefind[bin]'
 ```
 
-> Running Pagefind via npx will download the `pagefind_extended` release, which includes specialized support for indexing Chinese and Japanese pages.
+> Pagefind's Python package can also be imported and controlled from a script. See the [Python API documentation](/docs/py-api/) for details.
 
 ## Downloading a precompiled binary