Build 3.12

.github/workflows/ci.yml

@@ -12,12 +12,12 @@ jobs:
     name: "Python ${{ matrix.python-version }}"
     runs-on: "ubuntu-latest"
     env:
-      USING_COVERAGE: '3.8,3.9,3.10'
+      USING_COVERAGE: '3.8,3.9,3.10,3.11,3.12'
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: "actions/checkout@v3"
@@ -54,7 +54,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: "Install dependencies"
         run: |
           set -xe

MANIFEST.in

@@ -8,6 +8,7 @@ recursive-include papermill *.txt
 
 include setup.py
 include requirements.txt
+include tox_py_installer
 recursive-include requirements *.txt
 include tox.ini
 include pytest.ini
@@ -29,6 +30,9 @@ prune binder
 graft scripts
 # Test env
 prune .tox
+# Exclude notebooks checkpoints generated by testing
+recursive-exclude papermill/.ipynb_checkpoints *.ipynb
+recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb
 
 # Build files
 exclude .github

binder/cli-simple/simple_output.ipynb

@@ -2,12 +2,13 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "df2b0bde",
    "metadata": {
     "papermill": {
-     "duration": 0.00964,
-     "end_time": "2020-08-17T19:57:24.239226",
+     "duration": 0.003083,
+     "end_time": "2023-11-01T18:34:41.235066",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.229586",
+     "start_time": "2023-11-01T18:34:41.231983",
      "status": "completed"
     },
     "tags": []
@@ -19,18 +20,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "0d7af2f3",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.263404Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.262659Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.265395Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.264689Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.301099Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.300307Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.306171Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.305038Z"
     },
     "papermill": {
-     "duration": 0.012719,
-     "end_time": "2020-08-17T19:57:24.265567",
+     "duration": 0.012465,
+     "end_time": "2023-11-01T18:34:41.308519",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.252848",
+     "start_time": "2023-11-01T18:34:41.296054",
      "status": "completed"
     },
     "tags": [
@@ -45,18 +47,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "id": "a5d4ad35",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.274405Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.273934Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.276194Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.275695Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.314681Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.314338Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.318662Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.317802Z"
     },
     "papermill": {
-     "duration": 0.007713,
-     "end_time": "2020-08-17T19:57:24.276300",
+     "duration": 0.010058,
+     "end_time": "2023-11-01T18:34:41.320883",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.268587",
+     "start_time": "2023-11-01T18:34:41.310825",
      "status": "completed"
     },
     "tags": [
@@ -72,18 +75,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
+   "id": "99512b79",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.283119Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.282711Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.284846Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.284482Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.327359Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.326530Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.333035Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.331879Z"
     },
     "papermill": {
-     "duration": 0.006366,
-     "end_time": "2020-08-17T19:57:24.284929",
+     "duration": 0.012117,
+     "end_time": "2023-11-01T18:34:41.335151",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.278563",
+     "start_time": "2023-11-01T18:34:41.323034",
      "status": "completed"
     },
     "tags": []
@@ -120,22 +124,23 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.12.0"
   },
   "papermill": {
-   "duration": 0.727728,
-   "end_time": "2020-08-17T19:57:24.494284",
+   "default_parameters": {},
+   "duration": 1.612177,
+   "end_time": "2023-11-01T18:34:41.659543",
    "environment_variables": {},
    "exception": null,
    "input_path": "binder/cli-simple/simple_input.ipynb",
    "output_path": "binder/cli-simple/simple_output.ipynb",
    "parameters": {
     "msg": "Hello"
    },
-   "start_time": "2020-08-17T19:57:23.766556",
-   "version": "2.1.3"
+   "start_time": "2023-11-01T18:34:40.047366",
+   "version": "2.4.0"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
-}
+ "nbformat_minor": 5
+}

docs/conf.py

@@ -73,7 +73,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line foexitr these cases.
-language = None
+language = 'python'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -163,4 +163,4 @@
 ]
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/', None)}

docs/requirements.txt

@@ -1,8 +1,8 @@
 # Pin packages for RTD builds
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies
--r ../requirements.txt
-Sphinx==3.5.4
-furo==2021.4.11b34
-myst-parser==0.13.7
-moto==2.0.5
-sphinx-copybutton==0.3.1
+#-r ../requirements.txt
+Sphinx>=3.5.4
+furo>=2021.4.11b34
+myst-parser>=0.13.7
+moto>=2.0.5
+sphinx-copybutton>=0.3.1

papermill/cli.py

@@ -201,6 +201,10 @@ def papermill(
     from stdin and write it out to stdout.
 
     """
+    # Jupyter deps use frozen modules, so we disable the python 3.11+ warning about debugger if running the CLI
+    if 'PYDEVD_DISABLE_FILE_VALIDATION' not in os.environ:
+        os.environ['PYDEVD_DISABLE_FILE_VALIDATION'] = '1'
+
     if not help_notebook:
         required_output_path = not (INPUT_PIPED or OUTPUT_PIPED)
         if required_output_path and not output_path:

papermill/tests/test_hdfs.py

@@ -1,6 +1,8 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from ..iorw import HDFSHandler
 
 
@@ -38,6 +40,7 @@ def __init__(self, path):
         self.path = path
 
 
+@pytest.mark.skip(reason="No valid dep package for python 3.12 yet")
 @patch('papermill.iorw.HadoopFileSystem', side_effect=MockHadoopFileSystem())
 class HDFSTest(unittest.TestCase):
     def setUp(self):

requirements.txt

@@ -6,3 +6,4 @@ tqdm >= 4.32.2
 requests
 entrypoints
 tenacity >= 5.0.2
+aiohttp==3.9.0b0;python_version=="3.12"

setup.py

@@ -44,9 +44,12 @@ def read_reqs(fname, folder=None):
 gcs_reqs = read_reqs('gcs.txt', folder='requirements')
 hdfs_reqs = read_reqs('hdfs.txt', folder='requirements')
 github_reqs = read_reqs('github.txt', folder='requirements')
+docs_only_reqs = read_reqs('requirements.txt', folder='docs')
 black_reqs = ['black >= 19.3b0']
 all_reqs = s3_reqs + azure_reqs + gcs_reqs + hdfs_reqs + black_reqs
-dev_reqs = read_reqs('dev.txt', folder='requirements') + all_reqs
+docs_reqs = all_reqs + docs_only_reqs
+# Temporarily remove hdfs_reqs from dev deps until the pyarrow package is available for Python 3.12
+dev_reqs = read_reqs('dev.txt', folder='requirements') + s3_reqs + azure_reqs + gcs_reqs + black_reqs # all_reqs
 extras_require = {
     "test": dev_reqs,
     "dev": dev_reqs,
@@ -57,6 +60,7 @@ def read_reqs(fname, folder=None):
     "hdfs": hdfs_reqs,
     "github": github_reqs,
     "black": black_reqs,
+    "docs": docs_reqs,
 }
 
 # Get the long description from the README file
@@ -97,5 +101,6 @@ def read_reqs(fname, folder=None):
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
     ],
 )

tox.ini

@@ -1,13 +1,14 @@
 [tox]
 skipsdist = true
-envlist = py{37,38,39,310}, flake8, dist, manifest, docs, binder
+envlist = py{38,39,310,311,312}, flake8, dist, manifest, docs, binder
 
 [gh-actions]
 python =
-    3.7: py37
-    3.8: py38, dist
+    3.8: py38
     3.9: py39
     3.10: py310
+    3.11: py311
+    3.12: py312, dist
 
 # Linters
 [testenv:flake8]
@@ -27,9 +28,7 @@ ignore =
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs
 deps =
-    .[dev]
-    .[azure]
-    -r docs/requirements.txt
+    .[docs]
 extras = docs
 commands =
     sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs}
@@ -45,7 +44,6 @@ commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(
 # Distro
 [testenv:dist]
 skip_install = true
-# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line...
 commands =
     python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir}
     /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl'
@@ -54,7 +52,7 @@ commands =
 # Black
 [testenv:black]
 description = apply black linter with desired rules
-basepython = python3.8
+basepython = python3.12
 deps =
     black
 commands = black -S -l 100 .
@@ -67,15 +65,20 @@ setenv =
     AWS_SECRET_ACCESS_KEY=foobar_secret
 passenv = *
 basepython =
-    py37: python3.7
     py38: python3.8
     py39: python3.9
     py310: python3.10
-    flake8: python3.8
-    manifest: python3.8
-    dist: python3.8
-    docs: python3.8
-    binder: python3.8
-    black: python3.8
+    py311: python3.11
+    py312: python3.12
+    flake8: python3.12
+    manifest: python3.12
+    dist: python3.12
+    docs: python3.12
+    binder: python3.12
+    black: python3.12
 deps = .[dev]
+# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line...
+allowlist_externals = /bin/bash
+# Python 3.12 breaks default pip/setuptools versions ... force an upgrade of these before anything else
+install_command = /bin/bash ./tox_py_installer {opts} {packages}
 commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs}

tox_py_installer

@@ -0,0 +1,5 @@
+#!/bin/bash
+python -m ensurepip --upgrade
+python -m pip install --upgrade setuptools
+# python -m pip install {opts} {packages}
+python -m pip install $1 $2