diff --git a/MANIFEST.in b/MANIFEST.in
index 4d9dbcb3269..7ac5fdc69ff 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,7 +5,6 @@ include yt/visualization/mapserver/html/Leaflet.Coordinates-0.1.5.css
include yt/visualization/mapserver/html/Leaflet.Coordinates-0.1.5.src.js
include yt/utilities/tests/cosmology_answers.yml
include yt/utilities/mesh_types.yaml
-exclude scripts/pr_backport.py
exclude yt/utilities/lib/cykdtree/c_kdtree.cpp
prune docker
prune tests
diff --git a/doc/source/developing/deprecating_features.rst b/doc/source/developing/deprecating_features.rst
index 03f22141920..48368e6e929 100644
--- a/doc/source/developing/deprecating_features.rst
+++ b/doc/source/developing/deprecating_features.rst
@@ -1,3 +1,5 @@
+.. how-to-deprecate::
+
How to deprecate a feature
--------------------------
@@ -7,10 +9,10 @@ A functionality can be marked as deprecated using
message and two version numbers, indicating the earliest release deprecating the feature
and the one in which it will be removed completely.
-The message should indicate a viable alternative to replace the deprecated feature at
-the user level.
-``since`` and ``removal`` are required [#]_ keyword-only arguments so as to enforce
-readability of the source code.
+The message should indicate a viable alternative to replace the deprecated
+feature at the user level. ``since`` and ``removal`` arguments should indicate
+in which release something was first deprecated, and when it's expected to be
+removed. While ``since`` is required, ``removal`` is optional.
Here's an example call.
@@ -30,8 +32,6 @@ If a whole function or class is marked as deprecated, it should be removed from
``doc/source/reference/api/api.rst``.
-.. [#] ``since`` is not required yet as of yt 4.0.0 because existing warnings predate its introduction.
-
Deprecating Derived Fields
--------------------------
diff --git a/doc/source/developing/releasing.rst b/doc/source/developing/releasing.rst
index 5325f849ac4..2008a63a33f 100644
--- a/doc/source/developing/releasing.rst
+++ b/doc/source/developing/releasing.rst
@@ -1,19 +1,19 @@
How to Do a Release
-------------------
-Periodically, the yt development community issues new releases. Since yt follows
-`semantic versioning `_, the type of release can be read off
+Periodically, the yt development community issues new releases. yt loosely follows
+`semantic versioning `_. The type of release can be read off
from the version number used. Version numbers should follow the scheme
``MAJOR.MINOR.PATCH``. There are three kinds of possible releases:
* Bugfix releases
- These releases are regularly scheduled and will optimally happen approximately
- once a month. These releases should contain only fixes for bugs discovered in
+ These releases should contain only fixes for bugs discovered in
earlier releases and should not contain new features or API changes. Bugfix
- releases should increment the ``PATCH`` version number. Bugfix releases should
+ releases only increment the ``PATCH`` version number. Bugfix releases should
*not* be generated by merging from the ``main`` branch, instead bugfix pull
- requests should be manually backported. Version ``3.2.2`` is a bugfix release.
+ requests should be backported to a dedicated branch.
+ See :ref:`doing-a-bugfix-release`. Version ``3.2.2`` is a bugfix release.
* Minor releases
@@ -31,7 +31,7 @@ from the version number used. Version numbers should follow the scheme
* Major releases
These releases happen when the development community decides to make major
- backwards-incompatible changes. In principle a major version release could
+ backwards-incompatible changes intentionally. In principle a major version release could
include arbitrary changes to the library. Major version releases should only
happen after extensive discussion and vetting among the developer and user
community. Like minor releases, a major release should happen by merging the
@@ -42,6 +42,9 @@ from the version number used. Version numbers should follow the scheme
The job of doing a release differs depending on the kind of release. Below, we
describe the necessary steps for each kind of release in detail.
+
+.. _doing-a-bugfix-release:
+
Doing a Bugfix Release
~~~~~~~~~~~~~~~~~~~~~~
@@ -49,29 +52,41 @@ As described above, bugfix releases are regularly scheduled updates for minor
releases to ensure fixes for bugs make their way out to users in a timely
manner. Since bugfix releases should not include new features, we do not issue
bugfix releases by simply merging from the development ``main`` branch into
-the ``stable`` branch. Instead, we manually cherry-pick bugfixes from the from
-``main`` branch onto the ``stable`` branch.
-
-You may find the ``pr_backport.py`` script located in the ``scripts`` folder at
-the root of the repository to be helpful. This script uses the github API to
-find the list of pull requests made since the last release and prompts the user
-to backport each pull request individually. Note that the backport process is
-fully manual. The easiest way to do it is to download the diff for the pull
-request (the URL for the diff is printed out by the backport script) and then
-use ``git apply`` to apply the patch for the pull request to a local copy of yt
-with the ``stable`` branch checked out.
-
-Once you've finished backporting push your work to Github. Once you've pushed to
-your fork, you will be able to issue a pull request containing the backported
-fixes just like any other yt pull request.
+the ``stable`` branch. Instead, commits are cherry-picked from the ``main``
+branch to a backport branch, which is itself merged into ``stable`` when a release
+happens.
+
+Backport branches are named after the minor version then descend from, followed by
+an ``x``. For instance, ``yt-4.0.x`` is the backport branch for all releases in the 4.0 series.
+
+Backporting bugfixes can be done automatically using the `MeeseeksBox bot
+`_.
+
+This necessitates having a Github milestone dedicated to the release, configured
+with a comment in its description such as ``on-merge: backport to yt-4.0.x``.
+Then, every PR that was triaged into the milestone will be replicated as a
+backport PR by the bot when it's merged into main. Some backports are non-trivial and
+require human attention; if conflicts occur, the bot will provide detailed
+instructions to perfom the task manually.
+
+In short, a manual backport consist of 4 steps
+
+- checking out the backport branch locally
+- create a new branch from there
+- cherry-picking the merge commit from the original PR with ``git cherry-pick -m1 ``
+- opening a PR to the backport branch
+
Doing a Minor or Major Release
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This is much simpler than a bugfix release. All that needs to happen is the
-``main`` branch must get merged into the ``stable`` branch, and any conflicts
-that happen must be resolved, almost always in favor of the state of the code on
-the ``main`` branch.
+This is much simpler than a bugfix release. First, make sure that every
+deprecated features targeted for removal in the new release are removed from the
+``main`` branch, ideally in a single PR. Such a PR can be issued at any point
+between the previous minor or major release and the new one. Then, all that
+needs to happen is the ``main`` branch must get merged into the ``stable``
+branch, and any conflicts that happen must be resolved, almost always in favor
+of the state of the code on the ``main`` branch.
Incrementing Version Numbers and Tagging a Release
@@ -111,7 +126,7 @@ Where ```` follows the project's naming scheme for tags
(e.g. ``yt-3.2.1``). Once you are done, you will need to push the
tag to github::
- git push origin --tags
+ git push origin --tag
This assumes that you have configured the remote ``origin`` to point at the main
yt git repository. If you are doing a minor or major version number release, you
@@ -148,67 +163,49 @@ https://yt-project.org/sdist, like so::
You may find it helpful to set up an ssh config for dickenson to make this
command a bit easier to execute.
-Updating conda-forge and building wheels
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Before we finish the release, we need to generate new binary builds by updating
-yt's conda-forge feedstock and the yt-wheels repository.
-
-Wheels and ``multibuild``
-+++++++++++++++++++++++++
-
-Binary wheels for yt are managed via the ``multibuild`` project. For yt the main
-point of access is at https://github.com/yt-project/yt-wheels. Take a look at
-the pull requests from the previous few releases to get an idea of what to do,
-but briefly you will need to update the multibuild and yt submodules to their
-latest state and then commit the changes to the submodules::
-
- $ cd multibuild
- $ git pull origin devel
- $ cd ../yt
- $ git pull origin stable
- $ cd ..
- $ git commit -am "updating multibuild and yt submodules"
-
-Next you will need to update the ``.travis.yml`` and ``appveyor.yaml`` files to
-build the latest tag of yt. You may also need to update elsewhere in the file if
-yt's dependencies changed or if yt dropped or added support for a Python
-version. To generate new wheels you need to push the changes to GitHub. A good
-process to follow is to first submit a pull request to test the changes and make sure
-the wheels can be built. Once they pass, you can merge the changes into ``main``
-and wait for the wheel files to be uploaded to
-https://anaconda.org/multibuild-wheels-staging/yt/files
-(note that the wheels will not be uploaded until the changes have been
-merged into ``main``). Once the wheels are uploaded, download the
-wheel files for the release and copy them to the ``dist`` folder in the yt
-repository so that they are sitting next to the source distribution
-we created earlier. Here's a
-one-liner to download all of the wheels for the yt 3.6.1 release::
-
- $ wget -r -nd -A 'yt-3.6.1-*whl' https://anaconda.org/multibuild-wheels-staging/yt/files
-
-Uploading to PyPI
-+++++++++++++++++
-
-To actually upload the release to the Python Package Index, you just need to
-issue the following command:
+Publishing
+~~~~~~~~~~
+
+We distribute yt on two main channels: PyPI.org and conda-forge, in this order.
+
+PyPI
+++++
+
+The publication process for PyPI is automated for the most part, via Github
+actions, using ``.github/workflows/wheels.yaml``. Specifically, a release is
+pushed to PyPI when a new git tag starting with ``yt-`` is pushed to the main
+repo. Let's review the details here.
+
+PyPI releases contain the source code (as a tarball), and wheels. Wheels are
+compiled distributions of the source code. They are OS specific as well as
+Python-version specific. Producing wheels for every supported combination of OS
+and Python versions is done with `cibuildwheels
+`_
+
+Upload to PyPI is automated via Github Actions `upload-artifact
+`_ and `download-artifact
+`_.
+
+Note that automated uploads are currently perfomed using Matt Turk's
+credentials.
+
+If that worked, you can skip to the next section. Otherwise, upload can be
+perfomed manually by first downloading the artifacts ``wheels`` and ``tarball``
+from the workflow webpage, then at the command line (make sure that the
+``dist`` directory doesn't exist or is empty)
.. code-block:: bash
+ unzip tarball.zip -d dist
+ unzip wheels.zip -d dist
+ python -m pip install --upgrade twine
twine upload dist/*
-Please ensure that both the source distribution and binary wheels are present in
-the ``dist`` folder before doing this. Directions on generating binary wheels
-are described in the section immediately preceding this one.
-
You will be prompted for your PyPI credentials and then the package should
upload. Note that for this to complete successfully, you will need an account on
PyPI and that account will need to be registered as an "owner" or "maintainer"
of the yt package.
-Right now the following people have access to upload packages: Matt Turk,
-Britton Smith, Nathan Goldbaum, John ZuHone, Kacper Kowalik, and Madicken Munk.
-The yt package source distribution should be uploaded along with compiled
-binary wheel packages for various platforms that we support.
+
``conda-forge``
+++++++++++++++
@@ -217,9 +214,11 @@ Conda-forge packages for yt are managed via the yt feedstock, located at
https://github.com/conda-forge/yt-feedstock. When a release is pushed to PyPI a
bot should detect a new version and issue a PR to the feedstock with the new
version automatically. When this feedstock is updated, make sure that the
-SHA256 hash of the tarball matches the one you uploaded to dickenson and that
+SHA256 hash of the tarball matches the one you uploaded to PyPI and that
the version number matches the one that is being released.
+In case the automated PR fails CI, feedstock maintainers are allowed to push to
+the bot's branch with any fixes required.
Should you need to update the feedstock manually, you will
need to update the ``meta.yaml`` file located in the ``recipe`` folder in the
@@ -237,3 +236,8 @@ After the release is uploaded to `PyPI `_ an
you should send out an announcement
e-mail to the yt mailing lists as well as other possibly interested mailing
lists for all but bugfix releases.
+
+Creating a Github release attached to the tag also offers a couple advantages.
+Auto-generated release notes can be a good starting point, though it's best to
+edit out PRs that not directly affecting users, and these notes can be edited
+before (draft mode) and after the release, so errors can be corrected after the fact.
diff --git a/scripts/pr_backport.py b/scripts/pr_backport.py
deleted file mode 100644
index 130e02b58c7..00000000000
--- a/scripts/pr_backport.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import shutil
-import tempfile
-
-import dateutil.parser
-import git
-import requests
-
-API_URL = "https://api.github.com/graphql"
-
-YT_REPO = "https://github.com/yt-project/yt"
-
-PR_QUERY = """
-{
- repository(owner: "yt-project", name: "yt") {
- pullRequests(first:100 states:MERGED %s) {
- edges {
- node {
- number
- title
- mergedAt
- author {
- login
- }
- body
- url
- }
- }
- pageInfo {
- endCursor
- hasNextPage
- hasPreviousPage
- startCursor
- }
- }
- }
-}
-"""
-
-
-def clone_new_repo(source=None):
- """Clones a new copy of yt_analysis/yt and returns a path to it"""
- path = tempfile.mkdtemp()
- dest_repo_path = path + "/yt-backport"
- if source is None:
- source = YT_REPO
- git.Repo.clone_from(source, dest_repo_path)
- return dest_repo_path
-
-
-def get_date_of_last_tag(repo_path):
- repo = git.Repo(repo_path)
- tags = sorted(repo.tags, key=lambda t: t.commit.committed_date)
- return tags[-1].commit.committed_date
-
-
-def get_prs_since_last_release(date, key):
- headers = {"Authorization": "token %s" % key}
- resp = requests.post(url=API_URL, json={"query": PR_QUERY % ""}, headers=headers)
- ret = []
- while True:
- jsr = resp.json()
- cursor = jsr["data"]["repository"]["pullRequests"]["pageInfo"]["endCursor"]
- if cursor is None:
- break
- prs = jsr["data"]["repository"]["pullRequests"]["edges"]
- for pr in prs:
- pr_date = dateutil.parser.parse(pr["node"]["mergedAt"]).timestamp()
- if pr_date > date:
- ret.append(pr["node"])
- resp = requests.post(
- url=API_URL,
- json={"query": PR_QUERY % ('after:"%s"' % cursor)},
- headers=headers,
- )
- return ret
-
-
-def backport_prs(repo_path, prs):
- for pr in prs:
- print("")
- print("PR %s" % pr["number"])
- print(pr["title"])
- print(pr["author"]["login"])
- print(pr["body"])
- print(pr["url"])
- print("%s.diff" % pr["url"])
- input("Press any key to continue")
-
-
-if __name__ == "__main__":
- key = input(
- "Please enter your github OAuth API key\n"
- "See the github help for instructions on how to "
- "generate a personal access token.\n>>> "
- )
- print("")
- print("Gathering PR information, this may take a minute.")
- print("Don't worry, yt loves you.")
- print("")
- repo_path = clone_new_repo()
- try:
- date = get_date_of_last_tag(repo_path)
- prs = get_prs_since_last_release(date, key)
- print("In another terminal window, navigate to the following path:")
- print("%s" % repo_path)
- input("Press any key to continue")
- backport_prs(repo_path, prs)
- input(
- "Now you need to push your backported changes. The temporary\n"
- "repository currently being used will be deleted as soon as you\n"
- "press any key."
- )
- finally:
- shutil.rmtree(repo_path)