Merge pull request xapi-project#5896 from xapi-project/feature/py3

Python3 update feature merge
edwintorok · Sep 4, 2024 · 59664cc · 59664cc
2 parents b1fbddc + 09603bd
commit 59664cc
Show file tree

Hide file tree

Showing 173 changed files with 5,293 additions and 8,439 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -45,7 +45,6 @@ jobs:
       - name: Make install smoketest
         run: |
           opam exec -- make install DESTDIR=$(mktemp -d)
-          opam exec -- make install DESTDIR=$(mktemp -d) BUILD_PY2=NO
 
       - name: Check disk space
         run: df -h || true

diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["2.7", "3.11"]
+        python-version: ["3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -39,48 +39,11 @@ jobs:
 
       - uses: pre-commit/action@v3.0.1
         name: Run pre-commit checks (no spaces at end of lines, etc)
-        if: ${{ matrix.python-version != '2.7' }}
         with:
           extra_args: --all-files --verbose --hook-stage commit
         env:
           SKIP: no-commit-to-branch
 
-      - name: Install dependencies only needed for python 2
-        if: ${{ matrix.python-version == '2.7' }}
-        run: pip install enum
-
-      - name: Install dependencies only needed for python 3
-        if: ${{ matrix.python-version != '2.7' }}
-        run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
-
-      - name: Install common dependencies for Python ${{matrix.python-version}}
-        run: pip install future mock pytest-coverage pytest-mock
-
-      - name: Run Pytest for python 2 and get code coverage
-        if: ${{ matrix.python-version == '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd
-          scripts/ ocaml/xcp-rrdd -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-
-      - name: Run Pytest for python 3 and get code coverage
-        if: ${{ matrix.python-version != '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
-          scripts/ ocaml/xcp-rrdd python3/ -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-          PYTHONPATH: "python3:python3/tests/stubs"
-
       - name: Upload coverage report to Coveralls
         uses: coverallsapp/github-action@v2
         with:
@@ -90,7 +53,6 @@ jobs:
           parallel: true
 
       - uses: dciborow/action-pylint@0.1.0
-        if: ${{ matrix.python-version != '2.7' }}
         with:
           reporter: github-pr-review
           level: warning
@@ -99,8 +61,7 @@ jobs:
         continue-on-error: true
 
       - name: Run pytype checks
-        if: ${{ matrix.python-version != '2.7' }}
-        run: ./pytype_reporter.py
+        run: pip install pandas pytype toml && ./pytype_reporter.py
         env:
           PR_NUMBER: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: XenAPI
-          path: scripts/examples/python/dist/
+          path: python3/examples/dist/
 
   build-sdks:
     name: Build and upload SDK artifacts

diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml
@@ -28,9 +28,9 @@ runs:
       shell: bash
       run: sudo apt-get update
 
-    - name: Install python2
+    - name: Install python3
       shell: bash
-      run: sudo apt-get install python2
+      run: sudo apt-get install python3
 
     - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME
       shell: bash

diff --git a/.gitignore b/.gitignore
@@ -16,10 +16,10 @@ config.mk
 # python packaging
 **/__pycache__/
 **/*.pyc
-scripts/examples/python/setup.py
-scripts/examples/python/XenAPI.egg-info/
-scripts/examples/python/build/
-scripts/examples/python/dist/
+python3/examples/setup.py
+python3/examples/XenAPI.egg-info/
+python3/examples/build/
+python3/examples/dist/
 
 # ignore file needed for building the SDK
 ocaml/sdk-gen/csharp/XE_SR_ERRORCODES.xml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,6 +14,8 @@
 # pre-commit run -av --hook-stage pre-push
 #
 default_stages: [commit, push]
+default_language_version:
+    python: python3.11
 repos:
 # Recommendation for a minimal git pre-commit hook:
 # https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md:
@@ -29,6 +31,95 @@ repos:
     -   id: check-executables-have-shebangs
         exclude: ocaml
 
+
+# Improve Python formatting incrementally:
+# https://dev.to/akaihola/improving-python-code-incrementally-3f7a
+#
+# darker checks if staged python changes are formatted according using
+# the PEP8-aligned black formatter. It also checks if the imports are sorted.
+#
+# It is a good idea to run this before committing, and it is also run in the
+# GitHub Workflow.
+#
+# Note: darker only checks the changes in files ending in .py!
+# Python scripts that don't end in .py should be renamed to have the .py extension
+# when moving them to python3/bin.
+# (remove the .py extension in the Makefile when installing the file)
+#
+-   repo: https://github.com/akaihola/darker
+    rev: 1.7.3
+    hooks:
+    -   id: darker
+        files: python3/
+        name: check changes in Python3 tree using darker and isort
+        args: [--diff, --skip-string-normalization, --isort, -tpy36]
+        additional_dependencies: [isort]
+
+#
+# Run pytest and diff-cover to check that the new /python3 test suite in passes.
+# This hook uses a local venv containing the required dependencies. When adding
+# new dependencies, they should be added to the additional_dependencies below.
+#
+-   repo: local
+    hooks:
+    -   id: pytest
+        files: python3/
+        name: check that the Python3 test suite in passes
+        entry: env PYTHONDEVMODE=yes sh -c 'coverage run && coverage xml &&
+            coverage html && coverage report &&
+            diff-cover --ignore-whitespace --compare-branch=origin/feature/py3
+            --show-uncovered --html-report .git/coverage-diff.html
+            --fail-under 50 .git/coverage3.11.xml'
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies:
+        - coverage
+        - diff-cover
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest-mock
+        - mock
+        - wrapt
+        - XenAPI
+
+
+-   repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.372
+    hooks:
+    -   id: pyright
+        name: check that python3 tree passes pyright/VSCode check
+        files: python3/
+        additional_dependencies:
+        - mock
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest
+        - pyudev
+        - XenAPI
+
+
+# Check that pylint passes for the changes in new /python3 code.
+-   repo: local
+    hooks:
+    -   id: pylint
+        files: python3/
+        stages: [push]
+        name: check that changes to python3 tree pass pylint
+        entry: diff-quality --violations=pylint
+            --ignore-whitespace --compare-branch=origin/feature/py3
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [diff-cover, pylint, pytest]
+
+
+# pre-push hook (it only runs if you install pre-commit as a pre-push hook):
+# It can be manually tested using: `pre-commit run -av --hook-stage push`
 # Recommendation for a minimal git pre-push hook:
 # While using pre-commit yields great results, it
 # is "not fast". Therefore only run it pre-push,
@@ -53,4 +144,12 @@ repos:
         # developers have such version installed, it can be configured here:
         # language_version: python3.11
         require_serial: true
-        additional_dependencies: [pandas, pytype]
+        additional_dependencies:
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pandas
+        - pytest
+        - pytype
+        files: python3/
diff --git a/Makefile b/Makefile
@@ -68,9 +68,7 @@ test:
 	 trap "kill $${PSTREE_SLEEP_PID}" INT TERM EXIT; \
 	 timeout --foreground $(TEST_TIMEOUT2) \
 		 dune runtest --profile=$(PROFILE) --error-reporting=twice -j $(JOBS)
-ifneq ($(PY_TEST), NO)
 	dune build @runtest-python --profile=$(PROFILE)
-endif
 
 stresstest:
 	dune build @stresstest --profile=$(PROFILE) --no-buffer -j $(JOBS)
@@ -123,7 +121,7 @@ sdk:
 	cp -r _build/default/ocaml/sdk-gen/java/autogen/* $(XAPISDK)/java
 	cp -r _build/default/ocaml/sdk-gen/powershell/autogen/* $(XAPISDK)/powershell
 	cp -r _build/default/ocaml/sdk-gen/go/autogen/* $(XAPISDK)/go
-	cp scripts/examples/python/XenAPI/XenAPI.py $(XAPISDK)/python
+	cp python3/examples/XenAPI/XenAPI.py $(XAPISDK)/python
 	sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp
 	sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell
 
@@ -138,7 +136,7 @@ sdk-build-java: sdk
 	cd _build/install/default/xapi/sdk/java && mvn -f xen-api/pom.xml -B clean package install -Drevision=0.0
 
 python:
-	$(MAKE) -C scripts/examples/python build
+	$(MAKE) -C python3/examples build
 
 doc-json:
 	dune exec --profile=$(PROFILE) -- ocaml/idl/json_backend/gen_json.exe -destdir $(XAPIDOC)/jekyll
@@ -239,7 +237,6 @@ install: build doc sdk doc-json
 	install -D ./ocaml/xenopsd/scripts/block $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/block
 	install -D ./ocaml/xenopsd/scripts/xen-backend.rules $(DESTDIR)/$(ETCDIR)/udev/rules.d/xen-backend.rules
 	install -D ./ocaml/xenopsd/scripts/tap $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/tap
-	install -D ./ocaml/xenopsd/scripts/qemu-vif-script $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/qemu-vif-script
 	install -D ./ocaml/xenopsd/scripts/setup-vif-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-vif-rules
 	install -D ./_build/install/default/bin/pvs-proxy-ovs-setup $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/pvs-proxy-ovs-setup
 	(cd $(DESTDIR)/$(XENOPSD_LIBEXECDIR) && ln -s pvs-proxy-ovs-setup setup-pvs-proxy-rules)