From cc6655017032dcceb1992f28cb9222b62ed7ef55 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:01:36 +0100 Subject: [PATCH 1/3] add build-pgo make target --- Makefile | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 122eca7cf..91d907607 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ black = black python/pydantic_core tests generate_self_schema.py wasm-preview/ru ruff = ruff python/pydantic_core tests generate_self_schema.py wasm-preview/run_tests.py mypy-stubtest = python -m mypy.stubtest pydantic_core._pydantic_core --allowlist .mypy-stubtest-allowlist +# using pip install cargo (via maturin via pip) doesn't get the tty handle +# so doesn't render color without some help +export CARGO_TERM_COLOR=$(shell (test -t 0 && echo "always") || echo "auto") + .PHONY: install install: pip install -U pip wheel pre-commit @@ -16,23 +20,35 @@ install-rust-coverage: cargo install rustfilt coverage-prepare rustup component add llvm-tools-preview +.PHONY: install-pgo + rustup component add llvm-tools-preview + .PHONY: build-dev build-dev: @rm -f python/pydantic_core/*.so - cargo build --features extension-module - @rm -f target/debug/lib_pydantic_core.d - @rm -f target/debug/lib_pydantic_core.rlib - @mv target/debug/lib_pydantic_core.* python/pydantic_core/_pydantic_core.so + pip install -v -e . --config-settings=build-args='--profile dev' .PHONY: build-prod build-prod: @rm -f python/pydantic_core/*.so - maturin develop --release + pip install -v -e . .PHONY: build-coverage build-coverage: - rm -f python/pydantic_core/*.so - maturin develop -- -C instrument-coverage + @rm -f python/pydantic_core/*.so + RUSTFLAGS='-C instrument-coverage' pip install -v -e . + +.PHONY: build-pgo +build-pgo: + @rm -f python/pydantic_core/*.so + $(eval PROFDATA := $(shell mktemp -d)) + RUSTFLAGS='-Cprofile-generate=$(PROFDATA)' pip install -v -e . + pytest tests/benchmarks + $(eval LLVM_PROFDATA := $(shell rustup run stable bash -c 'echo $$RUSTUP_HOME/toolchains/$$RUSTUP_TOOLCHAIN/lib/rustlib/$$(rustc -Vv | grep host | cut -d " " -f 2)/bin/llvm-profdata')) + $(LLVM_PROFDATA) merge -o $(PROFDATA)/merged.profdata $(PROFDATA) + RUSTFLAGS='-Cprofile-use=$(PROFDATA)/merged.profdata' pip install -v -e . + @rm -rf $(PROFDATA) + .PHONY: build-wasm build-wasm: From cd0d07bbe44b2103efa8e46c0df4a3746f88fd3a Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:56:56 +0100 Subject: [PATCH 2/3] add support for maturin in build-dev --- Makefile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Makefile b/Makefile index 91d907607..fc27e256e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ mypy-stubtest = python -m mypy.stubtest pydantic_core._pydantic_core --allowlist # using pip install cargo (via maturin via pip) doesn't get the tty handle # so doesn't render color without some help export CARGO_TERM_COLOR=$(shell (test -t 0 && echo "always") || echo "auto") +# maturin develop only makes sense inside a virtual env, is otherwise +# more or less equivalent to pip install -e just a little nicer +USE_MATURIN = $(shell [ "$$VIRTUAL_ENV" != "" ] && (which maturin)) .PHONY: install install: @@ -26,27 +29,47 @@ install-rust-coverage: .PHONY: build-dev build-dev: @rm -f python/pydantic_core/*.so +ifneq ($(USE_MATURIN),) + maturin develop +else pip install -v -e . --config-settings=build-args='--profile dev' +endif .PHONY: build-prod build-prod: @rm -f python/pydantic_core/*.so +ifneq ($(USE_MATURIN),) + maturin develop --release +else pip install -v -e . +endif .PHONY: build-coverage build-coverage: @rm -f python/pydantic_core/*.so +ifneq ($(USE_MATURIN),) + RUSTFLAGS='-C instrument-coverage' maturin develop --release +else RUSTFLAGS='-C instrument-coverage' pip install -v -e . +endif .PHONY: build-pgo build-pgo: @rm -f python/pydantic_core/*.so $(eval PROFDATA := $(shell mktemp -d)) +ifneq ($(USE_MATURIN),) + RUSTFLAGS='-Cprofile-generate=$(PROFDATA)' maturin develop --release +else RUSTFLAGS='-Cprofile-generate=$(PROFDATA)' pip install -v -e . +endif pytest tests/benchmarks $(eval LLVM_PROFDATA := $(shell rustup run stable bash -c 'echo $$RUSTUP_HOME/toolchains/$$RUSTUP_TOOLCHAIN/lib/rustlib/$$(rustc -Vv | grep host | cut -d " " -f 2)/bin/llvm-profdata')) $(LLVM_PROFDATA) merge -o $(PROFDATA)/merged.profdata $(PROFDATA) +ifneq ($(USE_MATURIN),) + RUSTFLAGS='-Cprofile-use=$(PROFDATA)/merged.profdata' maturin develop --release +else RUSTFLAGS='-Cprofile-use=$(PROFDATA)/merged.profdata' pip install -v -e . +endif @rm -rf $(PROFDATA) From 6b357d354ae4b4a93b8452133f670b341c6caf29 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:32:35 +0100 Subject: [PATCH 3/3] use build-dev in ci where it makes sense --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e112fbeb..e2b1d56bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: python-version: '3.11' - run: pip install -r tests/requirements.txt - - run: pip install -e . --config-settings=build-args='--profile dev' + - run: make build-dev - run: pip freeze - run: pytest @@ -194,7 +194,7 @@ jobs: - run: pip install -r tests/requirements-linting.txt if: steps.cache-py.outputs.cache-hit != 'true' - - run: pip install . + - run: make build-dev - run: pip freeze