Skip to content

Commit

Permalink
Merge branch 'main' into pythongh-115997
Browse files Browse the repository at this point in the history
  • Loading branch information
illia-v authored Mar 27, 2024
2 parents 9f8f278 + b3e8c78 commit d1cf02f
Show file tree
Hide file tree
Showing 906 changed files with 43,249 additions and 21,408 deletions.
4 changes: 2 additions & 2 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37

ENV CC=clang

ENV WASI_SDK_VERSION=20
ENV WASI_SDK_VERSION=21
ENV WASI_SDK_PATH=/opt/wasi-sdk

ENV WASMTIME_HOME=/opt/wasmtime
ENV WASMTIME_VERSION=14.0.4
ENV WASMTIME_VERSION=18.0.3
ENV WASMTIME_CPU_ARCH=x86_64

RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \
Expand Down
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ Include/opcode_ids.h generated
Include/token.h generated
Lib/_opcode_metadata.py generated
Lib/keyword.py generated
Lib/test/certdata/*.pem generated
Lib/test/certdata/*.0 generated
Lib/test/levenshtein_examples.json generated
Lib/test/test_stable_abi_ctypes.py generated
Lib/token.py generated
Expand Down
3 changes: 2 additions & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ Python/dynload_*.c @ericsnowcurrently
Lib/test/test_module/ @ericsnowcurrently
Doc/c-api/module.rst @ericsnowcurrently
**/*importlib/resources/* @jaraco @warsaw @FFY00
**/importlib/metadata/* @jaraco @warsaw
**/*importlib/metadata/* @jaraco @warsaw

# Dates and times
**/*datetime* @pganssle @abalkin
Expand Down Expand Up @@ -247,6 +247,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently
/Tools/wasm/ @brettcannon

# SBOM
/Misc/externals.spdx.json @sethmlarson
/Misc/sbom.spdx.json @sethmlarson
/Tools/build/generate_sbom.py @sethmlarson

Expand Down
41 changes: 39 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:
- name: Get a list of the changed documentation-related files
if: github.event_name == 'pull_request'
id: changed-docs-files
uses: Ana06/get-changed-files@v2.2.0
uses: Ana06/get-changed-files@v2.3.0
with:
filter: |
Doc/**
Expand Down Expand Up @@ -206,6 +206,8 @@ jobs:
uses: ./.github/workflows/reusable-macos.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
# macos-14 is M1, macos-13 is Intel
os-matrix: '["macos-14", "macos-13"]'

build_macos_free_threading:
name: 'macOS (free-threading)'
Expand All @@ -215,6 +217,8 @@ jobs:
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
free-threading: true
# macos-14 is M1
os-matrix: '["macos-14"]'

build_ubuntu:
name: 'Ubuntu'
Expand Down Expand Up @@ -301,6 +305,14 @@ jobs:
- name: SSL tests
run: ./python Lib/test/ssltests.py

build_wasi:
name: 'WASI'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-wasi.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}

test_hypothesis:
name: "Hypothesis tests on Ubuntu"
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -472,6 +484,24 @@ jobs:
- name: Tests
run: xvfb-run make test

build_tsan:
name: 'Thread sanitizer'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-tsan.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
options: ./configure --config-cache --with-thread-sanitizer --with-pydebug

build_tsan_free_threading:
name: 'Thread sanitizer (free-threading)'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-tsan.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug

# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
cifuzz:
name: CIFuzz
Expand All @@ -484,7 +514,8 @@ jobs:
strategy:
fail-fast: false
matrix:
sanitizer: [address, undefined, memory]
# sanitizer: [address, undefined, memory] -- memory skipped temporarily until GH-116886 is solved.
sanitizer: [address, undefined]
steps:
- name: Build fuzzers (${{ matrix.sanitizer }})
id: build
Expand Down Expand Up @@ -525,10 +556,13 @@ jobs:
- build_ubuntu
- build_ubuntu_free_threading
- build_ubuntu_ssltests
- build_wasi
- build_windows
- build_windows_free_threading
- test_hypothesis
- build_asan
- build_tsan
- build_tsan_free_threading
- cifuzz

runs-on: ubuntu-latest
Expand Down Expand Up @@ -558,9 +592,12 @@ jobs:
build_ubuntu,
build_ubuntu_free_threading,
build_ubuntu_ssltests,
build_wasi,
build_windows,
build_windows_free_threading,
build_asan,
build_tsan,
build_tsan_free_threading,
'
|| ''
}}
Expand Down
34 changes: 23 additions & 11 deletions .github/workflows/jit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ jobs:
target:
- i686-pc-windows-msvc/msvc
- x86_64-pc-windows-msvc/msvc
- aarch64-pc-windows-msvc/msvc
- x86_64-apple-darwin/clang
- aarch64-apple-darwin/clang
- x86_64-unknown-linux-gnu/gcc
Expand All @@ -49,6 +50,10 @@ jobs:
architecture: x64
runner: windows-latest
compiler: msvc
- target: aarch64-pc-windows-msvc/msvc
architecture: ARM64
runner: windows-latest
compiler: msvc
- target: x86_64-apple-darwin/clang
architecture: x86_64
runner: macos-13
Expand All @@ -70,13 +75,13 @@ jobs:
runner: ubuntu-latest
compiler: gcc
# These fail because of emulation, not because of the JIT:
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
- target: aarch64-unknown-linux-gnu/clang
architecture: aarch64
runner: ubuntu-latest
compiler: clang
# These fail because of emulation, not because of the JIT:
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
env:
CC: ${{ matrix.compiler }}
steps:
Expand All @@ -85,19 +90,26 @@ jobs:
with:
python-version: '3.11'

- name: Windows
if: runner.os == 'Windows'
- name: Native Windows
if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
run: |
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
- name: macOS
# No PGO or tests (yet):
- name: Emulated Windows
if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
run: |
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
- name: Native macOS
if: runner.os == 'macOS'
run: |
brew install llvm@${{ matrix.llvm }}
export SDKROOT="$(xcrun --show-sdk-path)"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
SDKROOT="$(xcrun --show-sdk-path)" \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
make all --jobs 4
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
Expand All @@ -122,10 +134,10 @@ jobs:
sudo apt install --yes "gcc-$HOST" qemu-user
${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
export CPP="$CC --preprocess"
export HOSTRUNNER=qemu-${{ matrix.architecture }}
export QEMU_LD_PREFIX="/usr/$HOST"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
CPP="$CC --preprocess" \
HOSTRUNNER=qemu-${{ matrix.architecture }} \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
make all --jobs 4
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.x"
- uses: pre-commit/action@v3.0.0
- uses: pre-commit/action@v3.0.1
2 changes: 1 addition & 1 deletion .github/workflows/project-updater.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- { project: 32, label: sprint }

steps:
- uses: actions/add-to-project@v0.1.0
- uses: actions/add-to-project@v0.6.0
with:
project-url: https://github.com/orgs/python/projects/${{ matrix.project }}
github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/reusable-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ on:
required: false
type: boolean
default: false
os-matrix:
required: false
type: string

jobs:
build_macos:
Expand All @@ -17,14 +20,12 @@ jobs:
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTO_UPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
PYTHONSTRICTEXTENSIONBUILD: 1
strategy:
fail-fast: false
matrix:
os: [
"macos-14", # M1
"macos-13", # Intel
]
os: ${{fromJson(inputs.os-matrix)}}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
Expand Down
51 changes: 51 additions & 0 deletions .github/workflows/reusable-tsan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string
options:
required: true
type: string

jobs:
build_tsan_reusable:
name: 'Thread sanitizer'
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- name: Runner image version
run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
- name: Restore config.cache
uses: actions/cache@v4
with:
path: config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: Install Dependencies
run: |
sudo ./.github/workflows/posix-deps-apt.sh
sudo apt install -y clang
# Reduce ASLR to avoid TSAN crashing
sudo sysctl -w vm.mmap_rnd_bits=28
- name: TSAN Option Setup
run: |
echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV
echo "CC=clang" >> $GITHUB_ENV
echo "CXX=clang++" >> $GITHUB_ENV
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
with:
save: ${{ github.event_name == 'push' }}
max-size: "200M"
- name: Configure CPython
run: ${{ inputs.options }}
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Tests
run: ./python -m test --tsan -j4
71 changes: 71 additions & 0 deletions .github/workflows/reusable-wasi.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string

jobs:
build_wasi_reusable:
name: 'build and test'
timeout-minutes: 60
runs-on: ubuntu-20.04
env:
WASMTIME_VERSION: 18.0.3
WASI_SDK_VERSION: 21
WASI_SDK_PATH: /opt/wasi-sdk
CROSS_BUILD_PYTHON: cross-build/build
CROSS_BUILD_WASI: cross-build/wasm32-wasi
steps:
- uses: actions/checkout@v4
# No problem resolver registered as one doesn't currently exist for Clang.
- name: "Install wasmtime"
uses: jcbhmr/setup-wasmtime@v2
with:
wasmtime-version: ${{ env.WASMTIME_VERSION }}
- name: "Restore WASI SDK"
id: cache-wasi-sdk
uses: actions/cache@v4
with:
path: ${{ env.WASI_SDK_PATH }}
key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}
- name: "Install WASI SDK"
if: steps.cache-wasi-sdk.outputs.cache-hit != 'true'
run: |
mkdir ${{ env.WASI_SDK_PATH }} && \
curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \
tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip
- name: "Configure ccache action"
uses: hendrikmuhs/ccache-action@v1.2
with:
save: ${{ github.event_name == 'push' }}
max-size: "200M"
- name: "Add ccache to PATH"
run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: "Install Python"
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: "Restore Python build config.cache"
uses: actions/cache@v4
with:
path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: "Configure build Python"
run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
- name: "Make build Python"
run: python3 Tools/wasm/wasi.py make-build-python
- name: "Restore host config.cache"
uses: actions/cache@v4
with:
path: ${{ env.CROSS_BUILD_WASI }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}
- name: "Configure host"
# `--with-pydebug` inferred from configure-build-python
run: python3 Tools/wasm/wasi.py configure-host -- --config-cache
- name: "Make host"
run: python3 Tools/wasm/wasi.py make-host
- name: "Display build info"
run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo
- name: "Test"
run: make --directory ${{ env.CROSS_BUILD_WASI }} test
Loading

0 comments on commit d1cf02f

Please sign in to comment.