diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 15a83dd0370e19..fcfac85ed9c442 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index 492e4e34bb2dab..96361961ea75eb 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,7 +12,7 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme +- script: python -m pip install sphinx==2.0.1 blurb python-docs-theme displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 0bd7921bcbefcc..2486f88a63fb15 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000000..3d072e3b43e17e --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,129 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + __RealSigningCertificate: 'Python Software Foundation' +# QUEUE TIME VARIABLES +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false +# PyDotOrgUsername: '' +# PyDotOrgServer: '' +# BuildToPublish: '' + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-store.yml + + +- stage: PublishExistingPyDotOrg + displayName: Publish existing build to python.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishExistingNuget + displayName: Publish existing build to nuget.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishExistingStore + displayName: Publish existing build to Store + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000000..d4563cd0d722c1 --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,83 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: bin_$(Name) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000000..d42d55fff08dda --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000000..e4de78555b3f66 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/gpg-sign.yml b/.azure-pipelines/windows-release/gpg-sign.yml new file mode 100644 index 00000000000000..0855af8d703df9 --- /dev/null +++ b/.azure-pipelines/windows-release/gpg-sign.yml @@ -0,0 +1,28 @@ +parameters: + GPGKeyFile: $(GPGKey) + GPGPassphrase: $(GPGPassphrase) + Files: '*' + WorkingDirectory: $(Build.BinariesDirectory) + +steps: +- task: DownloadSecureFile@1 + name: gpgkey + inputs: + secureFile: ${{ parameters.GPGKeyFile }} + displayName: 'Download GPG key' + +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" + gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" + (gci -File ${{ parameters.Files }}).FullName | %{ + gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_ + "Made signature for $_" + } + displayName: 'Generate GPG signatures' + workingDirectory: ${{ parameters.WorkingDirectory }} + +- powershell: | + $p = gps "gpg-agent" -EA 0 + if ($p) { $p.Kill() } + displayName: 'Kill GPG agent' + condition: true diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000000..2dcd6ed26ca3a8 --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,15 @@ +steps: +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "{0}\bin\python.exe" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{0}\bin" + --temp "{0}\layout-temp" + --include-cat "{0}\bin\python.cat" + --doc-build "{0}\doc"' + -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") + ) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000000..30f7d34fa61d23 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000000..c55fa534eaec86 --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,124 @@ +steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + BuildForRelease: true + SuppressMinGWLib: true + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + BuildForRelease: true + SuppressMinGWLib: true + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishPipelineArtifact@0 + displayName: 'Publish MSI' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\msi' + artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000000..ce7b38176935df --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,160 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: doc' + inputs: + targetPath: $(Build.ArtifactStagingDirectory)\Doc + artifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + # Allow up to five hours for PGO + timeoutInMinutes: 300 + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' + artifactName: tcltk_lib_win32 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + artifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000000..09857ff676b355 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,56 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000000..8b412dffcc8240 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,62 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000000..7d66e8f9821ca5 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,84 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' + artifactName: layout_appxstore_$(Name) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + targetPath: $(Build.BinariesDirectory)\cert + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx' + artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000000..01512975e9db01 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,44 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\nuget' + artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000000..7afc816a0c6e9c --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000000..eebc63fb8809bc --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,127 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - checkout: none + - template: ./find-sdk.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000000..f59bbe9b39a8d7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,41 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000000..296eb28648b98e --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,41 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + inputs: + command: push + packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000000..2dd354a8c276f6 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,154 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + #vmName: win2016-vs2017 + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: Doc' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): Doc' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): msi' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact from $(BuildToPublish): embed' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + + - template: ./gpg-sign.yml + parameters: + GPGKeyFile: 'python-signing.key' + Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip' + + - powershell: > + $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 + -build msi + -user $(PyDotOrgUsername) + -server $(PyDotOrgServer) + -doc_htmlhelp doc\htmlhelp + -embed embed + -skippurge + -skiptest + -skiphash + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload files to python.org' + + - powershell: > + python + "$(Build.SourcesDirectory)\Tools\msi\purge.py" + (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Purge CDN' + + - powershell: | + $failures = 0 + gci "msi\*\*-webinstall.exe" -File | %{ + $d = mkdir "tests\$($_.BaseName)" -Force + gci $d -r -File | del + $ic = copy $_ $d -PassThru + "Checking layout for $($ic.Name)" + Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" + if (-not $?) { + Write-Error "Failed to validate layout of $($inst.Name)" + $failures += 1 + } + } + if ($failures) { + Write-Error "Failed to validate $failures installers" + exit 1 + } + #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Test layouts' + + - powershell: | + $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | ` + Sort-Object Name | ` + Format-Table Name, @{ + Label="MD5"; + Expression={(Get-FileHash $_ -Algorithm MD5).Hash} + }, Length -AutoSize | ` + Out-String -Width 4096 + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + $hashes | Out-File "$d\hashes.txt" -Encoding ascii + $hashes + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Generate hashes' + + - powershell: | + "Copying:" + (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force + gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Copy GPG signatures for build' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\hashes' + artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000000..b22147b1ab45e7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,35 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000000..d6984a0a137cea --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,113 @@ +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + - template: ./find-sdk.yml + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + continueOnError: true + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + targetPath: '$(Build.BinariesDirectory)\bin' + artifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: cert' + inputs: + targetPath: '$(Build.BinariesDirectory)\cert' + artifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000000..b33176266a20da --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,41 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" + Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000000..27b0c96987a7a0 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000000..1f8b601d0d023b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9b7a4032fb4370..8c3970babcc58d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,8 +14,14 @@ Objects/dict* @methane # Hashing -**/*hashlib* @python/crypto-team -**/*pyhash* @python/crypto-team +**/*hashlib* @python/crypto-team @tiran +**/*pyhash* @python/crypto-team @tiran +**/*sha* @python/crypto-team @tiran +**/*md5* @python/crypto-team @tiran +**/*blake* @python/crypto-team @tiran +/Modules/_blake2/** @python/crypto-team @tiran +/Modules/_sha3/** @python/crypto-team @tiran + # HTML /Lib/html/ @ezio-melotti @@ -26,15 +32,25 @@ Objects/dict* @methane # Ignoring importlib.h so as to not get flagged on # all pull requests that change the emitted # bytecode. -**/*import*.c @python/import-team -**/*import*.py @python/import-team +**/*import*.c @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw +**/*import*.py @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw # SSL -**/*ssl* @python/crypto-team +**/*ssl* @python/crypto-team @tiran +**/*.pem @python/crypto-team @tiran # CSPRNG -Python/bootstrap_hash.c @python/crypto-team +Python/bootstrap_hash.c @python/crypto-team @tiran + +# Dates and times +**/*datetime* @pganssle @abalkin +**/*str*time* @pganssle @abalkin +Doc/library/time.rst @pganssle @abalkin +Lib/test/test_time.py @pganssle @abalkin +Modules/timemodule.c @pganssle @abalkin +Python/pytime.c @pganssle @abalkin +Include/pytime.h @pganssle @abalkin # Email and related **/*mail* @python/email-team diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000000000..36c4e2771843e4 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://www.python.org/psf/donations/python-dev/ diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 00000000000000..28aea946623cc5 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Supported Versions + +The Python team applies security fixes according to the table +in [the devguide]( +https://devguide.python.org/#status-of-python-branches +). + +## Reporting a Vulnerability + +Please read the guidelines on reporting security issues [on the +official website]( +https://www.python.org/news/security/#reporting-security-issues-in-python +) for instructions on how to report a security-related problem to +the Python team responsibly. + +To reach the response team, email `security at python dot org`. diff --git a/.github/appveyor.yml b/.github/appveyor.yml index e8012f69ee5b2e..8556cf8437e509 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -1,4 +1,4 @@ -version: 3.8build{build} +version: 3.9build{build} clone_depth: 5 branches: only: diff --git a/.gitignore b/.gitignore index 9e60f5a20690bf..9445ef1e2c5252 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ Lib/test/data/* Makefile Makefile.pre Misc/python.pc +Misc/python-embed.pc Misc/python-config.sh Modules/Setup Modules/Setup.config @@ -70,6 +71,7 @@ PCbuild/*.VC.opendb PCbuild/.vs/ PCbuild/amd64/ PCbuild/arm32/ +PCbuild/arm64/ PCbuild/obj/ PCbuild/win32/ .purify diff --git a/.travis.yml b/.travis.yml index 6d57ebb1d2fb96..cff401f5bcb2ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,10 @@ cache: env: global: - - OPENSSL=1.1.0i + - OPENSSL=1.1.1c - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - # Use -O3 because we don't use debugger on Travis-CI - - CFLAGS="-I${OPENSSL_DIR}/include -O3" + - CFLAGS="-I${OPENSSL_DIR}/include" - LDFLAGS="-L${OPENSSL_DIR}/lib" # Set rpath with env var instead of -Wl,-rpath linker flag # OpenSSL ignores LDFLAGS when linking bin/openssl @@ -32,7 +31,8 @@ matrix: allow_failures: - env: OPTIONAL=true include: - - os: linux + - name: "CPython tests" + os: linux language: c compiler: clang # gcc also works, but to keep the # of concurrent builds down, we use one C @@ -42,8 +42,10 @@ matrix: addons: apt: packages: + - gdb - xvfb - - os: linux + - name: "Documentation build" + os: linux language: python # Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs. python: 3.6 @@ -53,10 +55,11 @@ matrix: # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. # (Updating the version is fine as long as no warnings are raised by doing so.) # The theme used by the docs is stored separately, so we need to install that as well. - - python -m pip install sphinx==1.8.2 blurb python-docs-theme + - python -m pip install sphinx==2.0.1 blurb python-docs-theme script: - make check suspicious html SPHINXOPTS="-q -W -j4" - - os: linux + - name: "Documentation tests" + os: linux language: c compiler: clang env: TESTING=doctest @@ -70,7 +73,8 @@ matrix: - make -C Doc/ PYTHON=../python venv script: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest - - os: osx + - name: "Mac OS X tests" + os: osx language: c compiler: clang # Testing under macOS is optional until testing stability has been demonstrated. @@ -79,29 +83,45 @@ matrix: # Python 3 is needed for Argument Clinic and multissl - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3 - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH - - os: linux + - name: "Test code coverage (Python)" + os: linux language: c compiler: gcc env: OPTIONAL=true addons: apt: packages: - - lcov - xvfb before_script: - ./configure - - make coverage -s -j4 + - make -j4 # Need a venv that can parse covered code. - ./python -m venv venv - ./venv/bin/python -m pip install -U coverage - ./venv/bin/python -m test.pythoninfo script: # Skip tests that re-run the entire test suite. - - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures + - xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true after_script: # Probably should be after_success once test suite updated to run under coverage.py. # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate - - make coverage-lcov + - bash <(curl -s https://codecov.io/bash) + - name: "Test code coverage (C)" + os: linux + language: c + compiler: gcc + env: OPTIONAL=true + addons: + apt: + packages: + - lcov + - xvfb + before_script: + - ./configure + script: + - xvfb-run make -j4 coverage-report + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + - make pythoninfo - bash <(curl -s https://codecov.io/bash) @@ -146,7 +166,8 @@ install: # Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - - ./configure --with-pydebug + # -Og is much faster than -O0 + - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug - make -j4 regen-all - changes=`git status --porcelain` - | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index ed15b520548c74..c5f24abe2ca661 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,11 +2,11 @@ Please note that all interactions on [Python Software Foundation](https://www.python.org/psf-landing/)-supported -infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties>) +infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties) by the [PSF Code of Conduct](https://www.python.org/psf/codeofconduct/), -which includes all infrastructure used in the development of Python itself +which includes all the infrastructure used in the development of Python itself (e.g. mailing lists, issue trackers, GitHub, etc.). -In general this means everyone is expected to be open, considerate, and -respectful of others no matter what their position is within the project. +In general, this means that everyone is expected to be **open**, **considerate**, and +**respectful** of others no matter what their position is within the project. diff --git a/Doc/Makefile b/Doc/Makefile index 53877e61329089..3bcd9f23752b00 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -48,11 +48,19 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif [ -d ../Misc/NEWS.d ]; then \ - echo "Building NEWS from Misc/NEWS.d with blurb"; \ - $(BLURB) merge -f build/NEWS; \ + elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + if [ -d ../Misc/NEWS.d ]; then \ + echo "Building NEWS from Misc/NEWS.d with blurb"; \ + $(BLURB) merge -f build/NEWS; \ + else \ + echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + exit 1; \ + fi \ else \ - echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + echo ""; \ + echo "Missing the required blurb or sphinx-build tools."; \ + echo "Please run 'make venv' to install local copies."; \ + echo ""; \ exit 1; \ fi $(SPHINXBUILD) $(ALLSPHINXOPTS) @@ -124,7 +132,8 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -U pip setuptools + $(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.0.1 blurb python-docs-theme @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/README.rst b/Doc/README.rst index 31f8a8b7f59328..380ea4fa9b26ad 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -113,6 +113,15 @@ Then, from the ``Doc`` directory, run :: where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). +Deprecation header +================== + +You can define the ``outdated`` variable in ``html_context`` to show a +red banner on each page redirecting to the "latest" version. + +The link points to the same page on ``/3/``, sadly for the moment the +language is lost during the process. + Contributing ============ diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst index ad538811127dfb..0edd1d5f6240af 100644 --- a/Doc/c-api/abstract.rst +++ b/Doc/c-api/abstract.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _abstract: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 25a867f139cc7a..33b0c06a9ebc2e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _allocating-objects: @@ -48,7 +48,7 @@ Allocating Objects on the Heap improving the memory management efficiency. -.. c:function:: void PyObject_Del(PyObject *op) +.. c:function:: void PyObject_Del(void *op) Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 890a0383931933..b8a8f2ff886219 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _apiabiversion: diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index b41130ede416e3..ba9ca5e0d30b92 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _arg-parsing: diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index a9fb342f7c0f8e..ce8de6e22f44ca 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _boolobjects: diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index c7c1e3cc745ac4..72d965053cfda3 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. index:: single: buffer protocol diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 41b6e3c71be53d..b4a9660a9169e7 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _bytearrayobjects: diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 5b9ebf6b6af5f5..9a62fb682d6897 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _bytesobjects: diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 8eb6695e22de18..3c921bbde3486f 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _capsules: diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 427259cc24d8d8..8514afe928e7d3 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _cell-objects: diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 10d89f297c843f..3c4f66923da5a9 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _codeobjects: @@ -35,11 +35,18 @@ bound into a function. .. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - Return a new code object. If you need a dummy code object to - create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling - :c:func:`PyCode_New` directly can bind you to a precise Python - version since the definition of the bytecode changes often. + Return a new code object. If you need a dummy code object to create a frame, + use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly + can bind you to a precise Python version since the definition of the bytecode + changes often. + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags c.PyCode_New + +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) + + Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positonal-only arguments. + + .. versionadded:: 3.8 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index fc63b57a85521c..06dbb2572725ee 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _complexobjects: @@ -129,4 +129,10 @@ Complex Numbers as Python Objects If *op* is not a Python complex number object but has a :meth:`__complex__` method, this method will first be called to convert *op* to a Python complex - number object. Upon failure, this method returns ``-1.0`` as a real value. + number object. If ``__complex__()`` is not defined then it falls back to + :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + value. + + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 9558a4a583cba5..f83d711795c2c6 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _concrete: diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index c344c8d71ae32d..a7cde7fb1958a0 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _contextvarsobjects: diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c46722d782a2fb..ed101c791ec743 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _string-conversion: diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index 2fe50b5d8c4473..50428c7eb43e3d 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _coro-objects: diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 78724619ea3c52..44a04373d19683 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _datetimeobjects: @@ -98,12 +98,28 @@ Macros to create objects: minute, second and microsecond. +.. c:function:: PyObject* PyDateTime_FromDateAndTimeAndFold(int year, int month, int day, int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.datetime` object with the specified year, month, day, hour, + minute, second, microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) Return a :class:`datetime.time` object with the specified hour, minute, second and microsecond. +.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.time` object with the specified hour, minute, second, + microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) Return a :class:`datetime.timedelta` object representing the given number diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index c8f6fa5bcd7fc8..1005140c7acb3a 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _descriptor-objects: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0ced5a5fd00170..e970771893eb30 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _dictobjects: diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8c5f66cbef722b..d8173935596bb8 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _exceptionhandling: @@ -53,8 +53,8 @@ Printing and clearing .. c:function:: void PyErr_PrintEx(int set_sys_last_vars) Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``. In that case the no traceback - is printed and Python process will exit with the error code specified by + **Unless** the error is a ``SystemExit``, in that case no traceback is + printed and the Python process will exit with the error code specified by the ``SystemExit`` instance. Call this function **only** when the error indicator is set. Otherwise it @@ -72,6 +72,9 @@ Printing and clearing .. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + This utility function prints a warning message to ``sys.stderr`` when an exception has been set but it is impossible for the interpreter to actually raise the exception. It is used, for example, when an exception occurs in an @@ -81,6 +84,8 @@ Printing and clearing in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + An exception must be set when calling this function. + Raising exceptions ================== @@ -315,7 +320,7 @@ an error value). :mod:`warnings` module and the :option:`-W` option in the command line documentation. There is no C API for warning control. -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path) +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) Much like :c:func:`PyErr_SetImportError` but this function allows for specifying a subclass of :exc:`ImportError` to raise. @@ -514,13 +519,13 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - This function simulates the effect of a :const:`SIGINT` signal arriving --- the - next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will - be raised. It may be called without holding the interpreter lock. - - .. % XXX This was described as obsolete, but is used in - .. % _thread.interrupt_main() (used from IDLE), so it's still needed. + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. c:function:: int PySignal_SetWakeupFd(int fd) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 6f2ecee51fd86a..543dc60b9f55ab 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _fileobjects: @@ -60,6 +60,32 @@ the :mod:`io` APIs instead. raised if the end of the file is reached immediately. +.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler) + + Overrides the normal behavior of :func:`io.open_code` to pass its parameter + through the provided handler. + + The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + As this hook is intentionally used during import, avoid importing new modules + during its execution unless they are known to be frozen or available in + ``sys.modules``. + + Once a hook has been set, it cannot be removed or replaced, and later calls to + :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns + -1 and sets an exception if the interpreter has been initialized. + + This function is safe to call before :c:func:`Py_Initialize`. + + .. versionadded:: 3.8 + + + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) .. index:: single: Py_PRINT_RAW diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 27a75e3e0c2eeb..057ff522516a3a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _floatobjects: @@ -47,9 +47,13 @@ Floating Point Objects Return a C :c:type:`double` representation of the contents of *pyfloat*. If *pyfloat* is not a Python floating point object but has a :meth:`__float__` method, this method will first be called to convert *pyfloat* into a float. + If ``__float__()`` is not defined then it falls back to :meth:`__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. + .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 17279c732ae076..02c4ebdbed4546 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _function-objects: diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index b739aacea8d385..b3f30b2ed9e999 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _supporting-cycle-detection: diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 1efbae4fcba098..8d54021c181457 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _gen-objects: diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8cdc256e7c9e0e..3bc50609714a5b 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _importing: @@ -223,21 +223,6 @@ Importing Modules Return a new reference to the finder object. -.. c:function:: void _PyImport_Init() - - Initialize the import mechanism. For internal use only. - - -.. c:function:: void PyImport_Cleanup() - - Empty the module table. For internal use only. - - -.. c:function:: void _PyImport_Fini() - - Finalize the import mechanism. For internal use only. - - .. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) Load a frozen module named *name*. Return ``1`` for success, ``0`` if the diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 3bfbaf4eedde0e..9a8f1507b3f4cc 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -21,6 +21,7 @@ document the API functions in detail. abstract.rst concrete.rst init.rst + init_config.rst memory.rst objimpl.rst apiabiversion.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 367c069a7ff4cd..a0ac4d21d13936 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _initialization: @@ -877,13 +877,6 @@ code, or when embedding the Python interpreter: and is not released. -.. c:function:: void PyEval_ReInitThreads() - - This function is called from :c:func:`PyOS_AfterFork_Child` to ensure - that newly created child processes don't hold locks referring to threads - which are not running in the child process. - - The following functions use thread-local storage, and are not compatible with sub-interpreters: diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst new file mode 100644 index 00000000000000..d2c1f9a2f3e373 --- /dev/null +++ b/Doc/c-api/init_config.rst @@ -0,0 +1,1025 @@ +.. highlight:: c + +.. _init-config: + +*********************************** +Python Initialization Configuration +*********************************** + +.. versionadded:: 3.8 + +Structures: + +* :c:type:`PyConfig` +* :c:type:`PyPreConfig` +* :c:type:`PyStatus` +* :c:type:`PyWideStringList` + +Functions: + +* :c:func:`PyConfig_Clear` +* :c:func:`PyConfig_InitIsolatedConfig` +* :c:func:`PyConfig_InitPythonConfig` +* :c:func:`PyConfig_Read` +* :c:func:`PyConfig_SetArgv` +* :c:func:`PyConfig_SetBytesArgv` +* :c:func:`PyConfig_SetBytesString` +* :c:func:`PyConfig_SetString` +* :c:func:`PyConfig_SetWideStringList` +* :c:func:`PyPreConfig_InitIsolatedConfig` +* :c:func:`PyPreConfig_InitPythonConfig` +* :c:func:`PyStatus_Error` +* :c:func:`PyStatus_Exception` +* :c:func:`PyStatus_Exit` +* :c:func:`PyStatus_IsError` +* :c:func:`PyStatus_IsExit` +* :c:func:`PyStatus_NoMemory` +* :c:func:`PyStatus_Ok` +* :c:func:`PyWideStringList_Append` +* :c:func:`PyWideStringList_Insert` +* :c:func:`Py_ExitStatusException` +* :c:func:`Py_InitializeFromConfig` +* :c:func:`Py_PreInitialize` +* :c:func:`Py_PreInitializeFromArgs` +* :c:func:`Py_PreInitializeFromBytesArgs` +* :c:func:`Py_RunMain` + +The preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. + +.. seealso:: + :pep:`587` "Python Initialization Configuration". + + +PyWideStringList +---------------- + +.. c:type:: PyWideStringList + + List of ``wchar_t*`` strings. + + If *length* is non-zero, *items* must be non-NULL and all strings must be + non-NULL. + + Methods: + + .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) + + Append *item* to *list*. + + Python must be preinitialized to call this function. + + .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) + + Insert *item* into *list* at *index*. If *index* is greater than *list* + length, just append *item* to *list*. + + Python must be preinitialized to call this function. + + Structure fields: + + .. c:member:: Py_ssize_t length + + List length. + + .. c:member:: wchar_t** items + + List items. + +PyStatus +-------- + +.. c:type:: PyStatus + + Structure to store an initialization function status: success, error + or exit. + + For an error, it can store the C function name which created the error. + + Structure fields: + + .. c:member:: int exitcode + + Exit code. Argument passed to ``exit()``. + + .. c:member:: const char *err_msg + + Error message. + + .. c:member:: const char *func + + Name of the function which created an error, can be ``NULL``. + + Functions to create a status: + + .. c:function:: PyStatus PyStatus_Ok(void) + + Success. + + .. c:function:: PyStatus PyStatus_Error(const char *err_msg) + + Initialization error with a message. + + .. c:function:: PyStatus PyStatus_NoMemory(void) + + Memory allocation failure (out of memory). + + .. c:function:: PyStatus PyStatus_Exit(int exitcode) + + Exit Python with the specified exit code. + + Functions to handle a status: + + .. c:function:: int PyStatus_Exception(PyStatus status) + + Is the status an error or an exit? If true, the exception must be + handled; by calling :c:func:`Py_ExitStatusException` for example. + + .. c:function:: int PyStatus_IsError(PyStatus status) + + Is the result an error? + + .. c:function:: int PyStatus_IsExit(PyStatus status) + + Is the result an exit? + + .. c:function:: void Py_ExitStatusException(PyStatus status) + + Call ``exit(exitcode)`` if *status* is an exit. Print the error + message and exit with a non-zero exit code if *status* is an error. Must + only be called if ``PyStatus_Exception(status)`` is non-zero. + +.. note:: + Internally, Python uses macros which set ``PyStatus.func``, + whereas functions to create a status set ``func`` to ``NULL``. + +Example:: + + PyStatus alloc(void **ptr, size_t size) + { + *ptr = PyMem_RawMalloc(size); + if (*ptr == NULL) { + return PyStatus_NoMemory(); + } + return PyStatus_Ok(); + } + + int main(int argc, char **argv) + { + void *ptr; + PyStatus status = alloc(&ptr, 16); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + PyMem_Free(ptr); + return 0; + } + + +PyPreConfig +----------- + +.. c:type:: PyPreConfig + + Structure used to preinitialize Python: + + * Set the Python memory allocator + * Configure the LC_CTYPE locale + * Set the UTF-8 mode + + Function to initialize a preconfiguration: + + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Python Configuration + `. + + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Isolated Configuration + `. + + Structure fields: + + .. c:member:: int allocator + + Name of the memory allocator: + + * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators + (use defaults) + * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): default memory allocators + * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): default memory allocators with + debug hooks + * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): force usage of ``malloc()`` + * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of + ``malloc()`` with debug hooks + * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory + allocator ` + * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc + memory allocator ` with debug hooks + + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` + are not supported if Python is configured using ``--without-pymalloc`` + + See :ref:`Memory Management `. + + .. c:member:: int configure_locale + + Set the LC_CTYPE locale to the user preferred locale? If equals to 0, set + :c:member:`coerce_c_locale` and :c:member:`coerce_c_locale_warn` to 0. + + .. c:member:: int coerce_c_locale + + If equals to 2, coerce the C locale; if equals to 1, read the LC_CTYPE + locale to decide if it should be coerced. + + .. c:member:: int coerce_c_locale_warn + If non-zero, emit a warning if the C locale is coerced. + + .. c:member:: int dev_mode + + See :c:member:`PyConfig.dev_mode`. + + .. c:member:: int isolated + + See :c:member:`PyConfig.isolated`. + + .. c:member:: int legacy_windows_fs_encoding (Windows only) + + If non-zero, disable UTF-8 Mode, set the Python filesystem encoding to + ``mbcs``, set the filesystem error handler to ``replace``. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int parse_argv + + If non-zero, :c:func:`Py_PreInitializeFromArgs` and + :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the + same way the regular Python parses command line arguments: see + :ref:`Command Line Arguments `. + + .. c:member:: int use_environment + + See :c:member:`PyConfig.use_environment`. + + .. c:member:: int utf8_mode + + If non-zero, enable the UTF-8 mode. + +Preinitialization with PyPreConfig +---------------------------------- + +Functions to preinitialize Python: + +.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) + + Preinitialize Python from *preconfig* preconfiguration. + +.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (bytes strings). + +.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (wide strings). + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +For :ref:`Python Configuration ` +(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with +command line arguments, the command line arguments must also be passed to +preinitialize Python, since they have an effect on the pre-configuration +like encodings. For example, the :option:`-X` ``utf8`` command line option +enables the UTF-8 Mode. + +``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and +before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. +It can be called before :c:func:`Py_PreInitialize` if +:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. + +Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be +used before Python preinitialization, whereas calling directly ``malloc()`` and +``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called before +the preinitialization. + +Example using the preinitialization to enable the UTF-8 Mode:: + + PyPreConfig preconfig; + PyPreConfig_InitPythonConfig(&preconfig); + + preconfig.utf8_mode = 1; + + PyStatus status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* at this point, Python will speak UTF-8 */ + + Py_Initialize(); + /* ... use Python API here ... */ + Py_Finalize(); + + +PyConfig +-------- + +.. c:type:: PyConfig + + Structure containing most parameters to configure Python. + + Structure methods: + + .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) + + Initialize configuration with :ref:`Python Configuration + `. + + .. c:function:: PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) + + Initialize configuration with :ref:`Isolated Configuration + `. + + .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) + + Copy the wide character string *str* into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) + + Decode *str* using ``Py_DecodeLocale()`` and set the result into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) + + Set command line arguments from wide character strings. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) + + Set command line arguments: decode bytes using :c:func:`Py_DecodeLocale`. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) + + Set the list of wide strings *list* to *length* and *items*. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) + + Read all Python configuration. + + Fields which are already initialized are left unchanged. + + Preinitialize Python if needed. + + .. c:function:: void PyConfig_Clear(PyConfig *config) + + Release configuration memory. + + Most ``PyConfig`` methods preinitialize Python if needed. In that case, the + Python preinitialization configuration in based on the :c:type:`PyConfig`. + If configuration fields which are in common with :c:type:`PyPreConfig` are + tuned, they must be set before calling a :c:type:`PyConfig` method: + + * :c:member:`~PyConfig.dev_mode` + * :c:member:`~PyConfig.isolated` + * :c:member:`~PyConfig.parse_argv` + * :c:member:`~PyConfig.use_environment` + + Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` + is used, this method must be called first, before other methods, since the + preinitialization configuration depends on command line arguments (if + :c:member:`parse_argv` is non-zero). + + The caller of these methods is responsible to handle exceptions (error or + exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. + + Structure fields: + + .. c:member:: PyWideStringList argv + + Command line arguments, :data:`sys.argv`. See + :c:member:`~PyConfig.parse_argv` to parse :c:member:`~PyConfig.argv` the + same way the regular Python parses Python command line arguments. If + :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure + that :data:`sys.argv` always exists and is never empty. + + .. c:member:: wchar_t* base_exec_prefix + + :data:`sys.base_exec_prefix`. + + .. c:member:: wchar_t* base_prefix + + :data:`sys.base_prefix`. + + .. c:member:: int buffered_stdio + + If equals to 0, enable unbuffered mode, making the stdout and stderr + streams unbuffered. + + stdin is always opened in buffered mode. + + .. c:member:: int bytes_warning + + If equals to 1, issue a warning when comparing :class:`bytes` or + :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with + :class:`int`. If equal or greater to 2, raise a :exc:`BytesWarning` + exception. + + .. c:member:: wchar_t* check_hash_pycs_mode + + Control the validation behavior of hash-based ``.pyc`` files (see + :pep:`552`): :option:`--check-hash-based-pycs` command line option value. + + Valid values: ``always``, ``never`` and ``default``. + + The default value is: ``default``. + + .. c:member:: int configure_c_stdio + + If non-zero, configure C standard streams (``stdio``, ``stdout``, + ``stdout``). For example, set their mode to ``O_BINARY`` on Windows. + + .. c:member:: int dev_mode + + Development mode: see :option:`-X` ``dev``. + + .. c:member:: int dump_refs + + If non-zero, dump all objects which are still alive at exit. + + Require a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + + .. c:member:: wchar_t* exec_prefix + + :data:`sys.exec_prefix`. + + .. c:member:: wchar_t* executable + + :data:`sys.executable`. + + .. c:member:: int faulthandler + + If non-zero, call :func:`faulthandler.enable`. + + .. c:member:: wchar_t* filesystem_encoding + + Filesystem encoding, :func:`sys.getfilesystemencoding`. + + .. c:member:: wchar_t* filesystem_errors + + Filesystem encoding errors, :func:`sys.getfilesystemencodeerrors`. + + .. c:member:: unsigned long hash_seed + .. c:member:: int use_hash_seed + + Randomized hash function seed. + + If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly + at Pythonstartup, and :c:member:`~PyConfig.hash_seed` is ignored. + + .. c:member:: wchar_t* home + + Python home directory. + + .. c:member:: int import_time + + If non-zero, profile import time. + + .. c:member:: int inspect + + Enter interactive mode after executing a script or a command. + + .. c:member:: int install_signal_handlers + + Install signal handlers? + + .. c:member:: int interactive + + Interactive mode. + + .. c:member:: int isolated + + If greater than 0, enable isolated mode: + + * :data:`sys.path` contains neither the script's directory (computed from + ``argv[0]`` or the current directory) nor the user's site-packages + directory. + * Python REPL doesn't import :mod:`readline` nor enable default readline + configuration on interactive prompts. + * Set :c:member:`~PyConfig.use_environment` and + :c:member:`~PyConfig.user_site_directory` to 0. + + .. c:member:: int legacy_windows_stdio + + If non-zero, use :class:`io.FileIO` instead of + :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` + and :data:`sys.stderr`. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int malloc_stats + + If non-zero, dump statistics on :ref:`Python pymalloc memory allocator + ` at exit. + + The option is ignored if Python is built using ``--without-pymalloc``. + + .. c:member:: wchar_t* pythonpath_env + + Module search paths as a string separated by ``DELIM`` + (:data:`os.path.pathsep`). + + Initialized from :envvar:`PYTHONPATH` environment variable value by + default. + + .. c:member:: PyWideStringList module_search_paths + .. c:member:: int module_search_paths_set + + :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is + equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden + by the function computing the :ref:`Path Configuration + `. + + .. c:member:: int optimization_level + + Compilation optimization level: + + * 0: Peephole optimizer (and ``__debug__`` is set to ``True``) + * 1: Remove assertions, set ``__debug__`` to ``False`` + * 2: Strip docstrings + + .. c:member:: int parse_argv + + If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular + Python command line arguments, and strip Python arguments from + :c:member:`~PyConfig.argv`: see :ref:`Command Line Arguments + `. + + .. c:member:: int parser_debug + + If non-zero, turn on parser debugging output (for expert only, depending + on compilation options). + + .. c:member:: int pathconfig_warnings + + If equal to 0, suppress warnings when computing the path configuration + (Unix only, Windows does not log any warning). Otherwise, warnings are + written into ``stderr``. + + .. c:member:: wchar_t* prefix + + :data:`sys.prefix`. + + .. c:member:: wchar_t* program_name + + Program name. + + .. c:member:: wchar_t* pycache_prefix + + ``.pyc`` cache prefix. + + .. c:member:: int quiet + + Quiet mode. For example, don't display the copyright and version messages + even in interactive mode. + + .. c:member:: wchar_t* run_command + + ``python3 -c COMMAND`` argument. + + .. c:member:: wchar_t* run_filename + + ``python3 FILENAME`` argument. + + .. c:member:: wchar_t* run_module + + ``python3 -m MODULE`` argument. + + .. c:member:: int show_alloc_count + + Show allocation counts at exit? + + Need a special Python build with ``COUNT_ALLOCS`` macro defined. + + .. c:member:: int show_ref_count + + Show total reference count at exit? + + Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + + .. c:member:: int site_import + + Import the :mod:`site` module at startup? + + .. c:member:: int skip_source_first_line + + Skip the first line of the source? + + .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdio_errors + + Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and + :data:`sys.stderr`. + + .. c:member:: int tracemalloc + + If non-zero, call :func:`tracemalloc.start`. + + .. c:member:: int use_environment + + If greater than 0, use :ref:`environment variables `. + + .. c:member:: int user_site_directory + + If non-zero, add user site directory to :data:`sys.path`. + + .. c:member:: int verbose + + If non-zero, enable verbose mode. + + .. c:member:: PyWideStringList warnoptions + + Options of the :mod:`warnings` module to build warnings filters. + + .. c:member:: int write_bytecode + + If non-zero, write ``.pyc`` files. + + .. c:member:: PyWideStringList xoptions + + :data:`sys._xoptions`. + +If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same +way the regular Python parses command line arguments, and Python +arguments are stripped from ``argv``: see :ref:`Command Line Arguments +`. + +The ``xoptions`` options are parsed to set other options: see :option:`-X` +option. + + +Initialization with PyConfig +---------------------------- + +Function to initialize Python: + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration. + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or +``PyImport_ExtendInittab()`` is used: they must be set or called after Python +preinitialization and before the Python initialization. + +Example setting the program name:: + + void init_python(void) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + + /* Set the program name. Implicitly preinitialize Python. */ + status = PyConfig_SetString(&config, &config.program_name, + L"/path/to/my_program"); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + return; + + fail: + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + +More complete example modifying the default configuration, read the +configuration, and then override some parameters:: + + PyStatus init_python(const char *program_name) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Set the program name before reading the configuraton + (decode byte string from the locale encoding). + + Implicitly preinitialize Python. */ + status = PyConfig_SetBytesString(&config, &config.program_name, + program_name); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Read all configuration at once */ + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Append our custom search path to sys.path */ + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/more/modules"); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Override executable computed by PyConfig_Read() */ + status = PyConfig_SetString(&config, &config.executable, + L"/path/to/my_executable"); + if (PyStatus_Exception(status)) { + goto done; + } + + status = Py_InitializeFromConfig(&config); + + done: + PyConfig_Clear(&config); + return status; + } + + +.. _init-isolated-conf: + +Isolated Configuration +---------------------- + +:c:func:`PyPreConfig_InitIsolatedConfig` and +:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to +isolate Python from the system. For example, to embed Python into an +application. + +This configuration ignores global configuration variables, environments +variables and command line arguments (:c:member:`PyConfig.argv` is not parsed). +The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left +unchanged by default. + +Configuration files are still used with this configuration. Set the +:ref:`Path Configuration ` ("output fields") to ignore these +configuration files and avoid the function computing the default path +configuration. + + +.. _init-python-config: + +Python Configuration +-------------------- + +:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` +functions create a configuration to build a customized Python which behaves as +the regular Python. + +Environments variables and command line arguments are used to configure +Python, whereas global configuration variables are ignored. + +This function enables C locale coercion (:pep:`538`) and UTF-8 Mode +(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and +:envvar:`PYTHONCOERCECLOCALE` environment variables. + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyConfig config; + PyStatus status; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + fail: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + + +.. _init-path-config: + +Path Configuration +------------------ + +:c:type:`PyConfig` contains multiple fields for the path configuration: + +* Path configuration input fields: + + * :c:member:`PyConfig.home` + * :c:member:`PyConfig.pythonpath_env` + * :c:member:`PyConfig.pathconfig_warnings` + +* Path configuration output fields: + + * :c:member:`PyConfig.exec_prefix` + * :c:member:`PyConfig.executable` + * :c:member:`PyConfig.prefix` + * :c:member:`PyConfig.module_search_paths_set`, + :c:member:`PyConfig.module_search_paths` + +If at least one "output field" is not set, Python computes the path +configuration to fill unset fields. If +:c:member:`~PyConfig.module_search_paths_set` is equal to 0, +:c:member:`~PyConfig.module_search_paths` is overriden and +:c:member:`~PyConfig.module_search_paths_set` is set to 1. + +It is possible to completely ignore the function computing the default +path configuration by setting explicitly all path configuration output +fields listed above. A string is considered as set even if it is non-empty. +``module_search_paths`` is considered as set if +``module_search_paths_set`` is set to 1. In this case, path +configuration input fields are ignored as well. + +Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when +computing the path configuration (Unix only, Windows does not log any warning). + +If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` +fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` +and :c:member:`~PyConfig.exec_prefix` respectively. + +:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: + +* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a + ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to + :data:`sys.path`. +* If :c:member:`~PyConfig.isolated` is zero: + + * If :c:member:`~PyConfig.run_module` is set, prepend the current directory + to :data:`sys.path`. Do nothing if the current directory cannot be read. + * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the + filename to :data:`sys.path`. + * Otherwise, prepend an empty string to :data:`sys.path`. + +If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be +modified by the :mod:`site` module. If +:c:member:`~PyConfig.user_site_directory` is non-zero and the user's +site-package directory exists, the :mod:`site` module appends the user's +site-package directory to :data:`sys.path`. + +The following configuration files are used by the path configuration: + +* ``pyvenv.cfg`` +* ``python._pth`` (Windows only) +* ``pybuilddir.txt`` (Unix only) + + +Py_RunMain() +------------ + +.. c:function:: int Py_RunMain(void) + + Execute the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. + + By default and when if :option:`-i` option is used, run the REPL. + + Finally, finalizes Python and returns an exit status that can be passed to + the ``exit()`` function. + +See :ref:`Python Configuration ` for an example of +customized Python always running in isolated mode using +:c:func:`Py_RunMain`. + + +Multi-Phase Initialization Private Provisional API +-------------------------------------------------- + +This section is a private provisional API introducing multi-phase +initialization, the core feature of the :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet); + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. + +Private provisional API: + +* :c:member:`PyConfig._init_main`: if set to 0, + :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom sys.meta_path importer or an import hook, etc. + +It may become possible to compute the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ + + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 69aef0da04f3ba..a1c8d34a7ea02f 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _api-intro: @@ -156,10 +156,22 @@ complete listing. .. c:macro:: Py_UNUSED(arg) Use this for unused arguments in a function definition to silence compiler - warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``. + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. .. versionadded:: 3.4 +.. c:macro:: Py_DEPRECATED(version) + + Use this for deprecated declarations. The macro must be placed before the + symbol name. + + Example:: + + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + + .. versionchanged:: 3.8 + MSVC support was added. + .. _api-objects: diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 2ba444d1de6d24..546efb518a7af5 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _iterator: diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 82cb4eba8ab743..4d91e4a224fe82 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _iterator-objects: diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 5b263a7b1cdf04..dc9026605be358 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _listobjects: @@ -59,9 +59,9 @@ List Objects .. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The - position must be positive, indexing from the end of the list is not - supported. If *index* is out of bounds, return *NULL* and set an - :exc:`IndexError` exception. + position must be non-negative; indexing from the end of the list is not + supported. If *index* is out of bounds (<0 or >=len(list)), + return *NULL* and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 8093f4b627a2b9..fdaefafe21ba09 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _longobjects: @@ -288,7 +288,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. @@ -307,7 +308,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index e37dec9949ab68..4244b47af75f1a 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _mapping: diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 17ec621610b5e3..b086830feb14f1 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _marshalling-utils: diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 65a207691b8ac8..f97ae453487f86 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _memory: @@ -67,7 +67,7 @@ example:: In this example, the memory request for the I/O buffer is handled by the C library allocator. The Python memory manager is involved only in the allocation -of the string object returned as a result. +of the bytes object returned as a result. In most situations, however, it is recommended to allocate memory from the Python heap specifically because the latter is under control of the Python diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 9f6bfd751ade5e..77afb020405a74 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _memoryview-objects: diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 7a2a84fe110dcf..1ad805e269aa2d 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _instancemethod-objects: diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 017b656854a8cd..68cbda2938f3f0 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _moduleobjects: diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index 45568fe657b938..26d2b7aab201ba 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _noneobject: diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 9fb220b192b9a6..74932f6e7445f0 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _number: diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst index 3572564b13e9b3..6b82a642d7ee42 100644 --- a/Doc/c-api/objbuffer.rst +++ b/Doc/c-api/objbuffer.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c Old Buffer Protocol ------------------- diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a64ff2e6b58b4c..2cf032821afbe5 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _object: @@ -253,6 +253,28 @@ Object Protocol and ``0`` otherwise. This function always succeeds. +.. c:function:: PyObject* PyObject_CallNoArgs(PyObject *callable) + + Call a callable Python object *callable* without any arguments. It is the + most efficient way to call a callable Python object without any argument. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* _PyObject_CallOneArg(PyObject *callable, PyObject *arg) + + Call a callable Python object *callable* with exactly 1 positional argument + *arg* and no keyword arguments. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + .. versionadded:: 3.9 + + .. c:function:: PyObject* PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) Call a callable Python object *callable*, with arguments given by the @@ -261,7 +283,8 @@ Object Protocol *args* must not be *NULL*, use an empty tuple if no arguments are needed. If no named arguments are needed, *kwargs* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args, **kwargs)``. @@ -272,7 +295,8 @@ Object Protocol Call a callable Python object *callable*, with arguments given by the tuple *args*. If no arguments are needed, then *args* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -283,7 +307,8 @@ Object Protocol The C arguments are described using a :c:func:`Py_BuildValue` style format string. The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -302,7 +327,8 @@ Object Protocol The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``obj.name(arg1, arg2, ...)``. @@ -320,7 +346,8 @@ Object Protocol :c:type:`PyObject\*` arguments. The arguments are provided as a variable number of parameters followed by *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(arg1, arg2, ...)``. @@ -331,9 +358,134 @@ Object Protocol Calls a method of the Python object *obj*, where the name of the method is given as a Python string object in *name*. It is called with a variable number of :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or + of parameters followed by *NULL*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + +.. c:function:: PyObject* _PyObject_CallMethodNoArgs(PyObject *obj, PyObject *name) + + Call a method of the Python object *obj* without arguments, + where the name of the method is given as a Python string object in *name*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* _PyObject_CallMethodOneArg(PyObject *obj, PyObject *name, PyObject *arg) + + Call a method of the Python object *obj* with a single positional argument + *arg*, where the name of the method is given as a Python string object in + *name*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a callable Python object *callable*, using + :c:data:`vectorcall ` if possible. + + *args* is a C array with the positional arguments. + + *nargsf* is the number of positional arguments plus optionally the flag + :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` (see below). + To get actual number of arguments, use + :c:func:`PyVectorcall_NARGS(nargsf) `. + + *kwnames* can be either NULL (no keyword arguments) or a tuple of keyword + names. In the latter case, the values of the keyword arguments are stored + in *args* after the positional arguments. + The number of keyword arguments does not influence *nargsf*. + + *kwnames* must contain only objects of type ``str`` (not a subclass), + and all keys must be unique. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This uses the vectorcall protocol if the callable supports it; + otherwise, the arguments are converted to use + :c:member:`~PyTypeObject.tp_call`. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + +.. c:var:: PY_VECTORCALL_ARGUMENTS_OFFSET + + If set in a vectorcall *nargsf* argument, the callee is allowed to + temporarily change ``args[-1]``. In other words, *args* points to + argument 1 (not 0) in the allocated vector. + The callee must restore the value of ``args[-1]`` before returning. + + For :c:func:`_PyObject_VectorcallMethod`, this flag means instead that + ``args[0]`` may be changed. + + Whenever they can do so cheaply (without additional allocation), callers + are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`. + Doing so will allow callables such as bound methods to make their onward + calls (which include a prepended *self* argument) cheaply. + + .. versionadded:: 3.8 + +.. c:function:: Py_ssize_t PyVectorcall_NARGS(size_t nargsf) + + Given a vectorcall *nargsf* argument, return the actual number of + arguments. + Currently equivalent to ``nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET``. + + .. versionadded:: 3.8 + +.. c:function:: PyObject* _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) + + Same as :c:func:`_PyObject_Vectorcall` except that the keyword arguments + are passed as a dictionary in *kwdict*. This may be *NULL* if there + are no keyword arguments. + + For callables supporting :c:data:`vectorcall `, + the arguments are internally converted to the vectorcall convention. + Therefore, this function adds some overhead compared to + :c:func:`_PyObject_Vectorcall`. + It should only be used if the caller already has a dictionary ready to use. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + +.. c:function:: PyObject* _PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a method using the vectorcall calling convention. The name of the method + is given as Python string *name*. The object whose method is called is + *args[0]* and the *args* array starting at *args[1]* represents the arguments + of the call. There must be at least one positional argument. + *nargsf* is the number of positional arguments including *args[0]*, + plus :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may + temporarily be changed. Keyword arguments can be passed just like in + :c:func:`_PyObject_Vectorcall`. + + If the object has the :const:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature, + this will actually call the unbound method object with the full + *args* vector as arguments. + + Return the result of the call on success, or raise an exception and return *NULL* on failure. + .. versionadded:: 3.9 .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 7023e519eda7b3..8bd8c107c98bdf 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _newtypes: diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 225a1feb250684..6b07c87d62eded 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _countingrefs: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 96893652f730cc..080ea3222cfbb2 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _reflection: diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index 6d22f35e22b1f2..d11a2dde54dd9e 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _sequence: diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 64b6dde3b7482e..074fcb877e1da7 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _setobjects: diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 8ad9a29b256ebc..d924308890b8d0 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _slice-objects: diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b771dd4adcefb..9c05cb3c82dfbe 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _stable: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index da45da1d3c70db..5184ad511cd9b1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _common-structs: @@ -114,10 +114,20 @@ the definition of all other Python objects. .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. .. c:type:: PyMethodDef @@ -149,10 +159,11 @@ specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are four basic calling conventions for positional arguments +and two of them can be combined with :const:`METH_KEYWORDS` to support +also keyword arguments. So there are a total of 6 calling conventions: .. data:: METH_VARARGS @@ -164,13 +175,41 @@ can be combined with a binding flag. using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly *NULL* + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject\*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the vectorcall protocol: + there is an additional fourth :c:type:`PyObject\*` parameter + which is a tuple representing the names of the keyword arguments + or possibly *NULL* if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 .. data:: METH_NOARGS diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index c8ea78363dfa51..838a97cacfdf10 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _os: @@ -289,6 +289,52 @@ accessible to C code. They all work with the current interpreter thread's .. versionadded:: 3.2 +.. c:function:: int PySys_Audit(const char *event, const char *format, ...) + + Raises an auditing event with any active hooks. Returns zero for success + and non-zero with an exception set on failure. + + If any hooks have been added, *format* and other arguments will be used + to construct a tuple to pass. Apart from ``N``, the same format characters + as used in :c:func:`Py_BuildValue` are available. If the built value is not + a tuple, it will be added into a single-element tuple. (The ``N`` format + option consumes a reference, but since there is no way to know whether + arguments to this function will be consumed, using it may cause reference + leaks.) + + :func:`sys.audit` performs the same function from Python code. + + .. versionadded:: 3.8 + + +.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) + + Adds to the collection of active auditing hooks. Returns zero for success + and non-zero on failure. If the runtime has been initialized, also sets an + error on failure. Hooks added through this API are called for all + interpreters created by the runtime. + + This function is safe to call before :c:func:`Py_Initialize`. When called + after runtime initialization, existing audit hooks are notified and may + silently abort the operation by raising an error subclassed from + :class:`Exception` (other errors will not be silenced). + + The hook function is of type :c:type:`int (*)(const char *event, PyObject + *args, void *userData)`, where *args* is guaranteed to be a + :c:type:`PyTupleObject`. The hook function is always called with the GIL + held by the Python interpreter that raised the event. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + See :pep:`578` for a detailed description of auditing. Functions in the + runtime and standard library that raise events include the details in each + function's documentation. + + .. versionadded:: 3.8 + + .. _processcontrol: Process Control diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 20bf9f0b0804cd..259ec4fb48515b 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _tupleobjects: diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 4dfd53fb9f076f..8f8367ab77c8c4 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _typeobjects: @@ -95,18 +95,6 @@ Type Objects from a type's base class. Return ``0`` on success, or return ``-1`` and sets an exception on error. -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - - Creates and returns a heap type object from the *spec* passed to the function. - -.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - - Creates and returns a heap type object from the *spec*. In addition to that, - the created heap type contains all types contained by the *bases* tuple as base - types. This allows the caller to reference other heap types as base types. - - .. versionadded:: 3.3 - .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -115,4 +103,107 @@ Type Objects Callers will typically cast the result pointer into the appropriate function type. + See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument. + + An exception is raised if *type* is not a heap type. + .. versionadded:: 3.4 + + +Creating Heap-Allocated Types +............................. + +The following functions and structs are used to create +:ref:`heap types `. + +.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) + + Creates and returns a heap type object from the *spec*. + + If *bases* is a tuple, the created heap type contains all types contained + in it as base types. + + If *bases* is *NULL*, the *Py_tp_base* slot is used instead. + If that also is *NULL*, the new type derives from :class:`object`. + + This function calls :c:func:`PyType_Ready` on the new type. + + .. versionadded:: 3.3 + +.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) + + Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``. + +.. c:type:: PyType_Spec + + Structure defining a type's behavior. + + .. c:member:: const char* PyType_Spec.name + + Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + + .. c:member:: const char* PyType_Spec.doc + + Type docstring, used to set :c:member:`PyTypeObject.tp_doc`. + + .. c:member:: int PyType_Spec.basicsize + .. c:member:: int PyType_Spec.itemsize + + Size of the instance in bytes, used to set + :c:member:`PyTypeObject.tp_basicsize` and + :c:member:`PyTypeObject.tp_itemsize`. + + .. c:member:: int PyType_Spec.flags + + Type flags, used to set :c:member:`PyTypeObject.tp_flags`. + + If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + .. c:member:: PyType_Slot *PyType_Spec.slots + + Array of :c:type:`PyType_Slot` structures. + Terminated by the special slot value ``{0, NULL}``. + +.. c:type:: PyType_Slot + + Structure defining optional functionality of a type, containing a slot ID + and a value pointer. + + .. c:member:: int PyType_Slot.slot + + A slot ID. + + Slot IDs are named like the field names of the structures + :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, + :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and + :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. + For example, use: + + * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` + * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` + * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + + The following fields cannot be set using *PyType_Spec* and *PyType_Slot*: + + * :c:member:`~PyTypeObject.tp_dict` + * :c:member:`~PyTypeObject.tp_mro` + * :c:member:`~PyTypeObject.tp_cache` + * :c:member:`~PyTypeObject.tp_subclasses` + * :c:member:`~PyTypeObject.tp_weaklist` + * :c:member:`~PyTypeObject.tp_print` + * :c:member:`~PyTypeObject.tp_weaklistoffset` + * :c:member:`~PyTypeObject.tp_dictoffset` + * :c:member:`~PyBufferProcs.bf_getbuffer` + * :c:member:`~PyBufferProcs.bf_releasebuffer` + + Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + To avoid issues, use the *bases* argument of + :py:func:`PyType_FromSpecWithBases` instead. + + .. c:member:: void *PyType_Slot.pfunc + + The desired value of the slot. In most cases, this is a pointer + to a function. + + May not be *NULL*. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0647a493303d26..638fb0c32d788b 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _type-structs: @@ -36,115 +36,115 @@ Quick Reference .. table:: :widths: 18,18,18,1,1,1,1 - +---------------------------------------------+-----------------------------------+-------------------+---------------+ - | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | - | | | methods/attrs +---+---+---+---+ - | | | | O | T | D | I | - +=============================================+===================================+===================+===+===+===+===+ - | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_print`) | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | - | | | __le__, | | | | | - | | | __eq__, | | | | | - | | | __ne__, | | | | | - | | | __gt__, | | | | | - | | | __ge__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | - | | | __delete__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + +------------------------------------------------+-----------------------------------+-------------------+---------------+ + | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | + | | | methods/attrs +---+---+---+---+ + | | | | O | T | D | I | + +================================================+===================================+===================+===+===+===+===+ + | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_vectorcall_offset` | Py_ssize_t | | | | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | + | | | __le__, | | | | | + | | | __eq__, | | | | | + | | | __ne__, | | | | | + | | | __gt__, | | | | | + | | | __ge__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | + | | | __delete__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ If :const:`COUNT_ALLOCS` is defined then the following (internal-only) fields exist as well: @@ -285,7 +285,7 @@ sub-slots +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __truediv__ | +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`binaryfunc` | __index__ | + | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | | | | __rmatmul__ | @@ -364,12 +364,6 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`reprfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`printfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | FILE * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ | :c:type:`getattrfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyObject` * | | @@ -675,9 +669,66 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes. -.. c:member:: printfunc PyTypeObject.tp_print +.. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset + + An optional offset to a per-instance function that implements calling + the object using the *vectorcall* protocol, a more efficient alternative + of the simpler :c:member:`~PyTypeObject.tp_call`. + + This field is only used if the flag :const:`_Py_TPFLAGS_HAVE_VECTORCALL` + is set. If so, this must be a positive integer containing the offset in the + instance of a :c:type:`vectorcallfunc` pointer. + The signature is the same as for :c:func:`_PyObject_Vectorcall`:: + + PyObject *vectorcallfunc(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + The *vectorcallfunc* pointer may be zero, in which case the instance behaves + as if :const:`_Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance + falls back to :c:member:`~PyTypeObject.tp_call`. - Reserved slot, formerly used for print formatting in Python 2.x. + Any class that sets ``_Py_TPFLAGS_HAVE_VECTORCALL`` must also set + :c:member:`~PyTypeObject.tp_call` and make sure its behaviour is consistent + with the *vectorcallfunc* function. + This can be done by setting *tp_call* to ``PyVectorcall_Call``: + + .. c:function:: PyObject *PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) + + Call *callable*'s *vectorcallfunc* with positional and keyword + arguments given in a tuple and dict, respectively. + + This function is intended to be used in the ``tp_call`` slot. + It does not fall back to ``tp_call`` and it currently does not check the + ``_Py_TPFLAGS_HAVE_VECTORCALL`` flag. + To call an object, use one of the :c:func:`PyObject_Call ` + functions instead. + + .. note:: + + It is not recommended for :ref:`heap types ` to implement + the vectorcall protocol. + When a user sets ``__call__`` in Python code, only ``tp_call`` is updated, + possibly making it inconsistent with the vectorcall function. + + .. note:: + + The semantics of the ``tp_vectorcall_offset`` slot are provisional and + expected to be finalized in Python 3.9. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionchanged:: 3.8 + + This slot was used for print formatting in Python 2.x. + In Python 3.0 to 3.7, it was reserved and named ``tp_print``. + + **Inheritance:** + + This field is inherited by subtypes together with + :c:member:`~PyTypeObject.tp_call`: a subtype inherits + :c:member:`~PyTypeObject.tp_vectorcall_offset` from its base type when + the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL. + + Note that `heap types`_ (including subclasses defined in Python) do not + inherit the :const:`_Py_TPFLAGS_HAVE_VECTORCALL` flag. .. c:member:: getattrfunc PyTypeObject.tp_getattr @@ -1045,6 +1096,32 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? + + .. data:: Py_TPFLAGS_METHOD_DESCRIPTOR + + This bit indicates that objects behave like unbound methods. + + If this flag is set for ``type(meth)``, then: + + - ``meth.__get__(obj, cls)(*args, **kwds)`` (with ``obj`` not None) + must be equivalent to ``meth(obj, *args, **kwds)``. + + - ``meth.__get__(None, cls)(*args, **kwds)`` + must be equivalent to ``meth(*args, **kwds)``. + + This flag enables an optimization for typical method calls like + ``obj.meth()``: it avoids creating a temporary "bound method" object for + ``obj.meth``. + + .. versionadded:: 3.8 + + **Inheritance:** + + This flag is never inherited by heap types. + For extension types, it is inherited whenever + :c:member:`~PyTypeObject.tp_descr_get` is inherited. + + .. XXX Document more flags here? @@ -1073,6 +1150,33 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.4 + .. deprecated:: 3.8 + This flag isn't necessary anymore, as the interpreter assumes the + :c:member:`~PyTypeObject.tp_finalize` slot is always present in the + type structure. + + .. data:: _Py_TPFLAGS_HAVE_VECTORCALL + + This bit is set when the class implements the vectorcall protocol. + See :c:member:`~PyTypeObject.tp_vectorcall_offset` for details. + + **Inheritance:** + + This bit is set on *static* subtypes if ``tp_flags`` is not overridden: + a subtype inherits ``_Py_TPFLAGS_HAVE_VECTORCALL`` from its base type + when the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL + and the subtype's ``Py_TPFLAGS_HEAPTYPE`` is not set. + + `Heap types`_ do not inherit ``_Py_TPFLAGS_HAVE_VECTORCALL``. + + .. note:: + + This flag is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + .. c:member:: const char* PyTypeObject.tp_doc @@ -1822,16 +1926,35 @@ objects on the thread which called tp_dealloc will not violate any assumptions of the library. +.. _heap-types: + Heap Types ---------- -In addition to defining Python types statically, you can define them -dynamically (i.e. to the heap) using :c:func:`PyType_FromSpec` and -:c:func:`PyType_FromSpecWithBases`. +Traditionally, types defined in C code are *static*, that is, +a static :c:type:`PyTypeObject` structure is defined directly in code +and initialized using :c:func:`PyType_Ready`. + +This results in types that are limited relative to types defined in Python: + +* Static types are limited to one base, i.e. they cannot use multiple + inheritance. +* Static type objects (but not necessarily their instances) are immutable. + It is not possible to add or modify the type object's attributes from Python. +* Static type objects are shared across + :ref:`sub-interpreters `, so they should not + include any subinterpreter-specific state. + +Also, since *PyTypeObject* is not part of the :ref:`stable ABI `, +any extension modules using static types must be compiled for a specific +Python minor version. -.. XXX Explain how to use PyType_FromSpec(). +An alternative to static types is *heap-allocated types*, or *heap types* +for short, which correspond closely to classes created by Python's +``class`` statement. -.. XXX Document PyType_Spec. +This is done by filling a :c:type:`PyType_Spec` structure and calling +:c:func:`PyType_FromSpecWithBases`. .. _number-structs: @@ -2236,6 +2359,14 @@ Slot Type typedefs .. c:type:: void (*destructor)(PyObject *) +.. c:type:: PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + See :c:member:`~PyTypeObject.tp_vectorcall_offset`. + + Arguments to ``vectorcallfunc`` are the same as for :c:func:`_PyObject_Vectorcall`. + + .. versionadded:: 3.8 + .. c:type:: void (*freefunc)(void *) See :c:member:`~PyTypeObject.tp_free`. @@ -2252,10 +2383,6 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_repr`. -.. c:type:: int (*printfunc)(PyObject *, FILE *, int) - - This is hidden if :const:`PY_LIMITED_API` is set. - .. c:type:: PyObject *(*getattrfunc)(PyObject *self, char *attr) Return the value of the named attribute for the object. @@ -2359,7 +2486,7 @@ with a more verbose initializer:: sizeof(MyObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)myobj_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 97c5ebcb10953c..a150a9c39d880c 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _unicodeobjects: @@ -22,14 +22,14 @@ in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated and inefficient; it should be avoided in performance- or memory-sensitive situations. -Due to the transition between the old APIs and the new APIs, unicode objects +Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: -* "canonical" unicode objects are all objects created by a non-deprecated - unicode API. They use the most efficient representation allowed by the +* "canonical" Unicode objects are all objects created by a non-deprecated + Unicode API. They use the most efficient representation allowed by the implementation. -* "legacy" unicode objects have been created through one of the deprecated +* "legacy" Unicode objects have been created through one of the deprecated APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the :c:type:`Py_UNICODE*` representation; you will have to call :c:func:`PyUnicode_READY` on them before calling any other API. @@ -152,7 +152,7 @@ access internal read-only data of Unicode objects: .. c:function:: void* PyUnicode_DATA(PyObject *o) - Return a void pointer to the raw unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -430,7 +430,7 @@ APIs: .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python unicode string and return + arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* ASCII-encoded string. The following format characters are allowed: @@ -504,9 +504,9 @@ APIs: | :attr:`%A` | PyObject\* | The result of calling | | | | :func:`ascii`. | +-------------------+---------------------+--------------------------------+ - | :attr:`%U` | PyObject\* | A unicode object. | + | :attr:`%U` | PyObject\* | A Unicode object. | +-------------------+---------------------+--------------------------------+ - | :attr:`%V` | PyObject\*, | A unicode object (which may be | + | :attr:`%V` | PyObject\*, | A Unicode object (which may be | | | const char\* | *NULL*) and a null-terminated | | | | C character array as a second | | | | parameter (which will be used, | @@ -1670,7 +1670,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) - Compare a unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1680,7 +1680,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) - Rich compare two unicode strings and return one of the following: + Rich compare two Unicode strings and return one of the following: * ``NULL`` in case an exception was raised * :const:`Py_True` or :const:`Py_False` for successful comparisons @@ -1708,7 +1708,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: void PyUnicode_InternInPlace(PyObject **string) Intern the argument *\*string* in place. The argument must be the address of a - pointer variable pointing to a Python unicode string object. If there is an + pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to it (decrementing the reference count of the old string object and incrementing the reference count of the interned string object), otherwise it leaves @@ -1721,6 +1721,6 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_InternFromString(const char *v) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string + :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string object that has been interned, or a new ("owned") reference to an earlier interned string object with the same value. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index d4484fb2712817..a805b564763c40 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _utilities: diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 317093e95615e3..e6704ddeca0906 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _veryhigh: @@ -42,6 +42,13 @@ the same library that the Python runtime is using. ``Py_InspectFlag`` is not set. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. + + .. versionadded:: 3.8 + + .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving @@ -381,11 +388,22 @@ the same library that the Python runtime is using. Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: + discarded. + + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in *cf_flags*. - struct PyCompilerFlags { - int cf_flags; - } + .. versionchanged:: 3.8 + Added *cf_feature_version* field. .. c:var:: int CO_FUTURE_DIVISION diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 6cb3e33fe843ac..7a4f8615b9c32b 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _weakrefobjects: diff --git a/Doc/conf.py b/Doc/conf.py index afe66270c10e8c..e85ea5b2d2ff49 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -23,6 +23,9 @@ except ImportError: _tkinter = None ''' + +manpages_url = 'https://manpages.debian.org/{path}' + # General substitutions. project = 'Python' copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 35527c179f3481..fda347eab10221 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -234,6 +234,24 @@ PyCode_Check:PyObject*:co:0: PyCode_GetNumFree:int::: PyCode_GetNumFree:PyCodeObject*:co:0: +PyCode_NewWithPosOnlyArgs:PyCodeObject*::+1: +PyCode_NewWithPosOnlyArgs:int:argcount:: +PyCode_NewWithPosOnlyArgs:int:posonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:kwonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:nlocals:: +PyCode_NewWithPosOnlyArgs:int:stacksize:: +PyCode_NewWithPosOnlyArgs:int:flags:: +PyCode_NewWithPosOnlyArgs:PyObject*:code:0: +PyCode_NewWithPosOnlyArgs:PyObject*:consts:0: +PyCode_NewWithPosOnlyArgs:PyObject*:names:0: +PyCode_NewWithPosOnlyArgs:PyObject*:varnames:0: +PyCode_NewWithPosOnlyArgs:PyObject*:freevars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:cellvars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:filename:0: +PyCode_NewWithPosOnlyArgs:PyObject*:name:0: +PyCode_NewWithPosOnlyArgs:int:firstlineno:: +PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0: + PyCode_New:PyCodeObject*::+1: PyCode_New:int:argcount:: PyCode_New:int:kwonlyargcount:: @@ -413,6 +431,16 @@ PyDateTime_FromDateAndTime:int:minute:: PyDateTime_FromDateAndTime:int:second:: PyDateTime_FromDateAndTime:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:PyObject*::+1: +PyDateTime_FromDateAndTimeAndFold:int:year:: +PyDateTime_FromDateAndTimeAndFold:int:month:: +PyDateTime_FromDateAndTimeAndFold:int:day:: +PyDateTime_FromDateAndTimeAndFold:int:hour:: +PyDateTime_FromDateAndTimeAndFold:int:minute:: +PyDateTime_FromDateAndTimeAndFold:int:second:: +PyDateTime_FromDateAndTimeAndFold:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:int:fold:: + PyDateTime_FromTimestamp:PyObject*::+1: PyDateTime_FromTimestamp:PyObject*:args:0: @@ -2210,6 +2238,13 @@ PyTime_FromTime:int:minute:: PyTime_FromTime:int:second:: PyTime_FromTime:int:usecond:: +PyTime_FromTimeAndFold:PyObject*::+1: +PyTime_FromTimeAndFold:int:hour:: +PyTime_FromTimeAndFold:int:minute:: +PyTime_FromTimeAndFold:int:second:: +PyTime_FromTimeAndFold:int:usecond:: +PyTime_FromTimeAndFold:int:fold:: + PyTraceMalloc_Track:int::: PyTraceMalloc_Track:unsigned int:domain:: PyTraceMalloc_Track:uintptr_t:ptr:: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5dd14b1f7a60b9..2e46c7ac8635e8 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -113,11 +113,17 @@ recommended tools`_. .. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations -Reading the guide -================= +.. index:: + single: Python Package Index (PyPI) + single: PyPI; (see Python Package Index (PyPI)) + +.. _publishing-python-packages: + +Reading the Python Packaging User Guide +======================================= The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: +involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ diff --git a/Doc/distutils/_setuptools_disclaimer.rst b/Doc/distutils/_setuptools_disclaimer.rst new file mode 100644 index 00000000000000..cc75858326d44d --- /dev/null +++ b/Doc/distutils/_setuptools_disclaimer.rst @@ -0,0 +1,5 @@ +.. note:: + + This document is being retained solely until the ``setuptools`` documentation + at https://setuptools.readthedocs.io/en/latest/setuptools.html + independently covers all of the relevant information currently included here. diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 1facc0408d5b0f..937f19f57be95a 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -4,6 +4,16 @@ API Reference ************* +.. seealso:: + + `New and changed setup.py arguments in setuptools `_ + The ``setuptools`` project adds new capabilities to the ``setup`` function + and other APIs, makes the API consistent across different Python versions, + and is hence recommended over using ``distutils`` directly. + +.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords + +.. include:: ./_setuptools_disclaimer.rst :mod:`distutils.core` --- Core Distutils functionality ====================================================== @@ -280,7 +290,7 @@ the full reference. .. versionchanged:: 3.8 On Unix, C extensions are no longer linked to libpython except on - Android. + Android and Cygwin. .. class:: Distribution @@ -1853,6 +1863,9 @@ Subclasses of :class:`Command` must define the following methods. .. module:: distutils.command.bdist_wininst :synopsis: Build a Windows installer +.. deprecated:: 3.8 + Use bdist_wheel (wheel packages) instead. + .. % todo diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index f1f3471261600e..b814f2e9508c9c 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -4,6 +4,8 @@ Creating Built Distributions **************************** +.. include:: ./_setuptools_disclaimer.rst + A "built distribution" is what you're probably used to thinking of either as a "binary package" or an "installer" (depending on your background). It's not necessarily binary, though, because it might contain only Python source code @@ -144,6 +146,9 @@ generated by each, are: | :command:`bdist_msi` | msi | +--------------------------+-------------------------------------+ +.. note:: + bdist_wininst is deprecated since Python 3.8. + The following sections give details on the individual :command:`bdist_\*` commands. @@ -296,6 +301,9 @@ file winds up deep in the "build tree," in a temporary directory created by Creating Windows Installers =========================== +.. warning:: + bdist_wininst is deprecated since Python 3.8. + Executable installers are the natural format for binary distributions on Windows. They display a nice graphical user interface, display some information about the module distribution to be installed taken from the metadata in the @@ -313,8 +321,8 @@ or the :command:`bdist` command with the :option:`!--formats` option:: If you have a pure module distribution (only containing pure Python modules and packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix -platforms or Mac OS X. +like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary +distributions in only supported on Windows systems. If you have a non-pure distribution, the extensions can only be created on a Windows platform, and will be Python version dependent. The installer filename @@ -457,3 +465,6 @@ Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-contr option. The default is 'none' (meaning no UAC handling is done), and other valid values are 'auto' (meaning prompt for UAC elevation if Python was installed for all users) and 'force' (meaning always prompt for elevation). + +.. note:: + bdist_wininst is deprecated since Python 3.8. diff --git a/Doc/distutils/commandref.rst b/Doc/distutils/commandref.rst index 6a2ac960f1e1ef..0f6fe2aba865f8 100644 --- a/Doc/distutils/commandref.rst +++ b/Doc/distutils/commandref.rst @@ -4,6 +4,8 @@ Command Reference ***************** +.. include:: ./_setuptools_disclaimer.rst + .. % \section{Building modules: the \protect\command{build} command family} .. % \label{build-cmds} .. % \subsubsection{\protect\command{build}} diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst index 0874d05fe703a1..2a5c8329e318b7 100644 --- a/Doc/distutils/configfile.rst +++ b/Doc/distutils/configfile.rst @@ -4,6 +4,8 @@ Writing the Setup Configuration File ************************************ +.. include:: ./_setuptools_disclaimer.rst + Often, it's not possible to write down everything needed to build a distribution *a priori*: you may need to get some information from the user, or from the user's system, in order to proceed. As long as that information is fairly diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index f81e06b5e60578..4ac552c7c6997a 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -4,6 +4,8 @@ Examples ******** +.. include:: ./_setuptools_disclaimer.rst + This chapter provides a number of basic examples to help get started with distutils. Additional information about using distutils can be found in the Distutils Cookbook. diff --git a/Doc/distutils/extending.rst b/Doc/distutils/extending.rst index 501fd7c564c6b9..1075e81779a7ba 100644 --- a/Doc/distutils/extending.rst +++ b/Doc/distutils/extending.rst @@ -4,6 +4,8 @@ Extending Distutils ******************* +.. include:: ./_setuptools_disclaimer.rst + Distutils can be extended in various ways. Most extensions take the form of new commands or replacements for existing commands. New commands may be written to support new types of platform-specific packaging, for example, while diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index d6f7640fcb6826..1f72a25542494a 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -12,10 +12,7 @@ :ref:`distributing-index` The up to date module distribution documentations -This document describes the Python Distribution Utilities ("Distutils") from -the module developer's point of view, describing how to use the Distutils to -make Python modules and extensions easily available to a wider audience with -very little overhead for build/release/install mechanics. +.. include:: ./_setuptools_disclaimer.rst .. note:: @@ -25,6 +22,11 @@ very little overhead for build/release/install mechanics. recommendations section `__ in the Python Packaging User Guide for more information. +This document describes the Python Distribution Utilities ("Distutils") from +the module developer's point of view, describing the underlying capabilities +that ``setuptools`` builds on to allow Python developers to make Python modules +and extensions readily available to a wider audience. + .. toctree:: :maxdepth: 2 :numbered: @@ -34,7 +36,6 @@ very little overhead for build/release/install mechanics. configfile.rst sourcedist.rst builtdist.rst - packageindex.rst examples.rst extending.rst commandref.rst diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst index 7721484fe73717..1f8a560e138616 100644 --- a/Doc/distutils/introduction.rst +++ b/Doc/distutils/introduction.rst @@ -4,6 +4,8 @@ An Introduction to Distutils **************************** +.. include:: ./_setuptools_disclaimer.rst + This document covers using the Distutils to distribute your Python modules, concentrating on the role of developer/distributor: if you're looking for information on installing Python modules, you should refer to the diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst index 50cb74f2b6ca90..ccb9a598b2b7a2 100644 --- a/Doc/distutils/packageindex.rst +++ b/Doc/distutils/packageindex.rst @@ -1,6 +1,4 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) +:orphan: .. _package-index: @@ -8,246 +6,11 @@ The Python Package Index (PyPI) ******************************* -The `Python Package Index (PyPI)`_ stores :ref:`meta-data ` -describing distributions packaged with distutils, as well as package data like -distribution files if a package author wishes. - -Distutils provides the :command:`register` and :command:`upload` commands for -pushing meta-data and distribution files to PyPI, respectively. See -:ref:`package-commands` for information on these commands. - - -PyPI overview -============= - -PyPI lets you submit any number of versions of your distribution to the index. -If you alter the meta-data for a particular version, you can submit it again -and the index will be updated. - -PyPI holds a record for each (name, version) combination submitted. The first -user to submit information for a given name is designated the Owner of that -name. Changes can be submitted through the :command:`register` command or -through the web interface. Owners can designate other users as Owners or -Maintainers. Maintainers can edit the package information, but not designate -new Owners or Maintainers. - -By default PyPI displays only the newest version of a given package. The web -interface lets one change this default behavior and manually select which -versions to display and hide. - -For each version, PyPI displays a home page. The home page is created from -the ``long_description`` which can be submitted via the :command:`register` -command. See :ref:`package-display` for more information. - - -.. _package-commands: - -Distutils commands -================== - -Distutils exposes two commands for submitting package data to PyPI: the -:ref:`register ` command for submitting meta-data to PyPI -and the :ref:`upload ` command for submitting distribution -files. Both commands read configuration data from a special file called a -:ref:`.pypirc file `. - - -.. _package-register: - -The ``register`` command ------------------------- - -The distutils command :command:`register` is used to submit your distribution's -meta-data to an index server. It is invoked as follows:: - - python setup.py register - -Distutils will respond with the following prompt:: - - running register - We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit - Your selection [default 1]: - -Note: if your username and password are saved locally, you will not see this -menu. Also, refer to :ref:`pypirc` for how to store your credentials in a -:file:`.pypirc` file. - -If you have not registered with PyPI, then you will need to do so now. You -should choose option 2, and enter your details as required. Soon after -submitting your details, you will receive an email which will be used to confirm -your registration. - -Once you are registered, you may choose option 1 from the menu. You will be -prompted for your PyPI username and password, and :command:`register` will then -submit your meta-data to the index. - -See :ref:`package-cmdoptions` for options to the :command:`register` command. - - -.. _package-upload: - -The ``upload`` command ----------------------- - -The distutils command :command:`upload` pushes the distribution files to PyPI. - -The command is invoked immediately after building one or more distribution -files. For example, the command :: - - python setup.py sdist bdist_wininst upload - -will cause the source distribution and the Windows installer to be uploaded to -PyPI. Note that these will be uploaded even if they are built using an earlier -invocation of :file:`setup.py`, but that only distributions named on the command -line for the invocation including the :command:`upload` command are uploaded. - -If a :command:`register` command was previously called in the same command, -and if the password was entered in the prompt, :command:`upload` will reuse the -entered password. This is useful if you do not want to store a password in -clear text in a :file:`.pypirc` file. - -You can use the ``--sign`` option to tell :command:`upload` to sign each -uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must -be available for execution on the system :envvar:`PATH`. You can also specify -which key to use for signing using the ``--identity=name`` option. - -See :ref:`package-cmdoptions` for additional options to the :command:`upload` -command. - - -.. _package-cmdoptions: - -Additional command options --------------------------- - -This section describes options common to both the :command:`register` and -:command:`upload` commands. - -The ``--repository`` or ``-r`` option lets you specify a PyPI server -different from the default. For example:: - - python setup.py sdist bdist_wininst upload -r https://example.com/pypi - -For convenience, a name can be used in place of the URL when the -:file:`.pypirc` file is configured to do so. For example:: - - python setup.py register -r other - -See :ref:`pypirc` for more information on defining alternate servers. - -The ``--show-response`` option displays the full response text from the PyPI -server, which is useful when debugging problems with registering and uploading. - - -.. index:: - single: .pypirc file - single: Python Package Index (PyPI); .pypirc file - -.. _pypirc: - -The ``.pypirc`` file --------------------- - -The :command:`register` and :command:`upload` commands both check for the -existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. -If this file exists, the command uses the username, password, and repository -URL configured in the file. The format of a :file:`.pypirc` file is as -follows: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - - [pypi] - repository: - username: - password: - -The *distutils* section defines an *index-servers* variable that lists the -name of all sections describing a repository. - -Each section describing a repository defines three variables: - -- *repository*, that defines the url of the PyPI server. Defaults to - ``https://upload.pypi.org/legacy/``. -- *username*, which is the registered username on the PyPI server. -- *password*, that will be used to authenticate. If omitted the user - will be prompt to type it when needed. - -If you want to define another server a new section can be created and -listed in the *index-servers* variable: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - other - - [pypi] - repository: - username: - password: - - [other] - repository: https://example.com/pypi - username: - password: - -This allows the :command:`register` and :command:`upload` commands to be -called with the ``--repository`` option as described in -:ref:`package-cmdoptions`. - -Specifically, you might want to add the `PyPI Test Repository -`_ to your ``.pypirc`` to facilitate -testing before doing your first upload to ``PyPI`` itself. - - -.. _package-display: - -PyPI package display -==================== - -The ``long_description`` field plays a special role at PyPI. It is used by -the server to display a home page for the registered package. - -If you use the `reStructuredText `_ -syntax for this field, PyPI will parse it and display an HTML output for -the package home page. - -The ``long_description`` field can be attached to a text file located -in the package:: - - from distutils.core import setup - - with open('README.txt') as file: - long_description = file.read() - - setup(name='Distutils', - long_description=long_description) - -In that case, :file:`README.txt` is a regular reStructuredText text file located -in the root of the package besides :file:`setup.py`. - -To prevent registering broken reStructuredText content, you can use the -:program:`rst2html` program that is provided by the :mod:`docutils` package and -check the ``long_description`` from the command line: - -.. code-block:: shell-session - - $ python setup.py --long-description | rst2html.py > output.html - -:mod:`docutils` will display a warning if there's something wrong with your -syntax. Because PyPI applies additional checks (e.g. by passing ``--no-raw`` -to ``rst2html.py`` in the command above), being able to run the command above -without warnings does not guarantee that PyPI will convert the content -successfully. +The `Python Package Index (PyPI)`_ stores metadata describing distributions +packaged with distutils and other publishing tools, as well the distribution +archives themselves. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. .. _Python Package Index (PyPI): https://pypi.org diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 54ed1aebc242d8..4386a60b664bfb 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -4,6 +4,8 @@ Writing the Setup Script ************************ +.. include:: ./_setuptools_disclaimer.rst + The setup script is the centre of all activity in building, distributing, and installing modules using the Distutils. The main purpose of the setup script is to describe your module distribution to the Distutils, so that the various @@ -523,7 +525,7 @@ following way:: setup(..., data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), - ('config', ['cfg/data.cfg']), + ('config', ['cfg/data.cfg'])], ) Each (*directory*, *files*) pair in the sequence specifies the installation @@ -612,9 +614,8 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - The ``long_description`` field is used by PyPI when you are - :ref:`registering ` a package, to - :ref:`build its home page `. + The ``long_description`` field is used by PyPI when you publish a package, + to build its project page. (5) The ``license`` field is a text indicating the license covering the diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst index 0ac8ef41ddc20f..0600663d00e9dc 100644 --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -4,6 +4,8 @@ Creating a Source Distribution ****************************** +.. include:: ./_setuptools_disclaimer.rst + As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command to create a source distribution. In the simplest case, :: diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst index 4bce6997f9f83a..4c391cab072ea6 100644 --- a/Doc/distutils/uploading.rst +++ b/Doc/distutils/uploading.rst @@ -4,4 +4,5 @@ Uploading Packages to the Package Index *************************************** -The contents of this page have moved to the section :ref:`package-index`. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 9bfad7fc3187d6..753b5511ed9d63 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _building: @@ -20,7 +20,7 @@ The initialization function has the signature: It returns either a fully-initialized module, or a :c:type:`PyModuleDef` instance. See :ref:`initializing-modules` for details. -.. highlightlang:: python +.. highlight:: python For modules with ASCII-only names, the function must be named ``PyInit_``, with ```` replaced by the name of the @@ -43,7 +43,7 @@ function corresponding to the filename is found. See the *"Multiple modules in one library"* section in :pep:`489` for details. -.. highlightlang:: c +.. highlight:: c Building C and C++ Extensions with distutils ============================================ diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 13d83b72f82ab6..483bc852f6c777 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _embedding: diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 433178ab64d8e5..e459514b2b5853 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _extending-intro: diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 8b9549d7e39e2e..7e0b18d6299fb3 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _new-types-topics: @@ -104,7 +104,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and /* This saves the current exception state */ PyErr_Fetch(&err_type, &err_value, &err_traceback); - cbresult = PyObject_CallObject(self->my_callback, NULL); + cbresult = PyObject_CallNoArgs(self->my_callback); if (cbresult == NULL) PyErr_WriteUnraisable(self->my_callback); else diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index b4bf9b9e6f75f8..59c8cc01222b2c 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _defining-new-types: diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 67bdd475aeb610..c7b92c6ea24ca8 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _building-on-windows: diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2d63a0323da66..387420c17bd152 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -24,14 +24,16 @@ programmers will encounter a fragment of code like this:: z++; Only the ``x++`` statement is executed if the condition is true, but the -indentation leads you to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering why ``y`` is being decremented even +indentation leads many to believe otherwise. Even experienced C programmers will +sometimes stare at it a long time wondering as to why ``y`` is being decremented even for ``x > y``. Because there are no begin/end brackets, Python is much less prone to coding-style conflicts. In C there are many different ways to place the braces. -If you're used to reading and writing code that uses one style, you will feel at -least slightly uneasy when reading (or being required to write) another style. +After becoming used to reading and writing code using a particular style, +it is normal to feel somewhat uneasy when reading (or being required to write) +in a different one. + Many coding styles place begin/end brackets on a line by themselves. This makes programs considerably longer and wastes valuable screen space, making it harder diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 4f9979bf55ed3a..781da467d18013 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -104,7 +104,7 @@ What platform-specific GUI toolkits exist for Python? ======================================================== By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's +`_, Python programs can use Mac OS X's Cocoa libraries. :ref:`Pythonwin ` by Mark Hammond includes an interface to the diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 31614189a62d2c..a36fa4aefe8864 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -16,6 +16,9 @@ Is there a source code level debugger with breakpoints, single-stepping, etc.? Yes. +Several debuggers for Python are described below, and the built-in function +:func:`breakpoint` allows you to drop into any of them. + The pdb module is a simple but adequate console-mode debugger for Python. It is part of the standard Python library, and is :mod:`documented in the Library Reference Manual `. You can also write your own debugger by using the code @@ -551,8 +554,8 @@ desired effect in a number of ways. 5) Or bundle up values in a class instance:: class callByRef: - def __init__(self, **args): - for (key, value) in args.items(): + def __init__(self, /, **args): + for key, value in args.items(): setattr(self, key, value) def func4(args): @@ -789,7 +792,7 @@ Its documentation looks like this:: invoked using the three argument form. The slash at the end of the parameter list means that all three parameters are -positional-only. Thus, calling :func:`pow` with keyword aguments would lead to +positional-only. Thus, calling :func:`pow` with keyword arguments would lead to an error:: >>> pow(x=3, y=4) @@ -797,10 +800,6 @@ an error:: File "", line 1, in TypeError: pow() takes no keyword arguments -Note that as of this writing this is only documentational and no valid syntax -in Python, although there is :pep:`570`, which proposes a syntax for -position-only parameters in Python. - Numbers and strings =================== diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index ad97cd0932ac86..a181086e9ce627 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -1,6 +1,6 @@ :tocdepth: 2 -.. highlightlang:: none +.. highlight:: none .. _windows-faq: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 5810a6b7997804..0f2a3a1fdf0510 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -15,10 +15,10 @@ Glossary ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering code for - an indented code block, when within a pair of matching left and right - delimiters (parentheses, square brackets, curly braces or triple quotes), - or after specifying a decorator. + * The default Python prompt of the interactive shell when entering the + code for an indented code block, when within a pair of matching left and + right delimiters (parentheses, square brackets, curly braces or triple + quotes), or after specifying a decorator. * The :const:`Ellipsis` built-in constant. @@ -225,6 +225,15 @@ Glossary statement by defining :meth:`__enter__` and :meth:`__exit__` methods. See :pep:`343`. + context variable + A variable which can have different values depending on its context. + This is similar to Thread-Local Storage in which each execution + thread may have a different value for a variable. However, with context + variables, there may be several contexts in one execution thread and the + main usage for context variables is to keep track of variables in + concurrent asynchronous tasks. + See :mod:`contextvars`. + contiguous .. index:: C-contiguous, Fortran contiguous @@ -503,8 +512,10 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not. Objects which are + Most of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not; immutable + containers (such as tuples and frozensets) are only hashable if + their elements are hashable. Objects which are instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 5b2457a1682bc6..cfd9f2e4075c4a 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c ********************** Argument Clinic How-To diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst index b638e32f5d0ecb..ce7700fc599062 100644 --- a/Doc/howto/cporting.rst +++ b/Doc/howto/cporting.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _cporting-howto: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29e590b20cba8..324625d7b3e809 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -145,7 +145,7 @@ print a message for each get or set. Overriding :meth:`__getattribute__` is alternate approach that could do this for every attribute. However, this descriptor is useful for monitoring just a few chosen attributes:: - class RevealAccess(object): + class RevealAccess: """A data descriptor that sets and returns values normally and prints a message logging their access. """ @@ -162,7 +162,7 @@ descriptor is useful for monitoring just a few chosen attributes:: print('Updating', self.name) self.val = val - >>> class MyClass(object): + >>> class MyClass: ... x = RevealAccess(10, 'var "x"') ... y = 5 ... @@ -194,7 +194,7 @@ triggers function calls upon access to an attribute. Its signature is:: The documentation shows a typical use to define a managed attribute ``x``:: - class C(object): + class C: def getx(self): return self.__x def setx(self, value): self.__x = value def delx(self): del self.__x @@ -203,7 +203,7 @@ The documentation shows a typical use to define a managed attribute ``x``:: To see how :func:`property` is implemented in terms of the descriptor protocol, here is a pure Python equivalent:: - class Property(object): + class Property: "Emulate PyProperty_Type() in Objects/descrobject.c" def __init__(self, fget=None, fset=None, fdel=None, doc=None): @@ -250,7 +250,7 @@ to be recalculated on every access; however, the programmer does not want to affect existing client code accessing the attribute directly. The solution is to wrap access to the value attribute in a property data descriptor:: - class Cell(object): + class Cell: . . . def getvalue(self): "Recalculate the cell before returning value" @@ -277,7 +277,7 @@ binding methods during attribute access. This means that all functions are non-data descriptors which return bound methods when they are invoked from an object. In pure Python, it works like this:: - class Function(object): + class Function: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" @@ -287,7 +287,7 @@ object. In pure Python, it works like this:: Running the interpreter shows how the function descriptor works in practice:: - >>> class D(object): + >>> class D: ... def f(self, x): ... return x ... @@ -367,7 +367,7 @@ It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` o Since staticmethods return the underlying function with no changes, the example calls are unexciting:: - >>> class E(object): + >>> class E: ... def f(x): ... print(x) ... f = staticmethod(f) @@ -380,7 +380,7 @@ calls are unexciting:: Using the non-data descriptor protocol, a pure Python version of :func:`staticmethod` would look like this:: - class StaticMethod(object): + class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" def __init__(self, f): @@ -393,7 +393,7 @@ Unlike static methods, class methods prepend the class reference to the argument list before calling the function. This format is the same for whether the caller is an object or a class:: - >>> class E(object): + >>> class E: ... def f(klass, x): ... return klass.__name__, x ... f = classmethod(f) @@ -410,7 +410,7 @@ is to create alternate class constructors. In Python 2.3, the classmethod :func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure Python equivalent is:: - class Dict(object): + class Dict: . . . def fromkeys(klass, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" @@ -428,7 +428,7 @@ Now a new dictionary of unique keys can be constructed like this:: Using the non-data descriptor protocol, a pure Python version of :func:`classmethod` would look like this:: - class ClassMethod(object): + class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" def __init__(self, f): diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 50cde3595034b5..909deb5fed33ff 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -332,6 +332,15 @@ Available static markers .. versionadded:: 3.7 +.. c:function:: audit(str event, void *tuple) + + Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. + ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` + pointer to a tuple object. + + .. versionadded:: 3.8 + + SystemTap Tapsets ----------------- diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 4956aa0dd957bc..87ac79ef8072d6 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -579,7 +579,7 @@ information. When you call one of the logging methods on an instance of information in the delegated call. Here's a snippet from the code of :class:`LoggerAdapter`:: - def debug(self, msg, *args, **kwargs): + def debug(self, msg, /, *args, **kwargs): """ Delegate a debug call to the underlying logger, after adding contextual information from this adapter instance. @@ -1079,7 +1079,7 @@ call ``str()`` on that object to get the actual format string. Consider the following two classes:: class BraceMessage: - def __init__(self, fmt, *args, **kwargs): + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1088,7 +1088,7 @@ following two classes:: return self.fmt.format(*self.args, **self.kwargs) class DollarMessage: - def __init__(self, fmt, **kwargs): + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -1143,7 +1143,7 @@ to the above, as in the following example:: import logging - class Message(object): + class Message: def __init__(self, fmt, args): self.fmt = fmt self.args = args @@ -1155,7 +1155,7 @@ to the above, as in the following example:: def __init__(self, logger, extra=None): super(StyleAdapter, self).__init__(logger, extra or {}) - def log(self, level, msg, *args, **kwargs): + def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) self.logger._log(level, Message(msg, args), (), **kwargs) @@ -1301,7 +1301,7 @@ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, *handlers, **kwargs): + def __init__(self, uri, /, *handlers, **kwargs): self.ctx = kwargs.get('ctx') or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything @@ -1706,8 +1706,8 @@ which uses JSON to serialise the event in a machine-parseable manner:: import json import logging - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1750,8 +1750,8 @@ as in the following complete example:: return o.encode('unicode_escape').decode('ascii') return super(Encoder, self).default(o) - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1982,8 +1982,8 @@ object as a message format string, and that the logging package will call :func:`str` on that object to get the actual format string. Consider the following two classes:: - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): + class BraceMessage: + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1991,8 +1991,8 @@ following two classes:: def __str__(self): return self.fmt.format(*self.args, **self.kwargs) - class DollarMessage(object): - def __init__(self, fmt, **kwargs): + class DollarMessage: + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -2266,9 +2266,9 @@ The script just arranges to decorate ``foo`` with a decorator which will do the conditional logging that's required. The decorator takes a logger as a parameter and attaches a memory handler for the duration of the call to the decorated function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. +a level at which flushing should occur, and a capacity for the buffer (number of +records buffered). These default to a :class:`~logging.StreamHandler` which +writes to ``sys.stderr``, ``logging.ERROR`` and ``100`` respectively. Here's the script:: @@ -2457,7 +2457,7 @@ scope of the context manager:: import logging import sys - class LoggingContext(object): + class LoggingContext: def __init__(self, logger, level=None, handler=None, close=True): self.logger = logger self.level = level diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 7a68ca89199c77..fbe5a118d18684 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -128,10 +128,18 @@ look at that next. Be sure to try the following in a newly-started Python interpreter, and don't just continue from the session described above:: import logging - logging.basicConfig(filename='example.log',level=logging.DEBUG) + logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) logging.debug('This message should go to the log file') logging.info('So should this') logging.warning('And this, too') + logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + +.. versionchanged:: 3.9 + The *encoding* argument was added. In earlier Python versions, or if not + specified, the encoding used is the default value used by :func:`open`. While + not shown in the above example, an *errors* argument can also now be passed, + which determines how encoding errors are handled. For available values and + the default, see the documentation for :func:`open`. And now if we open the file and look at what we have, we should find the log messages: @@ -141,6 +149,7 @@ messages: DEBUG:root:This message should go to the log file INFO:root:So should this WARNING:root:And this, too + ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 3be6bb380d663b..f7d12a1565c17e 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -31,20 +31,26 @@ are: #. Only worry about supporting Python 2.7 #. Make sure you have good test coverage (coverage.py_ can help; - ``pip install coverage``) + ``python -m pip install coverage``) #. Learn the differences between Python 2 & 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``pip install future``) +#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) #. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``pip install pylint``) + (``python -m pip install pylint``) #. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``pip install caniusepython3``) + use of Python 3 (``python -m pip install caniusepython3``) #. Once your dependencies are no longer blocking you, use continuous integration to make sure you stay compatible with Python 2 & 3 (tox_ can help test - against multiple versions of Python; ``pip install tox``) + against multiple versions of Python; ``python -m pip install tox``) #. Consider using optional static type checking to make sure your type usage works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both - Python 2 & Python 3). + Python 2 & Python 3; ``python -m pip install mypy``). +.. note:: + + Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke + is the one installed for the Python currently in use, whether it be + a system-wide ``pip`` or one installed within a + :ref:`virtual environment `. Details ======= @@ -71,7 +77,7 @@ Drop support for Python 2.6 and older While you can make Python 2.5 work with Python 3, it is **much** easier if you only have to work with Python 2.7. If dropping Python 2.5 is not an option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``pip install six``). Do realize, though, that nearly all the projects listed +(``python -m pip install six``). Do realize, though, that nearly all the projects listed in this HOWTO will not be available to you. If you are able to skip Python 2.5 and older, then the required changes diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 5339bf45bf0e80..24c3235e4add94 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -135,17 +135,22 @@ used than UTF-8.) UTF-8 uses the following rules: UTF-8 has several convenient properties: 1. It can handle any Unicode code point. -2. A Unicode string is turned into a sequence of bytes containing no embedded zero - bytes. This avoids byte-ordering issues, and means UTF-8 strings can be - processed by C functions such as ``strcpy()`` and sent through protocols that - can't handle zero bytes. +2. A Unicode string is turned into a sequence of bytes that contains embedded + zero bytes only where they represent the null character (U+0000). This means + that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent + through protocols that can't handle zero bytes for anything other than + end-of-string markers. 3. A string of ASCII text is also valid UTF-8 text. 4. UTF-8 is fairly compact; the majority of commonly used characters can be represented with one or two bytes. 5. If bytes are corrupted or lost, it's possible to determine the start of the next UTF-8-encoded code point and resynchronize. It's also unlikely that random 8-bit data will look like valid UTF-8. - +6. UTF-8 is a byte oriented encoding. The encoding specifies that each + character is represented by a specific sequence of one or more bytes. This + avoids the byte-ordering issues that can occur with integer and word oriented + encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending + on the hardware on which the string was encoded. References @@ -157,7 +162,7 @@ difficult reading. `A chronology `_ of the origin and development of Unicode is also available on the site. On the Computerphile Youtube channel, Tom Scott briefly -`discusses the history of Unicode and UTF-8 ` +`discusses the history of Unicode and UTF-8 `_ (9 minutes 36 seconds). To help understand the standard, Jukka Korpela has written `an introductory diff --git a/Doc/includes/sqlite3/adapter_datetime.py b/Doc/includes/sqlite3/adapter_datetime.py index be33395100c325..d5221d80c35c8a 100644 --- a/Doc/includes/sqlite3/adapter_datetime.py +++ b/Doc/includes/sqlite3/adapter_datetime.py @@ -13,3 +13,5 @@ def adapt_datetime(ts): now = datetime.datetime.now() cur.execute("select ?", (now,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_1.py b/Doc/includes/sqlite3/adapter_point_1.py index 6b1af8415648a4..77daf8f16d227b 100644 --- a/Doc/includes/sqlite3/adapter_point_1.py +++ b/Doc/includes/sqlite3/adapter_point_1.py @@ -14,3 +14,5 @@ def __conform__(self, protocol): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_2.py b/Doc/includes/sqlite3/adapter_point_2.py index d670700f0491b1..cb86331692b61d 100644 --- a/Doc/includes/sqlite3/adapter_point_2.py +++ b/Doc/includes/sqlite3/adapter_point_2.py @@ -15,3 +15,5 @@ def adapt_point(point): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/connect_db_1.py b/Doc/includes/sqlite3/connect_db_1.py deleted file mode 100644 index 1b975232865aef..00000000000000 --- a/Doc/includes/sqlite3/connect_db_1.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") diff --git a/Doc/includes/sqlite3/connect_db_2.py b/Doc/includes/sqlite3/connect_db_2.py deleted file mode 100644 index f9728b36135ed7..00000000000000 --- a/Doc/includes/sqlite3/connect_db_2.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py index ef3e70a2a9cfc5..112f47703a2ff4 100644 --- a/Doc/includes/sqlite3/countcursors.py +++ b/Doc/includes/sqlite3/countcursors.py @@ -13,3 +13,5 @@ def cursor(self, *args, **kwargs): cur1 = con.cursor() cur2 = con.cursor() print(con.numcursors) + +con.close() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 7af4ad1ecfbcca..6db77d45046e1f 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -14,3 +14,7 @@ con.execute("insert into person(firstname) values (?)", ("Joe",)) except sqlite3.IntegrityError: print("couldn't add Joe twice") + +# Connection object used as context manager only commits or rollbacks transactions, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py index 078873bfc979a9..115bcb50c7c754 100644 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ b/Doc/includes/sqlite3/execsql_fetchonerow.py @@ -15,3 +15,5 @@ cur.execute(SELECT) for row in cur: print('%s is %d years old.' % (row[0], row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py index a4ce5c528149c5..19306e6e3ca7d1 100644 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ b/Doc/includes/sqlite3/execsql_printall_1.py @@ -11,3 +11,5 @@ # Retrieve all rows as a sequence and print that sequence: print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index f864a8984e4e90..3466b1265a5bf2 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -14,3 +14,5 @@ cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) print(cur.fetchone()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_3.py b/Doc/includes/sqlite3/execute_3.py deleted file mode 100644 index 0353683fc70476..00000000000000 --- a/Doc/includes/sqlite3/execute_3.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -who = "Yeltsin" -age = 72 - -cur.execute("select name_last, age from people where name_last=:who and age=:age", - locals()) -print(cur.fetchone()) diff --git a/Doc/includes/sqlite3/executemany_1.py b/Doc/includes/sqlite3/executemany_1.py index efae10637c7e8f..edf6f8b7ebe61a 100644 --- a/Doc/includes/sqlite3/executemany_1.py +++ b/Doc/includes/sqlite3/executemany_1.py @@ -22,3 +22,5 @@ def __next__(self): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executemany_2.py b/Doc/includes/sqlite3/executemany_2.py index 527358ebc28e1f..02a594c861e15b 100644 --- a/Doc/includes/sqlite3/executemany_2.py +++ b/Doc/includes/sqlite3/executemany_2.py @@ -13,3 +13,5 @@ def char_generator(): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executescript.py b/Doc/includes/sqlite3/executescript.py index 7e5358178d4c0a..aea8943fbee598 100644 --- a/Doc/includes/sqlite3/executescript.py +++ b/Doc/includes/sqlite3/executescript.py @@ -22,3 +22,4 @@ 1987 ); """) +con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py index edbc79e7e5b6cc..10cf937243f6da 100644 --- a/Doc/includes/sqlite3/insert_more_people.py +++ b/Doc/includes/sqlite3/insert_more_people.py @@ -14,3 +14,5 @@ # The changes will not be saved unless the transaction is committed explicitly: con.commit() + +con.close() diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py index b997c70668ace4..624cfe262f38b3 100644 --- a/Doc/includes/sqlite3/load_extension.py +++ b/Doc/includes/sqlite3/load_extension.py @@ -24,3 +24,5 @@ """) for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"): print(row) + +con.close() diff --git a/Doc/includes/sqlite3/md5func.py b/Doc/includes/sqlite3/md5func.py index 0056b2d6ce84ef..16dc348bf001e2 100644 --- a/Doc/includes/sqlite3/md5func.py +++ b/Doc/includes/sqlite3/md5func.py @@ -9,3 +9,5 @@ def md5sum(t): cur = con.cursor() cur.execute("select md5(?)", (b"foo",)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/mysumaggr.py b/Doc/includes/sqlite3/mysumaggr.py index d2dfd2c0b98d3b..11f96395b6c485 100644 --- a/Doc/includes/sqlite3/mysumaggr.py +++ b/Doc/includes/sqlite3/mysumaggr.py @@ -18,3 +18,5 @@ def finalize(self): cur.execute("insert into test(i) values (2)") cur.execute("select mysum(i) from test") print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py index cc68c76459ecea..5f01dbfe1cb524 100644 --- a/Doc/includes/sqlite3/parse_colnames.py +++ b/Doc/includes/sqlite3/parse_colnames.py @@ -6,3 +6,5 @@ cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) dt = cur.fetchone()[0] print(dt, type(dt)) + +con.close() diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py index 68d49358a578b3..5d843f906b3062 100644 --- a/Doc/includes/sqlite3/pysqlite_datetime.py +++ b/Doc/includes/sqlite3/pysqlite_datetime.py @@ -18,3 +18,5 @@ row = cur.fetchone() print("current_date", row[0], type(row[0])) print("current_timestamp", row[1], type(row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/row_factory.py b/Doc/includes/sqlite3/row_factory.py index e436ffc6c80225..9de6e7b1b9052a 100644 --- a/Doc/includes/sqlite3/row_factory.py +++ b/Doc/includes/sqlite3/row_factory.py @@ -11,3 +11,5 @@ def dict_factory(cursor, row): cur = con.cursor() cur.execute("select 1 as a") print(cur.fetchone()["a"]) + +con.close() diff --git a/Doc/includes/sqlite3/rowclass.py b/Doc/includes/sqlite3/rowclass.py index 92b5ad60cb5791..fc60287069a854 100644 --- a/Doc/includes/sqlite3/rowclass.py +++ b/Doc/includes/sqlite3/rowclass.py @@ -10,3 +10,5 @@ assert row["name"] == row["nAmE"] assert row[1] == row["age"] assert row[1] == row["AgE"] + +con.close() diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 71600d4f60c55e..98a39411495cba 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -18,3 +18,7 @@ print(row) print("I just deleted", con.execute("delete from person").rowcount, "rows") + +# close is not a shortcut method and it's not called automatically, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py index 231d8726cd436e..148a1707f948bc 100644 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ b/Doc/includes/sqlite3/simple_tableprinter.py @@ -24,3 +24,5 @@ print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') print() # Finish the row with a newline. + +con.close() diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index 5f96cdb58da1ab..a857a155cdd4ff 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -25,3 +25,5 @@ cur.execute("select ?", ("bar",)) row = cur.fetchone() assert row[0] == "barfoo" + +con.close() diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 9f47899a198e01..9ada03cfc4a4cb 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -6,7 +6,7 @@ typedef struct _typeobject { /* Methods to implement standard operations */ destructor tp_dealloc; - printfunc tp_print; + Py_ssize_t tp_vectorcall_offset; getattrfunc tp_getattr; setattrfunc tp_setattr; PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) diff --git a/Doc/install/index.rst b/Doc/install/index.rst index f6a8cd6833a9ad..a91606c0f38e61 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _install-index: @@ -13,23 +13,10 @@ .. seealso:: :ref:`installing-index` - The up to date module installation documentations - -.. The audience for this document includes people who don't know anything - about Python and aren't about to learn the language just in order to - install and maintain it for their users, i.e. system administrators. - Thus, I have to be sure to explain the basics at some point: - sys.path and PYTHONPATH at least. Should probably give pointers to - other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. - - Finally, it might be useful to include all the material from my "Care - and Feeding of a Python Installation" talk in here somewhere. Yow! - -This document describes the Python Distribution Utilities ("Distutils") from the -end-user's point-of-view, describing how to extend the capabilities of a -standard Python installation by building and installing third-party Python -modules and extensions. + The up to date module installation documentation. For regular Python + usage, you almost certainly want that document rather than this one. +.. include:: ../distutils/_setuptools_disclaimer.rst .. note:: @@ -46,59 +33,26 @@ modules and extensions. Introduction ============ -Although Python's extensive standard library covers many programming needs, -there often comes a time when you need to add some new functionality to your -Python installation in the form of third-party modules. This might be necessary -to support your own programming, or to support an application that you want to -use and that happens to be written in Python. - -In the past, there has been little support for adding third-party modules to an -existing Python installation. With the introduction of the Python Distribution -Utilities (Distutils for short) in Python 2.0, this changed. - -This document is aimed primarily at the people who need to install third-party -Python modules: end-users and system administrators who just need to get some -Python application running, and existing Python programmers who want to add some -new goodies to their toolbox. You don't need to know Python to read this -document; there will be some brief forays into using Python's interactive mode -to explore your installation, but that's it. If you're looking for information -on how to distribute your own Python modules so that others may use them, see -the :ref:`distutils-index` manual. :ref:`debug-setup-script` may also be of -interest. - - -.. _inst-trivial-install: - -Best case: trivial installation -------------------------------- - -In the best case, someone will have prepared a special version of the module -distribution you want to install that is targeted specifically at your platform -and is installed just like any other software on your platform. For example, -the module developer might make an executable installer available for Windows -users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, -Mandrake, and many others), a Debian package for users of Debian-based Linux -systems, and so forth. - -In that case, you would download the installer appropriate to your platform and -do the obvious thing with it: run it if it's an executable installer, ``rpm ---install`` it if it's an RPM, etc. You don't need to run Python or a setup -script, you don't need to compile anything---you might not even need to read any -instructions (although it's always a good idea to do so anyway). - -Of course, things will not always be that easy. You might be interested in a -module distribution that doesn't have an easy-to-use installer for your -platform. In that case, you'll have to start with the source distribution -released by the module's author/maintainer. Installing from a source -distribution is not too hard, as long as the modules are packaged in the -standard way. The bulk of this document is about building and installing -modules from standard source distributions. +In Python 2.0, the ``distutils`` API was first added to the standard library. +This provided Linux distro maintainers with a standard way of converting +Python projects into Linux distro packages, and system administrators with a +standard way of installing them directly onto target systems. + +In the many years since Python 2.0 was released, tightly coupling the build +system and package installer to the language runtime release cycle has turned +out to be problematic, and it is now recommended that projects use the +``pip`` package installer and the ``setuptools`` build system, rather than +using ``distutils`` directly. + +See :ref:`installing-index` and :ref:`distributing-index` for more details. +This legacy documentation is being retained only until we're confident that the +``setuptools`` documentation covers everything needed. .. _inst-new-standard: -The new standard: Distutils ---------------------------- +Distutils based source distributions +------------------------------------ If you download a module source distribution, you can tell pretty quickly if it was packaged and distributed in the standard way, i.e. using the Distutils. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 747b9223b62cb8..dc44aa64b8e045 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _installing-index: diff --git a/Doc/library/_dummy_thread.rst b/Doc/library/_dummy_thread.rst deleted file mode 100644 index 7dccbc55475aae..00000000000000 --- a/Doc/library/_dummy_thread.rst +++ /dev/null @@ -1,22 +0,0 @@ -:mod:`_dummy_thread` --- Drop-in replacement for the :mod:`_thread` module -========================================================================== - -.. module:: _dummy_thread - :synopsis: Drop-in replacement for the _thread module. - -**Source code:** :source:`Lib/_dummy_thread.py` - -.. deprecated:: 3.7 - Python now always has threading enabled. Please use :mod:`_thread` - (or, better, :mod:`threading`) instead. - --------------- - -This module provides a duplicate interface to the :mod:`_thread` module. -It was meant to be imported when the :mod:`_thread` module was not provided -on a platform. - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. - diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index acffabf24bad5f..bd653ab32bb9c4 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -43,18 +43,32 @@ This module defines the following constants and functions: .. function:: start_new_thread(function, args[, kwargs]) - Start a new thread and return its identifier. The thread executes the function - *function* with the argument list *args* (which must be a tuple). The optional - *kwargs* argument specifies a dictionary of keyword arguments. When the function - returns, the thread silently exits. When the function terminates with an - unhandled exception, a stack trace is printed and then the thread exits (but - other threads continue to run). + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. .. function:: interrupt_main() - Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can - use this function to interrupt the main thread. + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. function:: exit() @@ -85,6 +99,18 @@ This module defines the following constants and functions: may be recycled when a thread exits and another thread is created. +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + .. function:: stack_size([size]) Return the thread stack size used when creating new threads. The optional diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index cef197f3055581..ef2fd42783c877 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -182,6 +182,10 @@ ArgumentParser objects .. versionchanged:: 3.5 *allow_abbrev* parameter was added. + .. versionchanged:: 3.8 + In previous versions, *allow_abbrev* also disabled grouping of short + flags such as ``-vv`` to mean ``-v -v``. + The following sections describe how each of these are used. @@ -797,6 +801,15 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 +* ``'extend'`` - This stores a list, and extends each argument value to the + list. + Example usage:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) + >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) + Namespace(foo=['f1', 'f2', 'f3', 'f4']) + You may also specify an arbitrary action by passing an Action subclass or other object that implements the same interface. The recommended way to do this is to extend :class:`Action`, overriding the ``__call__`` method diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 4ac7bb5391a7a4..59b94f10b43eca 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -83,6 +83,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. + .. audit-event:: array.__new__ typecode,initializer array.array .. data:: typecodes diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 1884bea80e8047..1e718382589c48 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -126,7 +126,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,11 +145,12 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to the minor version of an - earlier Python 3 version will attempt to parse using that version's - grammar. For example, setting ``feature_version=4`` will allow - the use of ``async`` and ``await`` as variable names. The lowest - supported value is 4; the highest is ``sys.version_info[1]``. + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. .. warning:: It is possible to crash the Python interpreter with a diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bf7c93a86fd041..8f7974be66eaa6 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -397,9 +397,27 @@ Opening network connections If given, these should all be integers from the corresponding :mod:`socket` module constants. + * *happy_eyeballs_delay*, if given, enables Happy Eyeballs for this + connection. It should + be a floating-point number representing the amount of time in seconds + to wait for a connection attempt to complete, before starting the next + attempt in parallel. This is the "Connection Attempt Delay" as defined + in :rfc:`8305`. A sensible default value recommended by the RFC is ``0.25`` + (250 milliseconds). + + * *interleave* controls address reordering when a host name resolves to + multiple IP addresses. + If ``0`` or unspecified, no reordering is done, and addresses are + tried in the order returned by :meth:`getaddrinfo`. If a positive integer + is specified, the addresses are interleaved by address family, and the + given integer is interpreted as "First Address Family Count" as defined + in :rfc:`8305`. The default is ``0`` if *happy_eyeballs_delay* is not + specified, and ``1`` if it is. + * *sock*, if given, should be an existing, already connected :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* + If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags*, + *happy_eyeballs_delay*, *interleave* and *local_addr* should be specified. * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used @@ -410,6 +428,10 @@ Opening network connections to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). + .. versionadded:: 3.8 + + The *happy_eyeballs_delay* and *interleave* parameters. + .. versionadded:: 3.7 The *ssl_handshake_timeout* parameter. @@ -482,8 +504,6 @@ Opening network connections transport. If specified, *local_addr* and *remote_addr* should be omitted (must be :const:`None`). - On Windows, with :class:`ProactorEventLoop`, this method is not supported. - See :ref:`UDP echo client protocol ` and :ref:`UDP echo server protocol ` examples. @@ -491,6 +511,9 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8 + Added support for Windows. + .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ path=None, \*, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -1195,32 +1218,52 @@ async/await code consider using the high-level Other parameters: - * *stdin*: either a file-like object representing a pipe to be - connected to the subprocess's standard input stream using - :meth:`~loop.connect_write_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new - pipe will be created and connected. - - * *stdout*: either a file-like object representing the pipe to be - connected to the subprocess's standard output stream using - :meth:`~loop.connect_read_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new pipe - will be created and connected. - - * *stderr*: either a file-like object representing the pipe to be - connected to the subprocess's standard error stream using - :meth:`~loop.connect_read_pipe`, or one of - :const:`subprocess.PIPE` (default) or :const:`subprocess.STDOUT` - constants. - - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess' standard - error stream will be connected to the same pipe as the standard - output stream. + * *stdin* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard input stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stdout* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard output stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stderr* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard error stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + * the :const:`subprocess.STDOUT` constant which will connect the standard + error stream to the process' standard output stream * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* - and *shell*, which should not be specified at all. + without interpretation, except for *bufsize*, *universal_newlines*, + *shell*, *text*, *encoding* and *errors*, which should not be specified + at all. + + The ``asyncio`` subprocess API does not support decoding the streams + as text. :func:`bytes.decode` can be used to convert the bytes returned + from the stream to text. See the constructor of the :class:`subprocess.Popen` class for documentation on other arguments. @@ -1411,7 +1454,7 @@ asyncio ships with two different event loop implementations: :class:`SelectorEventLoop` and :class:`ProactorEventLoop`. By default asyncio is configured to use :class:`SelectorEventLoop` -on all platforms. +on Unix and :class:`ProactorEventLoop` on Windows. .. class:: SelectorEventLoop @@ -1582,8 +1625,7 @@ Wait until a file descriptor received some data using the :meth:`loop.create_connection` method. * Another similar :ref:`example ` - using the high-level :func:`asyncio.open_connection` function - and streams. + using the high-level :func:`asyncio.connect` function and streams. .. _asyncio_example_unix_signals: @@ -1601,7 +1643,7 @@ using the :meth:`loop.add_signal_handler` method:: import os import signal - def ask_exit(signame): + def ask_exit(signame, loop): print("got signal %s: exit" % signame) loop.stop() @@ -1611,7 +1653,7 @@ using the :meth:`loop.add_signal_handler` method:: for signame in {'SIGINT', 'SIGTERM'}: loop.add_signal_handler( getattr(signal, signame), - functools.partial(ask_exit, signame)) + functools.partial(ask_exit, signame, loop)) await asyncio.sleep(3600) diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 81d840e23277ea..7e4a70f91c6ed5 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -53,9 +53,6 @@ All event loops on Windows do not support the following methods: :class:`ProactorEventLoop` has the following limitations: -* The :meth:`loop.create_datagram_endpoint` method - is not supported. - * The :meth:`loop.add_reader` and :meth:`loop.add_writer` methods are not supported. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 6212df85dbc10a..aa8f8f13eae021 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -117,6 +117,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. +.. _asyncio-watchers: Process Watchers ================ @@ -129,10 +130,11 @@ In asyncio, child processes are created with :func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` functions. -asyncio defines the :class:`AbstractChildWatcher` abstract base class, -which child watchers should implement, and has two different -implementations: :class:`SafeChildWatcher` (configured to be used -by default) and :class:`FastChildWatcher`. +asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child +watchers should implement, and has four different implementations: +:class:`ThreadedChildWatcher` (configured to be used by default), +:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and +:class:`FastChildWatcher`. See also the :ref:`Subprocess and Threads ` section. @@ -184,6 +186,15 @@ implementation used by the asyncio event loop: Note: loop may be ``None``. + .. method:: is_active() + + Return ``True`` if the watcher is ready to use. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + + .. versionadded:: 3.8 + .. method:: close() Close the watcher. @@ -191,16 +202,48 @@ implementation used by the asyncio event loop: This method has to be called to ensure that underlying resources are cleaned-up. -.. class:: SafeChildWatcher +.. class:: ThreadedChildWatcher + + This implementation starts a new waiting thread for every subprocess spawn. + + It works reliably even when the asyncio event loop is run in a non-main OS thread. + + There is no noticeable overhead when handling a big number of children (*O(1)* each + time a child terminates), but stating a thread per process requires extra memory. + + This watcher is used by default. + + .. versionadded:: 3.8 - This implementation avoids disrupting other code spawning processes +.. class:: MultiLoopChildWatcher + + This implementation registers a :py:data:`SIGCHLD` signal handler on + instantiation. That can break third-party code that installs a custom handler for + `SIGCHLD`. signal). + + The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. - This is a safe solution but it has a significant overhead when + There is no limitation for running subprocesses from different threads once the + watcher is installed. + + The solution is safe but it has a significant overhead when handling a big number of processes (*O(n)* each time a :py:data:`SIGCHLD` is received). - asyncio uses this safe implementation by default. + .. versionadded:: 3.8 + +.. class:: SafeChildWatcher + + This implementation uses active event loop from the main thread to handle + :py:data:`SIGCHLD` signal. If the main thread has no running event loop another + thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). + + The watcher avoids disrupting other code spawning processes + by polling every process explicitly on a :py:data:`SIGCHLD` signal. + + This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* + complexity but requires a running event loop in the main thread to work. .. class:: FastChildWatcher @@ -211,6 +254,9 @@ implementation used by the asyncio event loop: There is no noticeable overhead when handling a big number of children (*O(1)* each time a child terminates). + This solution requires a running event loop in the main thread to work, as + :class:`SafeChildWatcher`. + Custom Policies =============== diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index f08738dd62bb5c..3e5a4dd8b891f5 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -810,7 +810,7 @@ data, and waits until the connection is closed:: .. seealso:: The :ref:`TCP echo client using streams ` - example uses the high-level :func:`asyncio.open_connection` function. + example uses the high-level :func:`asyncio.connect` function. .. _asyncio-udp-echo-server-protocol: @@ -977,7 +977,7 @@ Wait until a socket receives data using the The :ref:`register an open socket to wait for data using streams ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. + created by the :func:`asyncio.connect` function in a coroutine. .. _asyncio_example_subprocess_proto: diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index e686a6a1c4cd32..dfe520de56bf04 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -18,17 +18,12 @@ streams:: import asyncio async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) + async with asyncio.connect('127.0.0.1', 8888) as stream: + print(f'Send: {message!r}') + await stream.write(message.encode()) - print(f'Send: {message!r}') - await writer.awrite(message.encode()) - - data = await reader.read(100) - print(f'Received: {data.decode()!r}') - - print('Close the connection') - await writer.aclose() + data = await stream.read(100) + print(f'Received: {data.decode()!r}') asyncio.run(tcp_echo_client('Hello World!')) @@ -42,6 +37,32 @@ The following top-level asyncio functions can be used to create and work with streams: +.. coroutinefunction:: connect(host=None, port=None, \*, \ + limit=2**16, ssl=None, family=0, \ + proto=0, flags=0, sock=None, local_addr=None, \ + server_hostname=None, ssl_handshake_timeout=None, \ + happy_eyeballs_delay=None, interleave=None) + + Connect to TCP socket on *host* : *port* address and return a :class:`Stream` + object of mode :attr:`StreamMode.READWRITE`. + + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the *limit* is set to 64 KiB. + + The rest of the arguments are passed directly to :meth:`loop.create_connection`. + + The function can be used with ``await`` to get a connected stream:: + + stream = await asyncio.connect('127.0.0.1', 8888) + + The function can also be used as an async context manager:: + + async with asyncio.connect('127.0.0.1', 8888) as stream: + ... + + .. versionadded:: 3.8 + .. coroutinefunction:: open_connection(host=None, port=None, \*, \ loop=None, limit=None, ssl=None, family=0, \ proto=0, flags=0, sock=None, local_addr=None, \ @@ -67,8 +88,12 @@ and work with streams: The *ssl_handshake_timeout* parameter. + .. deprecated-removed:: 3.8 3.10 + + `open_connection()` is deprecated in favor of :func:`connect`. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, \*, loop=None, limit=None, \ + port=None, \*, loop=None, limit=2**16, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -100,9 +125,60 @@ and work with streams: The *ssl_handshake_timeout* and *start_serving* parameters. + .. deprecated-removed:: 3.8 3.10 + + `start_server()` is deprecated if favor of :class:`StreamServer` + +.. coroutinefunction:: connect_read_pipe(pipe, *, limit=2**16) + + Takes a :term:`file-like object ` *pipe* to return a + :class:`Stream` object of the mode :attr:`StreamMode.READ` that has + similar API of :class:`StreamReader`. It can also be used as an async context manager. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the limit is set to 64 KiB. + + .. versionadded:: 3.8 + +.. coroutinefunction:: connect_write_pipe(pipe, *, limit=2**16) + + Takes a :term:`file-like object ` *pipe* to return a + :class:`Stream` object of the mode :attr:`StreamMode.WRITE` that has + similar API of :class:`StreamWriter`. It can also be used as an async context manager. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the limit is set to 64 KiB. + + .. versionadded:: 3.8 .. rubric:: Unix Sockets +.. function:: connect_unix(path=None, *, limit=2**16, ssl=None, \ + sock=None, server_hostname=None, \ + ssl_handshake_timeout=None) + + Establish a Unix socket connection to socket with *path* address and + return an awaitable :class:`Stream` object of the mode :attr:`StreamMode.READWRITE` + that can be used as a reader and a writer. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the *limit* is set to 64 KiB. + + The rest of the arguments are passed directly to :meth:`loop.create_unix_connection`. + + The function can be used with ``await`` to get a connected stream:: + + stream = await asyncio.connect_unix('/tmp/example.sock') + + The function can also be used as an async context manager:: + + async with asyncio.connect_unix('/tmp/example.sock') as stream: + ... + + .. availability:: Unix. + + .. versionadded:: 3.8 + .. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ limit=None, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -124,6 +200,10 @@ and work with streams: The *path* parameter can now be a :term:`path-like object` + .. deprecated-removed:: 3.8 3.10 + + ``open_unix_connection()`` is deprecated if favor of :func:`connect_unix`. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ \*, loop=None, limit=None, sock=None, \ @@ -146,9 +226,178 @@ and work with streams: The *path* parameter can now be a :term:`path-like object`. + .. deprecated-removed:: 3.8 3.10 + + ``start_unix_server()`` is deprecated in favor of :class:`UnixStreamServer`. + --------- +StreamServer +============ + +.. class:: StreamServer(client_connected_cb, /, host=None, port=None, *, \ + limit=2**16, family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, sock=None, backlog=100, \ + ssl=None, reuse_address=None, reuse_port=None, \ + ssl_handshake_timeout=None, shutdown_timeout=60) + + The *client_connected_cb* callback is called whenever a new client + connection is established. It receives a :class:`Stream` object of the + mode :attr:`StreamMode.READWRITE`. + + *client_connected_cb* can be a plain callable or a + :ref:`coroutine function `; if it is a coroutine function, + it will be automatically scheduled as a :class:`Task`. + + *limit* determines the buffer size limit used by the + returned :class:`Stream` instance. By default the *limit* + is set to 64 KiB. + + The rest of the arguments are passed directly to + :meth:`loop.create_server`. + + .. coroutinemethod:: start_serving() + + Binds to the given host and port to start the server. + + .. coroutinemethod:: serve_forever() + + Start accepting connections until the coroutine is cancelled. + Cancellation of ``serve_forever`` task causes the server + to be closed. + + This method can be called if the server is already accepting + connections. Only one ``serve_forever`` task can exist per + one *Server* object. + + .. method:: is_serving() + + Returns ``True`` if the server is bound and currently serving. + + .. method:: bind() + + Bind the server to the given *host* and *port*. This method is + automatically called during ``__aenter__`` when :class:`StreamServer` is + used as an async context manager. + + .. method:: is_bound() + + Return ``True`` if the server is bound. + + .. coroutinemethod:: abort() + + Closes the connection and cancels all pending tasks. + + .. coroutinemethod:: close() + + Closes the connection. This method is automatically called during + ``__aexit__`` when :class:`StreamServer` is used as an async context + manager. + + .. attribute:: sockets + + Returns a tuple of socket objects the server is bound to. + + .. versionadded:: 3.8 + + +UnixStreamServer +================ + +.. class:: UnixStreamServer(client_connected_cb, /, path=None, *, \ + limit=2**16, sock=None, backlog=100, \ + ssl=None, ssl_handshake_timeout=None, shutdown_timeout=60) + + The *client_connected_cb* callback is called whenever a new client + connection is established. It receives a :class:`Stream` object of the + mode :attr:`StreamMode.READWRITE`. + + *client_connected_cb* can be a plain callable or a + :ref:`coroutine function `; if it is a coroutine function, + it will be automatically scheduled as a :class:`Task`. + + *limit* determines the buffer size limit used by the + returned :class:`Stream` instance. By default the *limit* + is set to 64 KiB. + + The rest of the arguments are passed directly to + :meth:`loop.create_unix_server`. + + .. coroutinemethod:: start_serving() + + Binds to the given host and port to start the server. + + .. method:: is_serving() + + Returns ``True`` if the server is bound and currently serving. + + .. method:: bind() + + Bind the server to the given *host* and *port*. This method is + automatically called during ``__aenter__`` when :class:`UnixStreamServer` is + used as an async context manager. + + .. method:: is_bound() + + Return ``True`` if the server is bound. + + .. coroutinemethod:: abort() + + Closes the connection and cancels all pending tasks. + + .. coroutinemethod:: close() + + Closes the connection. This method is automatically called during + ``__aexit__`` when :class:`UnixStreamServer` is used as an async context + manager. + + .. attribute:: sockets + + Returns a tuple of socket objects the server is bound to. + + .. availability:: Unix. + + .. versionadded:: 3.8 + +Stream +====== + +.. class:: Stream + + Represents a Stream object that provides APIs to read and write data + to the IO stream . It includes the API provided by :class:`StreamReader` + and :class:`StreamWriter`. + + Do not instantiate *Stream* objects directly; use API like :func:`connect` + and :class:`StreamServer` instead. + + .. versionadded:: 3.8 + + +StreamMode +========== + +.. class:: StreamMode + + A subclass of :class:`enum.Flag` that defines a set of values that can be + used to determine the ``mode`` of :class:`Stream` objects. + + .. data:: READ + + The stream object is readable and provides the API of :class:`StreamReader`. + + .. data:: WRITE + + The stream object is writeable and provides the API of :class:`StreamWriter`. + + .. data:: READWRITE + + The stream object is readable and writeable and provides the API of both + :class:`StreamReader` and :class:`StreamWriter`. + + .. versionadded:: 3.8 + StreamReader ============ @@ -226,23 +475,70 @@ StreamWriter directly; use :func:`open_connection` and :func:`start_server` instead. - .. coroutinemethod:: awrite(data) + .. method:: write(data) + + The method attempts to write the *data* to the underlying socket immediately. + If that fails, the data is queued in an internal write buffer until it can be + sent. + + Starting with Python 3.8, it is possible to directly await on the `write()` + method:: + + await stream.write(data) + + The ``await`` pauses the current coroutine until the data is written to the + socket. + + Below is an equivalent code that works with Python <= 3.7:: + + stream.write(data) + await stream.drain() + + .. versionchanged:: 3.8 + Support ``await stream.write(...)`` syntax. + + .. method:: writelines(data) + + The method writes a list (or any iterable) of bytes to the underlying socket + immediately. + If that fails, the data is queued in an internal write buffer until it can be + sent. + + Starting with Python 3.8, it is possible to directly await on the `write()` + method:: + + await stream.writelines(lines) + + The ``await`` pauses the current coroutine until the data is written to the + socket. + + Below is an equivalent code that works with Python <= 3.7:: + + stream.writelines(lines) + await stream.drain() + + .. versionchanged:: 3.8 + Support ``await stream.writelines()`` syntax. - Write *data* to the stream. + .. method:: close() + + The method closes the stream and the underlying socket. - The method respects flow control, execution is paused if the write - buffer reaches the high watermark. + Starting with Python 3.8, it is possible to directly await on the `close()` + method:: - .. versionadded:: 3.8 + await stream.close() - .. coroutinemethod:: aclose() + The ``await`` pauses the current coroutine until the stream and the underlying + socket are closed (and SSL shutdown is performed for a secure connection). - Close the stream. + Below is an equivalent code that works with Python <= 3.7:: - Wait until all closing actions are complete, e.g. SSL shutdown for - secure sockets. + stream.close() + await stream.wait_closed() - .. versionadded:: 3.8 + .. versionchanged:: 3.8 + Support ``await stream.close()`` syntax. .. method:: can_write_eof() @@ -263,21 +559,6 @@ StreamWriter Access optional transport information; see :meth:`BaseTransport.get_extra_info` for details. - .. method:: write(data) - - Write *data* to the stream. - - This method is not subject to flow control. Calls to ``write()`` should - be followed by :meth:`drain`. The :meth:`awrite` method is a - recommended alternative the applies flow control automatically. - - .. method:: writelines(data) - - Write a list (or any iterable) of bytes to the stream. - - This method is not subject to flow control. Calls to ``writelines()`` - should be followed by :meth:`drain`. - .. coroutinemethod:: drain() Wait until it is appropriate to resume writing to the stream. @@ -293,10 +574,6 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. method:: close() - - Close the stream. - .. method:: is_closing() Return ``True`` if the stream is closed or in the process of @@ -322,22 +599,17 @@ Examples TCP echo client using streams ----------------------------- -TCP echo client using the :func:`asyncio.open_connection` function:: +TCP echo client using the :func:`asyncio.connect` function:: import asyncio async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) - - print(f'Send: {message!r}') - writer.write(message.encode()) - - data = await reader.read(100) - print(f'Received: {data.decode()!r}') + async with asyncio.connect('127.0.0.1', 8888) as stream: + print(f'Send: {message!r}') + await stream.write(message.encode()) - print('Close the connection') - writer.close() + data = await stream.read(100) + print(f'Received: {data.decode()!r}') asyncio.run(tcp_echo_client('Hello World!')) @@ -353,32 +625,28 @@ TCP echo client using the :func:`asyncio.open_connection` function:: TCP echo server using streams ----------------------------- -TCP echo server using the :func:`asyncio.start_server` function:: +TCP echo server using the :class:`asyncio.StreamServer` class:: import asyncio - async def handle_echo(reader, writer): - data = await reader.read(100) + async def handle_echo(stream): + data = await stream.read(100) message = data.decode() - addr = writer.get_extra_info('peername') + addr = stream.get_extra_info('peername') print(f"Received {message!r} from {addr!r}") print(f"Send: {message!r}") - writer.write(data) - await writer.drain() + await stream.write(data) print("Close the connection") - writer.close() + await stream.close() async def main(): - server = await asyncio.start_server( - handle_echo, '127.0.0.1', 8888) - - addr = server.sockets[0].getsockname() - print(f'Serving on {addr}') - - async with server: + async with asyncio.StreamServer( + handle_echo, '127.0.0.1', 8888) as server: + addr = server.sockets[0].getsockname() + print(f'Serving on {addr}') await server.serve_forever() asyncio.run(main()) @@ -402,11 +670,9 @@ Simple example querying HTTP headers of the URL passed on the command line:: async def print_http_headers(url): url = urllib.parse.urlsplit(url) if url.scheme == 'https': - reader, writer = await asyncio.open_connection( - url.hostname, 443, ssl=True) + stream = await asyncio.connect(url.hostname, 443, ssl=True) else: - reader, writer = await asyncio.open_connection( - url.hostname, 80) + stream = await asyncio.connect(url.hostname, 80) query = ( f"HEAD {url.path or '/'} HTTP/1.0\r\n" @@ -414,18 +680,14 @@ Simple example querying HTTP headers of the URL passed on the command line:: f"\r\n" ) - writer.write(query.encode('latin-1')) - while True: - line = await reader.readline() - if not line: - break - + stream.write(query.encode('latin-1')) + while (line := await stream.readline()): line = line.decode('latin1').rstrip() if line: print(f'HTTP header> {line}') # Ignore the body, close the socket - writer.close() + await stream.close() url = sys.argv[1] asyncio.run(print_http_headers(url)) @@ -446,7 +708,7 @@ Register an open socket to wait for data using streams ------------------------------------------------------ Coroutine waiting until a socket receives data using the -:func:`open_connection` function:: +:func:`asyncio.connect` function:: import asyncio import socket @@ -460,17 +722,15 @@ Coroutine waiting until a socket receives data using the rsock, wsock = socket.socketpair() # Register the open socket to wait for data. - reader, writer = await asyncio.open_connection(sock=rsock) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) + async with asyncio.connect(sock=rsock) as stream: + # Simulate the reception of data from the network + loop.call_soon(wsock.send, 'abc'.encode()) - # Wait for data - data = await reader.read(100) + # Wait for data + data = await stream.read(100) - # Got data, we are done: close the socket - print("Received:", data.decode()) - writer.close() + # Got data, we are done: close the socket + print("Received:", data.decode()) # Close the second socket wsock.close() diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 00dc66c48b2189..444fb6361b5eff 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -293,18 +293,26 @@ their completion. Subprocess and Threads ---------------------- -Standard asyncio event loop supports running subprocesses from -different threads, but there are limitations: +Standard asyncio event loop supports running subprocesses from different threads by +default. -* An event loop must run in the main thread. +On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), +:class:`SelectorEventLoop` has no subprocess support. -* The child watcher must be instantiated in the main thread - before executing subprocesses from other threads. Call the - :func:`get_child_watcher` function in the main thread to instantiate - the child watcher. +On UNIX *child watchers* are used for subprocess finish waiting, see +:ref:`asyncio-watchers` for more info. -Note that alternative event loop implementations might not share -the above limitations; please refer to their documentation. + +.. versionchanged:: 3.8 + + UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from + different threads without any limitation. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + +Note that alternative event loop implementations might have own limitations; +please refer to their documentation. .. seealso:: diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index e3f18ccb4341fe..79f6b02d85e2e5 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -66,6 +66,13 @@ Lock This method waits until the lock is *unlocked*, sets it to *locked* and returns ``True``. + When more than one coroutine is blocked in :meth:`acquire` + waiting for the lock to be unlocked, only one coroutine + eventually proceeds. + + Acquiring a lock is *fair*: the coroutine that proceeds will be + the first coroutine that started waiting on the lock. + .. method:: release() Release the lock. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index a1297f5fb7fc7f..1fcdcb985d8842 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -40,7 +40,7 @@ be executed:: >>> main() -To actually run a coroutine asyncio provides three main mechanisms: +To actually run a coroutine, asyncio provides three main mechanisms: * The :func:`asyncio.run` function to run the top-level entry point "main()" function (see the above example.) @@ -279,8 +279,8 @@ Sleeping ``sleep()`` always suspends the current task, allowing other tasks to run. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_sleep: @@ -437,8 +437,8 @@ Timeouts If the wait is cancelled, the future *aw* is also cancelled. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_waitfor: @@ -478,10 +478,12 @@ Waiting Primitives set concurrently and block until the condition specified by *return_when*. - If any awaitable in *aws* is a coroutine, it is automatically - scheduled as a Task. Passing coroutines objects to - ``wait()`` directly is deprecated as it leads to - :ref:`confusing behavior `. + .. deprecated:: 3.8 + + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. Passing coroutines objects to + ``wait()`` directly is deprecated as it leads to + :ref:`confusing behavior `. Returns two sets of Tasks/Futures: ``(done, pending)``. @@ -489,8 +491,8 @@ Waiting Primitives done, pending = await asyncio.wait(aws) - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. *timeout* (a float or int), if specified, can be used to control the maximum number of seconds to wait before returning. @@ -550,6 +552,8 @@ Waiting Primitives if task in done: # Everything will work as expected now. + .. deprecated:: 3.8 + Passing coroutine objects to ``wait()`` directly is deprecated. @@ -838,6 +842,12 @@ Task Object The *file* argument is an I/O stream to which the output is written; by default output is written to :data:`sys.stderr`. + .. method:: get_coro() + + Return the coroutine object wrapped by the :class:`Task`. + + .. versionadded:: 3.8 + .. method:: get_name() Return the name of the Task. @@ -868,8 +878,10 @@ Task Object If *loop* is ``None``, the :func:`get_event_loop` function is used to get the current loop. - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.all_tasks` function instead. + .. deprecated-removed:: 3.7 3.9 + + Do not call this as a task method. Use the :func:`asyncio.all_tasks` + function instead. .. classmethod:: current_task(loop=None) @@ -878,9 +890,10 @@ Task Object If *loop* is ``None``, the :func:`get_event_loop` function is used to get the current loop. - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.current_task` function - instead. + .. deprecated-removed:: 3.7 3.9 + + Do not call this as a task method. Use the + :func:`asyncio.current_task` function instead. .. _asyncio_generator_based_coro: @@ -916,12 +929,13 @@ enforced. async def main(): await old_style_coroutine() - This decorator is **deprecated** and is scheduled for removal in - Python 3.10. - This decorator should not be used for :keyword:`async def` coroutines. + .. deprecated-removed:: 3.8 3.10 + + Use :keyword:`async def` instead. + .. function:: iscoroutine(obj) Return ``True`` if *obj* is a :ref:`coroutine object `. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst new file mode 100644 index 00000000000000..c23b9c618323de --- /dev/null +++ b/Doc/library/audit_events.rst @@ -0,0 +1,21 @@ +.. _audit-events: + +.. index:: single: audit events + +Audit events table +================== + +This table contains all events raised by :func:`sys.audit` or +:c:func:`PySys_Audit` calls throughout the CPython runtime and the +standard library. + +See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for +information on handling these events. + +.. impl-detail:: + + This table is generated from the CPython documentation, and may not + represent events raised by other implementations. See your runtime + specific documentation for actual events raised. + +.. audit-event-table:: diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 116ffcf88e3b69..7e4066cd436ad5 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -343,7 +343,7 @@ The :mod:`bdb` module also defines two classes: For backwards compatibility. Calls the :meth:`run` method. - .. method:: runcall(func, *args, **kwds) + .. method:: runcall(func, /, *args, **kwds) Debug a single function call, and return its result. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 89ecddc7780fa3..98d8679fa3dcda 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -145,8 +145,8 @@ The :mod:`binascii` module defines the following functions: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: b2a_hex(data) - hexlify(data) +.. function:: b2a_hex(data[, sep[, bytes_per_sep=1]]) + hexlify(data[, sep[, bytes_per_sep=1]]) Return the hexadecimal representation of the binary *data*. Every byte of *data* is converted into the corresponding 2-digit hex representation. The @@ -155,6 +155,24 @@ The :mod:`binascii` module defines the following functions: Similar functionality (but returning a text string) is also conveniently accessible using the :meth:`bytes.hex` method. + If *sep* is specified, it must be a single character str or bytes object. + It will be inserted in the output after every *bytes_per_sep* input bytes. + Separator placement is counted from the right end of the output by default, + if you wish to count from the left, supply a negative *bytes_per_sep* value. + + >>> import binascii + >>> binascii.b2a_hex(b'\xb9\x01\xef') + b'b901ef' + >>> binascii.hexlify(b'\xb9\x01\xef', '-') + b'b9-01-ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b'_', 2) + b'b9_01ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b' ', -2) + b'b901 ef' + + .. versionchanged:: 3.8 + The *sep* and *bytes_per_sep* parameters were added. + .. function:: a2b_hex(hexstr) unhexlify(hexstr) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 946cc67dd301fc..277de601cb7b6f 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -83,7 +83,7 @@ All of the classes in this module may safely be accessed from multiple threads. The *buffering* argument is ignored. Its use is deprecated since Python 3.0. - If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between + If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be an integer between ``1`` and ``9`` specifying the level of compression: ``1`` produces the least compression, and ``9`` (default) produces the most compression. @@ -148,7 +148,7 @@ Incremental (de)compression incrementally. For one-shot compression, use the :func:`compress` function instead. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. .. method:: compress(data) @@ -234,9 +234,9 @@ One-shot (de)compression .. function:: compress(data, compresslevel=9) - Compress *data*. + Compress *data*, a :term:`bytes-like object `. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. For incremental compression, use a :class:`BZ2Compressor` instead. @@ -244,7 +244,7 @@ One-shot (de)compression .. function:: decompress(data) - Decompress *data*. + Decompress *data*, a :term:`bytes-like object `. If *data* is the concatenation of multiple compressed streams, decompress all of the streams. @@ -254,3 +254,77 @@ One-shot (de)compression .. versionchanged:: 3.3 Support for multi-stream inputs was added. +.. _bz2-usage-examples: + +Examples of usage +----------------- + +Below are some examples of typical usage of the :mod:`bz2` module. + +Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> c = bz2.compress(data) + >>> len(data) / len(c) # Data compression ratio + 1.513595166163142 + + >>> d = bz2.decompress(c) + >>> data == d # Check equality to original object after round-trip + True + +Using :class:`BZ2Compressor` for incremental compression: + + >>> import bz2 + + >>> def gen_data(chunks=10, chunksize=1000): + ... """Yield incremental blocks of chunksize bytes.""" + ... for _ in range(chunks): + ... yield b"z" * chunksize + ... + >>> comp = bz2.BZ2Compressor() + >>> out = b"" + >>> for chunk in gen_data(): + ... # Provide data to the compressor object + ... out = out + comp.compress(chunk) + ... + >>> # Finish the compression process. Call this once you have + >>> # finished providing data to the compressor. + >>> out = out + comp.flush() + +The example above uses a very "nonrandom" stream of data +(a stream of `b"z"` chunks). Random data tends to compress poorly, +while ordered, repetitive data usually yields a high compression ratio. + +Writing and reading a bzip2-compressed file in binary mode: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> with bz2.open("myfile.bz2", "wb") as f: + ... # Write compressed data to file + ... unused = f.write(data) + + >>> with bz2.open("myfile.bz2", "rb") as f: + ... # Decompress data from file + ... content = f.read() + + >>> content == data # Check equality to original object after round-trip + True diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 9d81730f201222..28cd96b0e12da9 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -6,13 +6,12 @@ -------------- -This module is always available. It provides access to mathematical functions -for complex numbers. The functions in this module accept integers, -floating-point numbers or complex numbers as arguments. They will also accept -any Python object that has either a :meth:`__complex__` or a :meth:`__float__` -method: these methods are used to convert the object to a complex or -floating-point number, respectively, and the function is then applied to the -result of the conversion. +This module provides access to mathematical functions for complex numbers. The +functions in this module accept integers, floating-point numbers or complex +numbers as arguments. They will also accept any Python object that has either a +:meth:`__complex__` or a :meth:`__float__` method: these methods are used to +convert the object to a complex or floating-point number, respectively, and +the function is then applied to the result of the conversion. .. note:: diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index b3246376846dd0..5048621bf0ad14 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1106,11 +1106,6 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp1258 | windows-1258 | Vietnamese | +-----------------+--------------------------------+--------------------------------+ -| cp65001 | | Windows only: Windows UTF-8 | -| | | (``CP_UTF8``) | -| | | | -| | | .. versionadded:: 3.3 | -+-----------------+--------------------------------+--------------------------------+ | euc_jp | eucjp, ujis, u-jis | Japanese | +-----------------+--------------------------------+--------------------------------+ | euc_jis_2004 | jisx0213, eucjis2004 | Japanese | @@ -1203,7 +1198,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | mac_iceland | maciceland | Icelandic | +-----------------+--------------------------------+--------------------------------+ -| mac_latin2 | maclatin2, maccentraleurope | Central and Eastern Europe | +| mac_latin2 | maclatin2, maccentraleurope, | Central and Eastern Europe | +| | mac_centeuro | | +-----------------+--------------------------------+--------------------------------+ | mac_roman | macroman, macintosh | Western Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1235,7 +1231,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_7 | U7, unicode-1-1-utf-7 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8 | U8, UTF, utf8 | all languages | +| utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ @@ -1246,6 +1242,9 @@ particular, the following variants typically exist: The utf-32\* decoders no longer decode byte sequences that correspond to surrogate code points. +.. versionchanged:: 3.8 + ``cp65001`` is now an alias to ``utf_8``. + Python Specific Encodings ------------------------- diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 64de970fec94b4..90a3f4bea9a45b 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -33,10 +33,10 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :class:`UserString` wrapper around string objects for easier string subclassing ===================== ==================================================================== -.. versionchanged:: 3.3 +.. deprecated-removed:: 3.3 3.9 Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. For backwards compatibility, they continue to be visible in this module through - Python 3.7. Subsequently, they will be removed entirely. + Python 3.8. :class:`ChainMap` objects @@ -1017,15 +1017,6 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`~somenamedtuple._replace` to -customize a prototype instance: - - >>> Account = namedtuple('Account', 'owner balance transaction_count') - >>> default_account = Account('', 0.0, 0) - >>> johns_account = default_account._replace(owner='John') - >>> janes_account = default_account._replace(owner='Jane') - - .. seealso:: * See :class:`typing.NamedTuple` for a way to add type hints for named @@ -1141,7 +1132,7 @@ original insertion position is changed and moved to the end:: def __setitem__(self, key, value): super().__setitem__(key, value) - super().move_to_end(key) + self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing variants of :func:`functools.lru_cache`:: @@ -1149,7 +1140,7 @@ variants of :func:`functools.lru_cache`:: class LRU(OrderedDict): 'Limit size, evicting the least recently looked-up key when full' - def __init__(self, maxsize=128, *args, **kwds): + def __init__(self, maxsize=128, /, *args, **kwds): self.maxsize = maxsize super().__init__(*args, **kwds) diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index 1360c7cc9f1db4..b672a05b39145d 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -21,7 +21,7 @@ spaces, the coordinates are all between 0 and 1. .. seealso:: More information about color spaces can be found at - http://poynton.ca/ColorFAQ.html and + https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. The :mod:`colorsys` module defines the following functions: diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 5e08616e9347b0..bb5000a0736ccc 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -158,7 +158,8 @@ Public functions The argument *workers* specifies how many workers are used to compile files in parallel. The default is to not use multiple workers. If the platform can't use multiple workers and *workers* argument is given, - then sequential compilation will be used as a fallback. If *workers* is + then sequential compilation will be used as a fallback. If *workers* + is 0, the number of cores in the system is used. If *workers* is lower than ``0``, a :exc:`ValueError` will be raised. *invalidation_mode* should be a member of the @@ -184,6 +185,9 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.8 + Setting *workers* to 0 now chooses the optimal number of cores. + .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) Compile the file with path *fullname*. Return a true value if the file diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 39cd9ff4826597..b150990b83b75b 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -28,5 +28,3 @@ The following are support modules for some of the above services: .. toctree:: _thread.rst - _dummy_thread.rst - dummy_threading.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 8d6b1e8d71ff47..d71f2d80c9e2d0 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -28,7 +28,7 @@ Executor Objects An abstract class that provides methods to execute calls asynchronously. It should not be used directly, but through its concrete subclasses. - .. method:: submit(fn, *args, **kwargs) + .. method:: submit(fn, /, *args, **kwargs) Schedules the callable, *fn*, to be executed as ``fn(*args **kwargs)`` and returns a :class:`Future` object representing the execution of the @@ -159,6 +159,15 @@ And:: .. versionchanged:: 3.7 Added the *initializer* and *initargs* arguments. + .. versionchanged:: 3.8 + Default value of *max_workers* is changed to ``min(32, os.cpu_count() + 4)``. + This default value preserves at least 5 workers for I/O bound tasks. + It utilizes at most 32 CPU cores for CPU bound tasks which release the GIL. + And it avoids using very large resources implicitly on many-core machines. + + ThreadPoolExecutor now reuses idle worker threads before starting + *max_workers* worker threads too. + .. _threadpoolexecutor-example: @@ -216,6 +225,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. given, it will default to the number of processors on the machine. If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` will be raised. + On Windows, *max_workers* must be equal or lower than ``61``. If it is not + then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then + the default chosen will be at most ``61``, even if more processors are + available. *mp_context* can be a multiprocessing context or None. It will be used to launch the workers. If *mp_context* is ``None`` or not given, the default multiprocessing context is used. @@ -293,9 +306,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -410,8 +424,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -442,7 +457,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 017a87a5648c39..0aa4ad76523480 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -416,7 +416,7 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. - .. method:: callback(callback, *args, **kwds) + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to the callback stack. @@ -473,7 +473,7 @@ Functions and classes provided: Similar to :meth:`push` but expects either an asynchronous context manager or a coroutine function. - .. method:: push_async_callback(callback, *args, **kwds) + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`callback` but expects a coroutine function. @@ -637,7 +637,7 @@ even further by means of a small helper class:: from contextlib import ExitStack class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): + def __init__(self, callback, /, *args, **kwds): super(Callback, self).__init__() self.callback(callback, *args, **kwds) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 40fca56d8029e9..43920210095951 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -49,7 +49,7 @@ The example below would like to show how to register a pickle function and how it will be used: >>> import copyreg, copy, pickle - >>> class C(object): + >>> class C: ... def __init__(self, a): ... self.a = a ... diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 17534fcc4615dc..49e22fa73ed265 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -443,7 +443,8 @@ read CSV files (assuming they support complex numbers at all). .. method:: csvwriter.writerow(row) Write the *row* parameter to the writer's file object, formatted according to - the current dialect. + the current dialect. Return the return value of the call to the *write* method + of the underlying file object. .. versionchanged:: 3.5 Added support of arbitrary iterables. @@ -467,9 +468,14 @@ DictWriter objects have the following public method: .. method:: DictWriter.writeheader() - Write a row with the field names (as specified in the constructor). + Write a row with the field names (as specified in the constructor) to + the writer's file object, formatted according to the current dialect. Return + the return value of the :meth:`csvwriter.writerow` call used internally. .. versionadded:: 3.2 + .. versionchanged:: 3.8 + :meth:`writeheader` now also returns the value returned by + the :meth:`csvwriter.writerow` method it uses internally. .. _csv-examples: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index baab0de8f8ac5e..680703d4483f37 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -576,7 +576,7 @@ Here is a simple example of a POINT structure, which contains two integers named >>> POINT(1, 2, 3) Traceback (most recent call last): File "", line 1, in - ValueError: too many initializers + TypeError: too many initializers >>> You can, however, build much more complicated structures. A structure can @@ -1509,6 +1509,17 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader + + Loading a library through any of these objects raises an + :ref:`auditing event ` ``ctypes.dlopen`` with string argument + ``name``, the name used to load the library. + +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader + + Accessing a function on a loaded library raises an auditing event + ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` + (the symbol's name as a string or integer). .. _ctypes-foreign-functions: @@ -2032,6 +2043,12 @@ Data types This method returns a ctypes type instance using the memory specified by *address* which must be an integer. + .. audit-event:: ctypes.cdata address ctypes._CData.from_address + + This method, and others that indirectly call this method, raises an + :ref:`auditing event ` ``ctypes.cdata`` with argument + ``address``. + .. method:: from_param(obj) This method adapts *obj* to a ctypes type. It is called with the actual @@ -2376,7 +2393,7 @@ other data types containing pointer type fields. and so on). Later assignments to the :attr:`_fields_` class variable will raise an AttributeError. - It is possible to defined sub-subclasses of structure types, they inherit + It is possible to define sub-subclasses of structure types, they inherit the fields of the base class plus the :attr:`_fields_` defined in the sub-subclass, if any. @@ -2424,7 +2441,7 @@ other data types containing pointer type fields. td.lptdesc = POINTER(some_type) td.u.lptdesc = POINTER(some_type) - It is possible to defined sub-subclasses of structures, they inherit the + It is possible to define sub-subclasses of structures, they inherit the fields of the base class. If the subclass definition has a separate :attr:`_fields_` variable, the fields specified in this are appended to the fields of the base class. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 2a4d9ce8a35a4e..c88d3520e988ff 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -656,7 +656,7 @@ The module :mod:`curses` defines the following functions: foreground color on the default background. -.. function:: wrapper(func, ...) +.. function:: wrapper(func, /, *args, **kwargs) Initialize curses and call another callable object, *func*, which should be the rest of your curses-using application. If the application raises an exception, @@ -708,9 +708,16 @@ the following methods and attributes: .. note:: - Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. + * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. + Attempting to write to the lower right corner of a window, subwindow, + or pad will cause an exception to be raised after the string is printed. + + * A `bug in ncurses `_, the backend + for this Python module, can cause SegFaults when resizing windows. This + is fixed in ncurses-6.1-20190511. If you are stuck with an earlier + ncurses, you can avoid triggering this if you do not call :func:`addstr` + with a *str* that has embedded newlines. Instead, call :func:`addstr` + separately for each line. .. method:: window.attroff(attr) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index db5c3e0c7e2893..9e0288290203dd 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -356,7 +356,7 @@ Module-level decorators, classes, and functions def add_one(self): return self.x + 1 -.. function:: replace(instance, **changes) +.. function:: replace(instance, /, **changes) Creates a new object of the same type of ``instance``, replacing fields with values from ``changes``. If ``instance`` is not a Data diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index abdc977354803e..c935e3526512fd 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1365,56 +1365,64 @@ Examples of working with datetime objects: Using datetime with tzinfo: - >>> from datetime import timedelta, datetime, tzinfo - >>> class GMT1(tzinfo): + >>> from datetime import timedelta, datetime, tzinfo, timezone + >>> class KabulTz(tzinfo): + ... # Kabul used +4 until 1945, when they moved to +4:30 + ... UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March - ... d = datetime(dt.year, 4, 1) # ends last Sunday in October - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... if dt.year < 1945: + ... return timedelta(hours=4) + ... elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + ... # If dt falls in the imaginary range, use fold to decide how + ... # to resolve. See PEP495 + ... return timedelta(hours=4, minutes=(30 if dt.fold else 0)) ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +1" + ... return timedelta(hours=4, minutes=30) + ... + ... def fromutc(self, dt): + ... # A custom implementation is required for fromutc as + ... # the input to this function is a datetime with utc values + ... # but with a tzinfo set to self + ... # See datetime.astimezone or fromtimestamp + ... + ... # Follow same validations as in datetime.tzinfo + ... if not isinstance(dt, datetime): + ... raise TypeError("fromutc() requires a datetime argument") + ... if dt.tzinfo is not self: + ... raise ValueError("dt.tzinfo is not self") + ... + ... if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: + ... return dt + timedelta(hours=4, minutes=30) + ... else: + ... return dt + timedelta(hours=4) ... - >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) ... def dst(self, dt): - ... d = datetime(dt.year, 4, 1) - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... return timedelta(0) + ... + ... def tzname(self, dt): + ... if dt >= self.UTC_MOVE_DATE: + ... return "+04:30" ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +2" + ... return "+04" ... - >>> gmt1 = GMT1() - >>> # Daylight Saving Time - >>> dt1 = datetime(2006, 11, 21, 16, 30, tzinfo=gmt1) - >>> dt1.dst() - datetime.timedelta(0) - >>> dt1.utcoffset() - datetime.timedelta(seconds=3600) - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=gmt1) - >>> dt2.dst() - datetime.timedelta(seconds=3600) - >>> dt2.utcoffset() - datetime.timedelta(seconds=7200) + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" + ... + >>> tz1 = KabulTz() + >>> # Datetime before the change + >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> print(dt1.utcoffset()) + 4:00:00 + >>> # Datetime after the change + >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> print(dt2.utcoffset()) + 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(GMT2()) - >>> dt3 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 14, 0, tzinfo=) - >>> dt2 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=) + >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 + datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) + >>> dt2 + datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) >>> dt2.utctimetuple() == dt3.utctimetuple() True @@ -1656,26 +1664,27 @@ Instance methods: Example: >>> from datetime import time, tzinfo, timedelta - >>> class GMT1(tzinfo): + >>> class TZ1(tzinfo): ... def utcoffset(self, dt): ... return timedelta(hours=1) ... def dst(self, dt): ... return timedelta(0) ... def tzname(self,dt): - ... return "Europe/Prague" + ... return "+01:00" + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=GMT1()) - >>> t # doctest: +ELLIPSIS - datetime.time(12, 10, 30, tzinfo=) - >>> gmt = GMT1() + >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t + datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() '12:10:30+01:00' >>> t.dst() datetime.timedelta(0) >>> t.tzname() - 'Europe/Prague' + '+01:00' >>> t.strftime("%H:%M:%S %Z") - '12:10:30 Europe/Prague' + '12:10:30 +01:00' >>> 'The {} is {:%H:%M}.'.format("time", t) 'The time is 12:10.' @@ -2048,6 +2057,9 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. +For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: +any components not specified in the format string will be pulled from the default value. [#]_ + The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2081,7 +2093,7 @@ format codes. | | where 0 is Sunday and 6 is | | | | | Saturday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | @@ -2094,29 +2106,29 @@ format codes. | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | | | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | | | number, zero-padded on the | 999999 | | @@ -2130,19 +2142,19 @@ format codes. | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Sunday as the first day of | | \(9) | | | the week) as a zero padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Monday as the first day of | | \(9) | | | the week) as a decimal number. | | | | | All days in a new year | | | | | preceding the first Monday | | | @@ -2182,8 +2194,8 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. | ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | | | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | | | the first day of the week. | | | | | Week 01 is the week containing | | | | | Jan 4. | | | @@ -2279,6 +2291,12 @@ Notes: :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. +(9) + When used with the :meth:`strptime` method, the leading zero is optional + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + ``%W``, and ``%V``. Format ``%y`` does require a leading zero. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 88a2fa62a56588..60223657a44043 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -5,10 +5,13 @@ Debugging and Profiling These libraries help you with Python development: the debugger enables you to step through code, analyze stack frames and set breakpoints etc., and the profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. +allowing you to identify bottlenecks in your programs. Auditing events +provide visibility into runtime behaviors that would otherwise require +intrusive debugging or patching. .. toctree:: + audit_events.rst bdb.rst faulthandler.rst pdb.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 060d4bb6997a4d..39a3e130afd3e0 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -645,10 +645,12 @@ the original TOS1. .. opcode:: MAP_ADD (i) - Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict + Calls ``dict.__setitem__(TOS1[-i], TOS1, TOS)``. Used to implement dict comprehensions. .. versionadded:: 3.1 + .. versionchanged:: 3.8 + Map value is TOS and map key is TOS1. Before, those were reversed. For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the @@ -708,7 +710,7 @@ iterations of the loop. Cleans up the value stack and the block stack. If *preserve_tos* is not ``0`` TOS first is popped from the stack and pushed on the stack after - perfoming other stack operations: + performing other stack operations: * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY` or :opcode:`CALL_FINALLY`) it is popped from the stack. @@ -1113,9 +1115,13 @@ All of the following opcodes use their arguments. .. opcode:: RAISE_VARARGS (argc) - Raises an exception. *argc* indicates the number of arguments to the raise - statement, ranging from 0 to 3. The handler will find the traceback as TOS2, - the parameter as TOS1, and the exception as TOS. + Raises an exception using one of the 3 forms of the ``raise`` statement, + depending on the value of *argc*: + + * 0: ``raise`` (re-raise previous exception) + * 1: ``raise TOS`` (raise exception instance or type at ``TOS``) + * 2: ``raise TOS1 from TOS`` (raise exception instance or type at ``TOS1`` + with ``__cause__`` set to ``TOS``) .. opcode:: CALL_FUNCTION (argc) @@ -1215,10 +1221,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/dummy_threading.rst b/Doc/library/dummy_threading.rst deleted file mode 100644 index dfc3289abb15c1..00000000000000 --- a/Doc/library/dummy_threading.rst +++ /dev/null @@ -1,20 +0,0 @@ -:mod:`dummy_threading` --- Drop-in replacement for the :mod:`threading` module -============================================================================== - -.. module:: dummy_threading - :synopsis: Drop-in replacement for the threading module. - -**Source code:** :source:`Lib/dummy_threading.py` - -.. deprecated:: 3.7 - Python now always has threading enabled. Please use :mod:`threading` instead. - --------------- - -This module provides a duplicate interface to the :mod:`threading` module. -It was meant to be imported when the :mod:`_thread` module was not provided -on a platform. - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index f02b35b910a0cf..09ea64a5a01aae 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -6,6 +6,7 @@ .. module:: email.message :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 + :noindex: The :class:`Message` class is very similar to the diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 70bf61323c39ed..e4752a5edf841f 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -12,6 +12,11 @@ This module is part of the legacy (``Compat32``) email API. In the new API the functionality is provided by the *cte* parameter of the :meth:`~email.message.EmailMessage.set_content` method. +This module is deprecated in Python 3. The functions provided here +should not be called explicitly since the :class:`~email.mime.text.MIMEText` +class sets the content type and CTE header using the *_subtype* and *_charset* +values passed during the instaniation of that class. + The remaining text in this section is the original documentation of the module. When creating :class:`~email.message.Message` objects from scratch, you often diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index fc535a3e4399ff..2d9bae6a7ee57b 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -36,7 +36,7 @@ something that contains only ASCII characters, using the standard email RFC Content Transfer Encoding techniques for encoding email messages for transport over channels that are not "8 bit clean". -To accomodate reproducible processing of SMIME-signed messages +To accommodate reproducible processing of SMIME-signed messages :class:`Generator` disables header folding for message parts of type ``multipart/signed`` and all subparts. @@ -188,7 +188,7 @@ to be using :class:`BytesGenerator`, and not :class:`Generator`. (This is required because strings cannot represent non-ASCII bytes.) Convert any bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, - transform parts with non-ASCII :mailheader:`Cotnent-Transfer-Encoding` + transform parts with non-ASCII :mailheader:`Content-Transfer-Encoding` (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ce283c6b596cf2..9376da2b8d39ce 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -107,7 +107,7 @@ headers. method if it wishes to set additional attributes beyond those provided by ``BaseHeader`` itself. Such an ``init`` method should look like this:: - def init(self, *args, **kw): + def init(self, /, *args, **kw): self._myattr = kw.pop('myattr') super().init(*args, **kw) @@ -321,19 +321,26 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. The default mappings are: - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader + :subject: UniqueUnstructuredHeader + :date: UniqueDateHeader + :resent-date: DateHeader + :orig-date: UniqueDateHeader + :sender: UniqueSingleAddressHeader + :resent-sender: SingleAddressHeader + :to: UniqueAddressHeader + :resent-to: AddressHeader + :cc: UniqueAddressHeader + :resent-cc: AddressHeader + :bcc: UniqueAddressHeader + :resent-bcc: AddressHeader + :from: UniqueAddressHeader + :resent-from: AddressHeader + :reply-to: UniqueAddressHeader + :mime-version: MIMEVersionHeader + :content-type: ContentTypeHeader + :content-disposition: ContentDispositionHeader + :content-transfer-encoding: ContentTransferEncodingHeader + :message-id: MessageIDHeader ``HeaderRegistry`` has the following methods: diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f63326d1a2..a2bb045e57e3c0 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -119,6 +119,8 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap + .. note:: The bootstrapping process has side effects on both ``sys.path`` and diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 19277d76995fed..d7d319a9513450 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -739,9 +739,11 @@ Some rules: :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. 5. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the mixed-in - type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or - :func:`repr` is desired, use the `!s` or `!r` format codes. + and :func:`format` will use the mixed-in type's :meth:`__format__` + unless :meth:`__str__` or :meth:`__format__` is overridden in the subclass, + in which case the overridden methods or :class:`Enum` methods will be used. + Use the !s and !r format codes to force usage of the :class:`Enum` class's + :meth:`__str__` and :meth:`__repr__` methods. When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------ diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 88112f6b7e545b..2db9674952d7b6 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -28,6 +28,10 @@ descriptor. Operations in this module used to raise an :exc:`IOError` where they now raise an :exc:`OSError`. +.. versionchanged:: 3.8 + The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and + ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file + descriptors. The module defines the following functions: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index af9dff34a80827..f5e5280a136399 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or @@ -54,7 +55,7 @@ provided by this module. The following function is the primary interface of this module: -.. function:: input(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -72,8 +73,9 @@ The following function is the primary interface of this module: .. versionchanged:: 3.2 Can be used as a context manager. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. + .. versionchanged:: 3.8 + The keyword parameters *mode* and *openhook* are now keyword-only. + The following functions use the global state created by :func:`fileinput.input`; if there is no active state, :exc:`RuntimeError` is raised. @@ -135,7 +137,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, @@ -160,18 +162,20 @@ available for subclassing as well: with FileInput(files=('spam.txt', 'eggs.txt')) as input: process(input) + .. versionchanged:: 3.2 Can be used as a context manager. .. deprecated:: 3.4 The ``'rU'`` and ``'U'`` modes. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. - .. deprecated:: 3.8 Support for :meth:`__getitem__` method is deprecated. + .. versionchanged:: 3.8 + The keyword parameter *mode* and *openhook* are now keyword-only. + + **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 6c39f9a59fc1fd..e006d019b837dc 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -190,6 +190,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 *source_address* parameter was added. @@ -223,6 +225,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Send a simple command string to the server and return the response string. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + .. method:: FTP.voidcmd(cmd) @@ -230,6 +234,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. nothing if a response code corresponding to success (codes in the range 200--299) is received. Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 613e4f74ac4176..e146f5a95acc3d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -128,6 +128,8 @@ are always available. They are listed here in alphabetical order. :func:`breakpoint` will automatically call that, allowing you to drop into the debugger of choice. + .. audit-event:: builtins.breakpoint breakpointhook breakpoint + .. versionadded:: 3.7 .. _func-bytearray: @@ -257,6 +259,12 @@ are always available. They are listed here in alphabetical order. can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on the :class:`~__future__._Feature` instance in the :mod:`__future__` module. + The optional argument *flags* also controls whether the compiled source is + allowed to contain top-level ``await``, ``async for`` and ``async with``. + When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code + object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively + executed via ``await eval(code_object)``. + The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as given by :option:`-O` options. Explicit levels are ``0`` (no optimization; @@ -269,6 +277,12 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. + .. audit-event:: compile source,filename compile + + Raises an :ref:`auditing event ` ``compile`` with arguments + ``source`` and ``filename``. This event may also be raised by implicit + compilation. + .. note:: When compiling a string with multi-line code in ``'single'`` or @@ -290,6 +304,10 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. + .. versionadded:: 3.8 + ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable + support for top-level ``await``, ``async for``, and ``async with``. + .. class:: complex([real[, imag]]) @@ -302,6 +320,11 @@ are always available. They are listed here in alphabetical order. :class:`int` and :class:`float`. If both arguments are omitted, returns ``0j``. + For a general Python object ``x``, ``complex(x)`` delegates to + ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back + to :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. + .. note:: When converting from a string, the string must not contain whitespace @@ -314,6 +337,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__complex__` and + :meth:`__float__` are not defined. + .. function:: delattr(object, name) @@ -463,6 +490,11 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. + .. audit-event:: exec code_object eval + + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. + .. index:: builtin: exec .. function:: exec(object[, globals[, locals]]) @@ -478,7 +510,8 @@ are always available. They are listed here in alphabetical order. :func:`exec` function. The return value is ``None``. In all cases, if the optional parts are omitted, the code is executed in the - current scope. If only *globals* is provided, it must be a dictionary, which + current scope. If only *globals* is provided, it must be a dictionary + (and not a subclass of dictionary), which will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember @@ -492,6 +525,11 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + .. audit-event:: exec code_object exec + + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. + .. note:: The built-in functions :func:`globals` and :func:`locals` return the current @@ -557,7 +595,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. + ``x.__float__()``. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. If no argument is given, ``0.0`` is returned. @@ -582,6 +621,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + .. index:: single: __format__ @@ -737,13 +779,24 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. + .. audit-event:: builtins.input prompt input + + Raises an :ref:`auditing event ` ``builtins.input`` with + argument ``prompt`` before reading input + + .. audit-event:: builtins.input/result result input + + Raises an auditing event ``builtins.input/result`` with the result after + successfully reading input. + .. class:: int([x]) int(x, base=10) Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -775,6 +828,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + .. function:: isinstance(object, classinfo) @@ -1166,6 +1222,11 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. + .. audit-event:: open file,mode,flags open + + The ``mode`` and ``flags`` arguments may have been modified or inferred from + the original call. + .. versionchanged:: 3.3 @@ -1218,9 +1279,24 @@ are always available. They are listed here in alphabetical order. operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. + returns ``100``, but ``10**-2`` returns ``0.01``. + + For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be + of integer type and *z* must be nonzero. If *z* is present and *y* is + negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x, + -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*. + + Here's an example of computing an inverse for ``38`` modulo ``97``:: + + >>> pow(38, -1, 97) + 23 + >>> 23 * 38 % 97 == 1 + True + + .. versionchanged:: 3.8 + For :class:`int` operands, the three-argument form of ``pow`` now allows + the second argument to be negative, permitting computation of modular + inverses. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -1488,11 +1564,11 @@ are always available. They are listed here in alphabetical order. about strings, see :ref:`textseq`. -.. function:: sum(iterable[, start]) +.. function:: sum(iterable, /, start=0) Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. + total. The *iterable*'s items are normally numbers, and the start value is not + allowed to be a string. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 16a779fa836858..d3debac8432b19 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -76,7 +76,8 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 -.. decorator:: lru_cache(maxsize=128, typed=False) +.. decorator:: lru_cache(user_function) + lru_cache(maxsize=128, typed=False) Decorator to wrap a function with a memoizing callable that saves up to the *maxsize* most recent calls. It can save time when an expensive or I/O bound @@ -90,6 +91,15 @@ The :mod:`functools` module defines the following functions: differ in their keyword argument order and may have two separate cache entries. + If *user_function* is specified, it must be a callable. This allows the + *lru_cache* decorator to be applied directly to a user function, leaving + the *maxsize* at its default value of 128:: + + @lru_cache + def count_vowels(sentence): + sentence = sentence.casefold() + return sum(sentence.count(vowel) for vowel in 'aeiou') + If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. The LRU feature performs best when *maxsize* is a power-of-two. @@ -165,6 +175,9 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.3 Added the *typed* option. + .. versionchanged:: 3.8 + Added the *user_function* option. + .. decorator:: total_ordering Given a class defining one or more rich comparison ordering methods, this @@ -208,7 +221,7 @@ The :mod:`functools` module defines the following functions: Returning NotImplemented from the underlying comparison function for unrecognised types is now supported. -.. function:: partial(func, *args, **keywords) +.. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called will behave like *func* called with the positional arguments *args* @@ -217,7 +230,7 @@ The :mod:`functools` module defines the following functions: supplied, they extend and override *keywords*. Roughly equivalent to:: - def partial(func, *args, **keywords): + def partial(func, /, *args, **keywords): def newfunc(*fargs, **fkeywords): newkeywords = {**keywords, **fkeywords} return func(*args, *fargs, **newkeywords) @@ -239,7 +252,7 @@ The :mod:`functools` module defines the following functions: 18 -.. class:: partialmethod(func, *args, **keywords) +.. class:: partialmethod(func, /, *args, **keywords) Return a new :class:`partialmethod` descriptor which behaves like :class:`partial` except that it is designed to be used as a method @@ -262,7 +275,7 @@ The :mod:`functools` module defines the following functions: Example:: - >>> class Cell(object): + >>> class Cell: ... def __init__(self): ... self._alive = False ... @property diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 7f4eab5843f5f2..937330bb201b08 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -39,7 +39,7 @@ class-based API instead. Bind the *domain* to the locale directory *localedir*. More concretely, :mod:`gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where - *languages* is searched for in the environment variables :envvar:`LANGUAGE`, + *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. If *localedir* is omitted or ``None``, then the current binding for *domain* is diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 2a5f0ddc4ef319..96584437cef16b 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -52,6 +52,8 @@ For example, ``'[?]'`` matches the character ``'?'``. more directories and subdirectories. If the pattern is followed by an ``os.sep``, only directories and subdirectories match. + .. audit-event:: glob.glob pathname,recursive glob.glob + .. note:: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. @@ -65,6 +67,8 @@ For example, ``'[?]'`` matches the character ``'?'``. Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. + .. audit-event:: glob.glob pathname,recursive glob.iglob + .. function:: escape(pathname) diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 8850a33f4abb90..3349a94446d0ce 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -59,6 +59,14 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. +.. exception:: BadGzipFile + + An exception raised for invalid gzip files. It inherits :exc:`OSError`. + :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip + files. + + .. versionadded:: 3.8 + .. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index beaa720d732b4c..4e761cd39a0109 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -94,6 +94,11 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. versionchanged:: 3.8 + This class now enables TLS 1.3 + :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or + when *cert_file* is passed with a custom *context*. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index a9d7321a42c43f..1788bd8f833d1d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -159,7 +159,7 @@ contained :class:`Cookie` objects. the :class:`CookieJar`'s :class:`CookiePolicy` instance are true and false respectively), the :mailheader:`Cookie2` header is also added when appropriate. - The *request* object (usually a :class:`urllib.request..Request` instance) + The *request* object (usually a :class:`urllib.request.Request` instance) must support the methods :meth:`get_full_url`, :meth:`get_host`, :meth:`get_type`, :meth:`unverifiable`, :meth:`has_header`, :meth:`get_header`, :meth:`header_items`, :meth:`add_unredirected_header` diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index f3457a0cdc7bc4..17792b200599bd 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -55,8 +55,9 @@ in Cookie name (as :attr:`~Morsel.key`). .. class:: SimpleCookie([input]) This class derives from :class:`BaseCookie` and overrides :meth:`value_decode` - and :meth:`value_encode` to be the identity and :func:`str` respectively. - + and :meth:`value_encode`. SimpleCookie supports strings as cookie values. + When setting the value, SimpleCookie calls the builtin :func:`str()` to convert + the value to a string. Values received from HTTP are kept as strings. .. seealso:: @@ -76,15 +77,16 @@ Cookie Objects .. method:: BaseCookie.value_decode(val) - Return a decoded value from a string representation. Return value can be any - type. This method does nothing in :class:`BaseCookie` --- it exists so it can be - overridden. + Return a tuple ``(real_value, coded_value)`` from a string representation. + ``real_value`` can be any type. This method does no decoding in + :class:`BaseCookie` --- it exists so it can be overridden. .. method:: BaseCookie.value_encode(val) - Return an encoded value. *val* can be any type, but return value must be a - string. This method does nothing in :class:`BaseCookie` --- it exists so it can + Return a tuple ``(real_value, coded_value)``. *val* can be any type, but + ``coded_value`` will always be converted to a string. + This method does no encoding in :class:`BaseCookie` --- it exists so it can be overridden. In general, it should be the case that :meth:`value_encode` and diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index ad449112832671..de58f266bf5e6e 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -50,7 +50,7 @@ default title and context menu. On macOS, there is one application menu. It dynamically changes according to the window currently selected. It has an IDLE menu, and some entries -described below are moved around to conform to Apple guidlines. +described below are moved around to conform to Apple guidelines. File menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -207,9 +207,13 @@ Strip trailing whitespace Run menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. _python-shell: + Python Shell Open or wake up the Python Shell window. +.. _check-module: + Check Module Check the syntax of the module currently open in the Editor window. If the module has not been saved IDLE will either prompt the user to save or @@ -217,8 +221,10 @@ Check Module there is a syntax error, the approximate location is indicated in the Editor window. +.. _run-module: + Run Module - Do Check Module (above). If no error, restart the shell to clean the + Do :ref:`Check Module `. If no error, restart the shell to clean the environment, then execute the module. Output is displayed in the Shell window. Note that output requires use of ``print`` or ``write``. When execution is complete, the Shell retains focus and displays a prompt. @@ -226,6 +232,14 @@ Run Module This is similar to executing a file with ``python -i file`` at a command line. +.. _run-custom: + +Run... Customized + Same as :ref:`Run Module `, but run the module with customized + settings. *Command Line Arguments* extend :data:`sys.argv` as if passed + on a command line. The module can be run in the Shell without restarting. + + Shell menu (Shell window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -281,16 +295,19 @@ Configure IDLE menu. For more, see :ref:`Setting preferences ` under Help and preferences. -Zoom/Restore Height - Toggles the window between normal size and maximum height. The initial size - defaults to 40 lines by 80 chars unless changed on the General tab of the - Configure IDLE dialog. - Show/Hide Code Context (Editor Window only) Open a pane at the top of the edit window which shows the block context of the code which has scrolled above the top of the window. See :ref:`Code Context ` in the Editing and Navigation section below. +Zoom/Restore Height + Toggles the window between normal size and maximum height. The initial size + defaults to 40 lines by 80 chars unless changed on the General tab of the + Configure IDLE dialog. The maximum height for a screen is determined by + momentarily maximizing a window the first time one is zoomed on the screen. + Changing screen settings may invalidate the saved height. This toogle has + no effect when a window is maximized. + Window menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -696,10 +713,17 @@ or ``print`` or ``write`` to sys.stdout or sys.stderr, IDLE should be started in a command line window. The secondary subprocess will then be attached to that window for input and output. +The IDLE code running in the execution process adds frames to the call stack +that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and +``sys.setrecursionlimit`` to reduce the effect of the additional stack frames. + If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. +When user code raises SystemExit either directly or by calling sys.exit, IDLE +returns to a Shell prompt instead of exiting. + User output in Shell ^^^^^^^^^^^^^^^^^^^^ @@ -769,7 +793,7 @@ facilitate development of tkinter programs. Enter ``import tkinter as tk; root = tk.Tk()`` in standard Python and nothing appears. Enter the same in IDLE and a tk window appears. In standard Python, one must also enter ``root.update()`` to see the window. IDLE does the equivalent in the -background, about 20 times a second, which is about every 50 milleseconds. +background, about 20 times a second, which is about every 50 milliseconds. Next enter ``b = tk.Button(root, text='button'); b.pack()``. Again, nothing visibly changes in standard Python until one enters ``root.update()``. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index f027f82ddebe61..df63d820cfe043 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -361,6 +361,8 @@ An :class:`IMAP4` instance has the following methods: :meth:`IMAP4.send`, and :meth:`IMAP4.shutdown` methods. You may override this method. + .. audit-event:: imaplib.open self,host,port imaplib.IMAP4.open + .. method:: IMAP4.partial(message_num, message_part, start, length) @@ -430,6 +432,8 @@ An :class:`IMAP4` instance has the following methods: Sends ``data`` to the remote server. You may override this method. + .. audit-event:: imaplib.send self,data imaplib.IMAP4.send + .. method:: IMAP4.setacl(mailbox, who, what) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst new file mode 100644 index 00000000000000..2126498e728cfa --- /dev/null +++ b/Doc/library/importlib.metadata.rst @@ -0,0 +1,257 @@ +.. _using: + +========================== + Using importlib.metadata +========================== + +.. note:: + This functionality is provisional and may deviate from the usual + version semantics of the standard library. + +``importlib.metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +``importlib.resources`` in `Python 3.7 +and newer`_ (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by `PEP 566`_ or its older specifications. +By default, package metadata can live on the file system or in zip archives on +``sys.path``. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it: + +.. code-block:: shell-session + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following: + +.. code-block:: pycon + + (example) $ python + >>> from importlib.metadata import version # doctest: +SKIP + >>> version('wheel') # doctest: +SKIP + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) # doctest: +SKIP + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. + + >>> eps = entry_points() # doctest: +SKIP + >>> list(eps) # doctest: +SKIP + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] # doctest: +SKIP + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + >>> wheel # doctest: +SKIP + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> main = wheel.load() # doctest: +SKIP + >>> main # doctest: +SKIP + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entrypoints, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') # doctest: +SKIP + +The keys of the returned data structure [#f1]_ name the metadata keywords, and +their values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') # doctest: +SKIP + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a `pathlib.Path`_ derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP + >>> util # doctest: +SKIP + PackagePath('wheel/util.py') + >>> util.size # doctest: +SKIP + 859 + >>> util.dist # doctest: +SKIP + + >>> util.hash # doctest: +SKIP + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) # doctest: +SKIP + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function. Note that this returns an iterator:: + + >>> list(requires('wheel')) # doctest: +SKIP + ["pytest (>=3.0.0) ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> d.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> d.metadata['License'] # doctest: +SKIP + 'MIT' + +The full set of available metadata is not described here. See `PEP 566 +`_ for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through ``sys.path`` searches, or +package loaders directly, the metadata for a package is found through import +system `finders`_. To find a distribution package's metadata, +``importlib.metadata`` queries the list of `meta path finders`_ on +`sys.meta_path`_. + +By default ``importlib.metadata`` installs a finder for distribution packages +found on the file system. This finder doesn't actually find any *packages*, +but it can find the packages' metadata. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib.metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +``sys.meta_path``. If the finder has this method, it must return +an iterator over instances of the ``Distribution`` abstract class. This +method must have the signature:: + + def find_distributions(name=None, path=None): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, you should derive from +``Distribution`` and implement the ``load_metadata()`` method. This takes a +single argument which is the name of the package whose metadata is being +found. This instance of the ``Distribution`` base abstract class is what your +finder's ``find_distributions()`` method should return. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`Python 3.7 and newer`: https://docs.python.org/3/library/importlib.html#module-importlib.resources +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html +.. _`PEP 566`: https://www.python.org/dev/peps/pep-0566/ +.. _`finders`: https://docs.python.org/3/reference/import.html#finders-and-loaders +.. _`meta path finders`: https://docs.python.org/3/glossary.html#term-meta-path-finder +.. _`sys.meta_path`: https://docs.python.org/3/library/sys.html#sys.meta_path +.. _`pathlib.Path`: https://docs.python.org/3/library/pathlib.html#pathlib.Path + + +.. rubric:: Footnotes + +.. [#f1] Technically, the returned distribution metadata object is an + `email.message.Message + `_ + instance, but this is an implementation detail, and not part of the + stable API. You should only use dictionary-like methods and syntax + to access the metadata contents. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 23831c75842f1f..9fab0779aa7421 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -500,7 +500,7 @@ ABC hierarchy:: packages or a module). Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_loader(fullname)`` which + provide a method called ``get_resource_reader(fullname)`` which returns an object implementing this ABC's interface. If the module specified by fullname is not a package, this method should return :const:`None`. An object compatible with this ABC should only be @@ -1638,15 +1638,16 @@ import, then you should use :func:`importlib.util.find_spec`. # For illustrative purposes. name = 'itertools' - spec = importlib.util.find_spec(name) - if spec is None: - print("can't find the itertools module") + if name in sys.modules: + print(f"{name!r} already in sys.modules") + elif (spec := importlib.util.find_spec(name)) is None: + print(f"can't find the {name!r} module") else: # If you chose to perform the actual import ... module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Adding the module to sys.modules is optional. sys.modules[name] = module + spec.loader.exec_module(module) + print(f"{name!r} has been imported") Importing a source file directly @@ -1665,10 +1666,9 @@ To import a Python source file directly, use the following recipe spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Optional; only necessary if you want to be able to import the module - # by name later. sys.modules[module_name] = module + spec.loader.exec_module(module) + Setting up an importer @@ -1740,8 +1740,8 @@ Python 3.6 and newer for other parts of the code). msg = f'No module named {absolute_name!r}' raise ModuleNotFoundError(msg, name=absolute_name) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) sys.modules[absolute_name] = module + spec.loader.exec_module(module) if path is not None: setattr(parent_module, child_name, module) return module diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index d12f122a57b599..2a71201a80b2cb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -169,6 +169,9 @@ attributes: | | | variables (referenced via | | | | a function's closure) | +-----------+-------------------+---------------------------+ +| | co_posonlyargcount| number of positional only | +| | | arguments | ++-----------+-------------------+---------------------------+ | | co_kwonlyargcount | number of keyword only | | | | arguments (not including | | | | \*\* arg) | @@ -724,13 +727,9 @@ function. | Name | Meaning | +========================+==============================================+ | *POSITIONAL_ONLY* | Value must be supplied as a positional | - | | argument. | - | | | - | | Python has no explicit syntax for defining | - | | positional-only parameters, but many built-in| - | | and extension module functions (especially | - | | those that accept only one or two parameters)| - | | accept them. | + | | argument. Positional only parameters are | + | | those which appear before a ``/`` entry (if | + | | present) in a Python function definition. | +------------------------+----------------------------------------------+ | *POSITIONAL_OR_KEYWORD*| Value may be supplied as either a keyword or | | | positional argument (this is the standard | @@ -948,11 +947,6 @@ Classes and functions APIs. This function is retained primarily for use in code that needs to maintain compatibility with the Python 2 ``inspect`` module API. - .. deprecated:: 3.8 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. - .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -1028,7 +1022,7 @@ Classes and functions metatype is in use, cls will be the first element of the tuple. -.. function:: getcallargs(func, *args, **kwds) +.. function:: getcallargs(func, /, *args, **kwds) Bind the *args* and *kwds* to the argument names of the Python function or method *func*, as if it was called with them. For bound methods, bind also the diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 0f1251687aeb13..70e01153d41966 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -120,6 +120,27 @@ High-level Module Interface This is an alias for the builtin :func:`open` function. + .. audit-event:: open path,mode,flags io.open + + This function raises an :ref:`auditing event ` ``open`` with + arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments may have been modified or inferred from the original call. + + +.. function:: open_code(path) + + Opens the provided file with mode ``'rb'``. This function should be used + when the intent is to treat the contents as executable code. + + ``path`` should be an absolute path. + + The behavior of this function may be overridden by an earlier call to the + :c:func:`PyFile_SetOpenCodeHook`, however, it should always be considered + interchangeable with ``open(path, 'rb')``. Overriding the behavior is + intended for additional validation or preprocessing of the file. + + .. versionadded:: 3.8 + .. exception:: BlockingIOError @@ -306,7 +327,7 @@ I/O Base Classes Note that it's already possible to iterate on file objects using ``for line in file: ...`` without calling ``file.readlines()``. - .. method:: seek(offset[, whence]) + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default @@ -810,7 +831,7 @@ Text I/O If *size* is specified, at most *size* characters will be read. - .. method:: seek(offset[, whence]) + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is @@ -924,7 +945,7 @@ Text I/O *errors*, *newline*, *line_buffering* and *write_through*. Parameters not specified keep current settings, except - ``errors='strict`` is used when *encoding* is specified but + ``errors='strict'`` is used when *encoding* is specified but *errors* is not specified. It is not possible to change the encoding or newline if some data diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index dee9a84e333764..b7445a135b742c 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -48,7 +48,7 @@ and :meth:`flush` methods). .. method:: emit(record) If a formatter is specified, it is used to format the record. The record - is then written to the stream with a terminator. If exception information + is then written to the stream followed by :attr:`terminator`. If exception information is present, it is formatted using :func:`traceback.print_exception` and appended to the stream. @@ -68,15 +68,19 @@ and :meth:`flush` methods). :return: the old stream, if the stream was changed, or *None* if it wasn't. - .. versionadded:: 3.7 + .. versionadded:: 3.7 + .. attribute:: terminator -.. versionchanged:: 3.2 - The ``StreamHandler`` class now has a ``terminator`` attribute, default - value ``'\n'``, which is used as the terminator when writing a formatted - record to a stream. If you don't want this newline termination, you can - set the handler instance's ``terminator`` attribute to the empty string. - In earlier versions, the terminator was hardcoded as ``'\n'``. + String used as the terminator when writing a formatted record to a stream. + Default value is ``'\n'``. + + If you don't want a newline termination, you can set the handler instance's + ``terminator`` attribute to the empty string. + + In earlier versions, the terminator was hardcoded as ``'\n'``. + + .. versionadded:: 3.2 .. _file-handler: @@ -89,23 +93,26 @@ sends logging output to a disk file. It inherits the output functionality from :class:`StreamHandler`. -.. class:: FileHandler(filename, mode='a', encoding=None, delay=False) +.. class:: FileHandler(filename, mode='a', encoding=None, delay=False, errors=None) Returns a new instance of the :class:`FileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is specified, it's used to determine how encoding errors are handled. .. versionchanged:: 3.6 As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: close() Closes the file. - .. method:: emit(record) Outputs the record to the file. @@ -168,18 +175,22 @@ exclusive locks - and so there is no need for such a handler. Furthermore, for this value. -.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False) +.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False, errors=None) Returns a new instance of the :class:`WatchedFileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is provided, it determines how encoding errors are handled. .. versionchanged:: 3.6 As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: reopenIfNeeded() Checks to see if the file has changed. If it has, the existing stream is @@ -205,7 +216,7 @@ module, is the base class for the rotating file handlers, not need to instantiate this class, but it has attributes and methods you may need to override. -.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False) +.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False, errors=None) The parameters are as for :class:`FileHandler`. The attributes are: @@ -284,13 +295,14 @@ The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` module, supports rotation of disk log files. -.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False) +.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False, errors=None) Returns a new instance of the :class:`RotatingFileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is provided, it determines how encoding errors are handled. You can use the *maxBytes* and *backupCount* values to allow the file to :dfn:`rollover` at a predetermined size. When the size is about to be exceeded, @@ -311,6 +323,9 @@ module, supports rotation of disk log files. As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: doRollover() Does a rollover, as described above. @@ -331,7 +346,7 @@ The :class:`TimedRotatingFileHandler` class, located in the timed intervals. -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None) +.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None, errors=None) Returns a new instance of the :class:`TimedRotatingFileHandler` class. The specified file is opened and used as the stream for logging. On rotating it also @@ -391,6 +406,9 @@ timed intervals. rollover, and subsequent rollovers would be calculated via the normal interval calculation. + If *errors* is specified, it's used to determine how encoding errors are + handled. + .. note:: Calculation of the initial rollover time is done when the handler is initialised. Calculation of subsequent rollover times is done only when rollover occurs, and rollover occurs only when emitting output. If @@ -411,6 +429,9 @@ timed intervals. As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: doRollover() Does a rollover, as described above. @@ -470,7 +491,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: makePickle(record) Pickles the record's attribute dictionary in binary format with a length - prefix, and returns it ready for transmission across the socket. + prefix, and returns it ready for transmission across the socket. The + details of this operation are equivalent to:: + + data = pickle.dumps(record_attr_dict, 1) + datalen = struct.pack('>L', len(data)) + return datalen + data Note that pickles aren't completely secure. If you are concerned about security, you may want to override this method to implement a more secure @@ -481,8 +507,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: send(packet) - Send a pickled string *packet* to the socket. This function allows for - partial sends which can happen when the network is busy. + Send a pickled byte-string *packet* to the socket. The format of the sent + byte-string is as described in the documentation for + :meth:`~SocketHandler.makePickle`. + + This function allows for partial sends, which can happen when the network + is busy. .. method:: createSocket() @@ -543,7 +573,8 @@ over UDP sockets. .. method:: send(s) - Send a pickled string to a socket. + Send a pickled byte-string to a socket. The format of the sent byte-string + is as described in the documentation for :meth:`SocketHandler.makePickle`. .. _syslog-handler: @@ -830,7 +861,8 @@ should, then :meth:`flush` is expected to do the flushing. .. class:: BufferingHandler(capacity) - Initializes the handler with a buffer of the specified capacity. + Initializes the handler with a buffer of the specified capacity. Here, + *capacity* means the number of logging records buffered. .. method:: emit(record) @@ -854,12 +886,13 @@ should, then :meth:`flush` is expected to do the flushing. .. class:: MemoryHandler(capacity, flushLevel=ERROR, target=None, flushOnClose=True) Returns a new instance of the :class:`MemoryHandler` class. The instance is - initialized with a buffer size of *capacity*. If *flushLevel* is not specified, - :const:`ERROR` is used. If no *target* is specified, the target will need to be - set using :meth:`setTarget` before this handler does anything useful. If - *flushOnClose* is specified as ``False``, then the buffer is *not* flushed when - the handler is closed. If not specified or specified as ``True``, the previous - behaviour of flushing the buffer will occur when the handler is closed. + initialized with a buffer size of *capacity* (number of records buffered). + If *flushLevel* is not specified, :const:`ERROR` is used. If no *target* is + specified, the target will need to be set using :meth:`setTarget` before this + handler does anything useful. If *flushOnClose* is specified as ``False``, + then the buffer is *not* flushed when the handler is closed. If not specified + or specified as ``True``, the previous behaviour of flushing the buffer will + occur when the handler is closed. .. versionchanged:: 3.6 The *flushOnClose* parameter was added. @@ -959,14 +992,21 @@ possible, while any potentially slow operations (such as sending an email via .. class:: QueueHandler(queue) Returns a new instance of the :class:`QueueHandler` class. The instance is - initialized with the queue to send messages to. The queue can be any - queue-like object; it's used as-is by the :meth:`enqueue` method, which needs - to know how to send messages to it. + initialized with the queue to send messages to. The *queue* can be any + queue-like object; it's used as-is by the :meth:`enqueue` method, which + needs to know how to send messages to it. The queue is not *required* to + have the task tracking API, which means that you can use + :class:`~queue.SimpleQueue` instances for *queue*. .. method:: emit(record) - Enqueues the result of preparing the LogRecord. + Enqueues the result of preparing the LogRecord. Should an exception + occur (e.g. because a bounded queue has filled up), the + :meth:`~logging.Handler.handleError` method is called to handle the + error. This can result in the record silently being dropped (if + :attr:`logging.raiseExceptions` is ``False``) or a message printed to + ``sys.stderr`` (if :attr:`logging.raiseExceptions` is ``True``). .. method:: prepare(record) @@ -1017,11 +1057,14 @@ possible, while any potentially slow operations (such as sending an email via initialized with the queue to send messages to and a list of handlers which will handle entries placed on the queue. The queue can be any queue-like object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. If ``respect_handler_level`` is ``True``, - a handler's level is respected (compared with the level for the message) when - deciding whether to pass messages to that handler; otherwise, the behaviour - is as in previous Python versions - to always pass each message to each - handler. + to know how to get messages from it. The queue is not *required* to have the + task tracking API (though it's used if available), which means that you can + use :class:`~queue.SimpleQueue` instances for *queue*. + + If ``respect_handler_level`` is ``True``, a handler's level is respected + (compared with the level for the message) when deciding whether to pass + messages to that handler; otherwise, the behaviour is as in previous Python + versions - to always pass each message to each handler. .. versionchanged:: 3.5 The ``respect_handler_levels`` argument was added. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 08555c3a357602..cc611fc8708626 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -50,8 +50,8 @@ listed below. Logger Objects -------------- -Loggers have the following attributes and methods. Note that Loggers are never -instantiated directly, but always through the module-level function +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function ``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same name will always return a reference to the same Logger object. @@ -1196,6 +1196,21 @@ functions. | | carrying out the configuration as specified | | | by the other arguments. | +--------------+---------------------------------------------+ + | *encoding* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | FileHandler is created, and thus used when | + | | opening the output file. | + +--------------+---------------------------------------------+ + | *errors* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | FileHandler is created, and thus used when | + | | opening the output file. If not specified, | + | | the value 'backslashreplace' is used. Note | + | | that if ``None`` is specified, it will be | + | | passed as such to func:`open`, which means | + | | that it will be treated the same as passing | + | | 'errors'. | + +--------------+---------------------------------------------+ .. versionchanged:: 3.2 The *style* argument was added. @@ -1209,6 +1224,9 @@ functions. .. versionchanged:: 3.8 The *force* argument was added. + .. versionchanged:: 3.9 + The *encoding* and *errors* arguments were added. + .. function:: shutdown() Informs the logging system to perform an orderly shutdown by flushing and @@ -1226,7 +1244,9 @@ functions. The class should define :meth:`__init__` such that only a name argument is required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. .. function:: setLogRecordFactory(factory) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index d901ad2cc1c48e..f82a3b200deb7c 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -308,6 +308,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. representation. If *create* is ``True``, the mailbox is created if it does not exist. + If *create* is ``True`` and the *dirname* path exists, it will be treated as + an existing maildir without attempting to verify its directory layout. + It is for historical reasons that *dirname* is named as such rather than *path*. Maildir is a directory-based mailbox format invented for the qmail mail diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 7129525c788767..be953cfe959948 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -10,8 +10,8 @@ -------------- -This module is always available. It provides access to the mathematical -functions defined by the C standard. +This module provides access to the mathematical functions defined by the C +standard. These functions cannot be used with complex numbers; use the functions of the same name from the :mod:`cmath` module if you require support for complex @@ -36,6 +36,24 @@ Number-theoretic and representation functions :class:`~numbers.Integral` value. +.. function:: comb(n, k) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of the + expression ``(1 + x) ** n``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: copysign(x, y) Return a float with the magnitude (absolute value) of *x* but the sign of @@ -50,9 +68,12 @@ Number-theoretic and representation functions .. function:: factorial(x) - Return *x* factorial. Raises :exc:`ValueError` if *x* is not integral or + Return *x* factorial as an integer. Raises :exc:`ValueError` if *x* is not integral or is negative. + .. deprecated:: 3.9 + Accepting floats with integral values (like ``5.0``) is deprecated. + .. function:: floor(x) @@ -166,6 +187,20 @@ Number-theoretic and representation functions Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. +.. function:: isqrt(n) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + .. versionadded:: 3.8 + + .. function:: ldexp(x, i) Return ``x * (2**i)``. This is essentially the inverse of function @@ -178,6 +213,23 @@ Number-theoretic and representation functions of *x* and are floats. +.. function:: perm(n, k=None) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is None, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: prod(iterable, *, start=1) Calculate the product of all the elements in the input *iterable*. @@ -314,17 +366,20 @@ Trigonometric functions .. function:: acos(x) - Return the arc cosine of *x*, in radians. + Return the arc cosine of *x*, in radians. The result is between ``0`` and + ``pi``. .. function:: asin(x) - Return the arc sine of *x*, in radians. + Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. .. function:: atan(x) - Return the arc tangent of *x*, in radians. + Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. .. function:: atan2(y, x) @@ -538,3 +593,6 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. + +.. |nbsp| unicode:: 0xA0 + :trim: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 5728407cb34c48..f610032acbe417 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -93,6 +93,10 @@ behavior of the module. Specifying an empty list for *files* will prevent the system defaults from being applied: only the well-known values will be present from a built-in list. + If *files* is ``None`` the internal data structure is completely rebuilt to its + initial default value. This is a stable operation and will produce the same results + when called multiple times. + .. versionchanged:: 3.2 Previously, Windows registry settings were ignored. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 0f895d76b83fa8..12b14d69332d72 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -67,6 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) :noindex: @@ -155,6 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length mm.close() + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap Memory-mapped file objects support the following methods: @@ -201,6 +203,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length exception was raised on error under Unix. + .. method:: madvise(option[, start[, length]]) + + Send advice *option* to the kernel about the memory region beginning at + *start* and extending *length* bytes. *option* must be one of the + :ref:`MADV_* constants ` available on the system. If + *start* and *length* are omitted, the entire mapping is spanned. On + some systems (including Linux), *start* must be a multiple of the + :const:`PAGESIZE`. + + Availability: Systems with the ``madvise()`` system call. + + .. versionadded:: 3.8 + + .. method:: move(dest, src, count) Copy the *count* bytes starting at offset *src* to the destination index @@ -290,3 +306,38 @@ To map anonymous memory, -1 should be passed as the fileno along with the length position of the file pointer; the file position is advanced by ``1``. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. + +.. _madvise-constants: + +MADV_* Constants +++++++++++++++++ + +.. data:: MADV_NORMAL + MADV_RANDOM + MADV_SEQUENTIAL + MADV_WILLNEED + MADV_DONTNEED + MADV_REMOVE + MADV_DONTFORK + MADV_DOFORK + MADV_HWPOISON + MADV_MERGEABLE + MADV_UNMERGEABLE + MADV_SOFT_OFFLINE + MADV_HUGEPAGE + MADV_NOHUGEPAGE + MADV_DONTDUMP + MADV_DODUMP + MADV_FREE + MADV_NOSYNC + MADV_AUTOSYNC + MADV_NOCORE + MADV_CORE + MADV_PROTECT + + These options can be passed to :meth:`mmap.madvise`. Not every option will + be present on every system. + + Availability: Systems with the madvise() system call. + + .. versionadded:: 3.8 diff --git a/Doc/library/modules.rst b/Doc/library/modules.rst index 6b2a40a1b71476..565ce0525c2c99 100644 --- a/Doc/library/modules.rst +++ b/Doc/library/modules.rst @@ -17,3 +17,4 @@ The full list of modules described in this chapter is: modulefinder.rst runpy.rst importlib.rst + importlib.metadata.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a5ecfa6cc1c703..d8182feab963a1 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -102,7 +102,7 @@ to start a process. These *start methods* are will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. - Available on Unix and Windows. The default on Windows. + Available on Unix and Windows. The default on Windows and macOS. *fork* The parent process uses :func:`os.fork` to fork the Python @@ -124,6 +124,12 @@ to start a process. These *start methods* are Available on Unix platforms which support passing file descriptors over Unix pipes. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. + .. versionchanged:: 3.4 *spawn* added on all unix platforms, and *forkserver* added for some unix platforms. @@ -131,13 +137,17 @@ to start a process. These *start methods* are handles on Windows. On Unix using the *spawn* or *forkserver* start methods will also -start a *semaphore tracker* process which tracks the unlinked named -semaphores created by processes of the program. When all processes -have exited the semaphore tracker unlinks any remaining semaphores. +start a *resource tracker* process which tracks the unlinked named +system resources (such as named semaphores or +:class:`~multiprocessing.shared_memory.SharedMemory` objects) created +by processes of the program. When all processes +have exited the resource tracker unlinks any remaining tracked object. Usually there should be none, but if a process was killed by a signal -there may be some "leaked" semaphores. (Unlinking the named semaphores -is a serious matter since the system allows only a limited number, and -they will not be automatically unlinked until the next reboot.) +there may be some "leaked" resources. (Neither leaked semaphores nor shared +memory segments will be automatically unlinked until the next reboot. This is +problematic for both objects because the system allows only a limited number of +named semaphores, and shared memory segments occupy some space in the main +memory.) To select a start method you use the :func:`set_start_method` in the ``if __name__ == '__main__'`` clause of the main module. For @@ -940,6 +950,14 @@ Miscellaneous An analogue of :func:`threading.current_thread`. +.. function:: parent_process() + + Return the :class:`Process` object corresponding to the parent process of + the :func:`current_process`. For the main process, ``parent_process`` will + be ``None``. + + .. versionadded:: 3.8 + .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been @@ -2262,6 +2280,10 @@ with the :class:`Pool` class. Return whether the call completed without raising an exception. Will raise :exc:`AssertionError` if the result is not ready. + .. versionchanged:: 3.7 + If the result is not ready, :exc:`ValueError` is raised instead of + :exc:`AssertionError`. + The following example demonstrates the use of a pool:: from multiprocessing import Pool diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst index 56188c7ef53880..46f1c0783551c2 100644 --- a/Doc/library/nntplib.rst +++ b/Doc/library/nntplib.rst @@ -79,6 +79,13 @@ The module itself defines the following classes: ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') >>> + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP + + .. audit-event:: nntplib.putline self,line nntplib.NNTP + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.2 *usenetrc* is now ``False`` by default. @@ -100,6 +107,14 @@ The module itself defines the following classes: STARTTLS as described below. However, some servers only support the former. + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL + + .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionadded:: 3.2 .. versionchanged:: 3.4 diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 5d0ea7dfdd8928..fa02bde84650e1 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -339,7 +339,7 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. function:: methodcaller(name[, args...]) +.. function:: methodcaller(name, /, *args, **kwargs) Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given @@ -352,7 +352,7 @@ expect a function argument. Equivalent to:: - def methodcaller(name, *args, **kwargs): + def methodcaller(name, /, *args, **kwargs): def caller(obj): return getattr(obj, name)(*args, **kwargs) return caller diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index a167e3b885fdfc..a673b81278ea74 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -87,9 +87,10 @@ the :mod:`glob` module.) .. function:: commonpath(paths) Return the longest common sub-path of each pathname in the sequence - *paths*. Raise ValueError if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. + *paths*. Raise :exc:`ValueError` if *paths* contain both absolute + and relative pathnames, the *paths* are on the different drives or + if *paths* is empty. Unlike :func:`commonprefix`, this returns a + valid path. .. availability:: Unix, Windows. @@ -324,9 +325,9 @@ the :mod:`glob` module.) .. function:: normcase(path) - Normalize the case of a pathname. On Unix and Mac OS X, this returns the - path unchanged; on case-insensitive filesystems, it converts the path to - lowercase. On Windows, it also converts forward slashes to backward slashes. + Normalize the case of a pathname. On Windows, convert all characters in the + pathname to lowercase, and also convert forward slashes to backward slashes. + On other operating systems, return the path unchanged. Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`` (directly or indirectly through the :class:`os.PathLike` interface). diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f3b5d964ac58a8..c74d687f08fb0a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -32,12 +32,13 @@ Notes on the availability of these functions: objects, and result in an object of the same type, if a path or file name is returned. +* On VxWorks, os.fork, os.execv and os.spawn*p* are not supported. .. note:: - All functions in this module raise :exc:`OSError` in the case of invalid or - inaccessible file names and paths, or other arguments that have the correct - type, but are not accepted by the operating system. + All functions in this module raise :exc:`OSError` (or subclasses thereof) in + the case of invalid or inaccessible file names and paths, or other arguments + that have the correct type, but are not accepted by the operating system. .. exception:: error @@ -650,7 +651,7 @@ process and user. File Object Creation -------------------- -This function creates new :term:`file objects `. (See also +These functions create new :term:`file objects `. (See also :func:`~os.open` for opening file descriptors.) @@ -706,6 +707,28 @@ as internal buffering of data. pass +.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None) + + Copy *count* bytes from file descriptor *src*, starting from offset + *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + If *offset_src* is None, then *src* is read from the current position; + respectively for *offset_dst*. The files pointed by *src* and *dst* + must reside in the same filesystem, otherwise an :exc:`OSError` is + raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + + This copy is done without the additional cost of transferring data + from the kernel to user space and then back into the kernel. Additionally, + some filesystems could implement extra optimizations. The copy is done as if + both files are opened as binary. + + The return value is the amount of bytes copied. This could be less than the + amount requested. + + .. availability:: Linux kernel >= 4.5 or glibc >= 2.27. + + .. versionadded:: 3.8 + + .. function:: device_encoding(fd) Return a string describing the encoding of the device associated with *fd* @@ -828,11 +851,14 @@ as internal buffering of data. most *length* bytes in size. As of Python 3.3, this is equivalent to ``os.truncate(fd, length)``. + .. audit-event:: os.truncate fd,length os.ftruncate + .. availability:: Unix, Windows. .. versionchanged:: 3.5 Added support for Windows + .. function:: get_blocking(fd) Get the blocking mode of the file descriptor: ``False`` if the @@ -844,6 +870,7 @@ as internal buffering of data. .. versionadded:: 3.5 + .. function:: isatty(fd) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -911,6 +938,8 @@ as internal buffering of data. This function can support :ref:`paths relative to directory descriptors ` with the *dir_fd* parameter. + .. audit-event:: open path,mode,flags os.open + .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. @@ -1453,16 +1482,19 @@ features: .. _path_fd: * **specifying a file descriptor:** - For some functions, the *path* argument can be not only a string giving a path - name, but also a file descriptor. The function will then operate on the file - referred to by the descriptor. (For POSIX systems, Python will call the - ``f...`` version of the function.) - - You can check whether or not *path* can be specified as a file descriptor on - your platform using :data:`os.supports_fd`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. + Normally the *path* argument provided to functions in the :mod:`os` module + must be a string specifying a file path. However, some functions now + alternatively accept an open file descriptor for their *path* argument. + The function will then operate on the file referred to by the descriptor. + (For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + + You can check whether or not *path* can be specified as a file descriptor + for a particular function on your platform using :data:`os.supports_fd`. + If this functionality is unavailable, using it will raise a + :exc:`NotImplementedError`. - If the function also supports *dir_fd* or *follow_symlinks* arguments, it is + If the function also supports *dir_fd* or *follow_symlinks* arguments, it's an error to specify one of those when supplying *path* as a file descriptor. .. _dir_fd: @@ -1471,23 +1503,24 @@ features: should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call - the ``...at`` or ``f...at`` version of the function.) + the variant of the function with an ``at`` suffix and possibly prefixed with + ``f`` (e.g. call ``faccessat`` instead of ``access``). - You can check whether or not *dir_fd* is supported on your platform using - :data:`os.supports_dir_fd`. If it is unavailable, using it will raise a - :exc:`NotImplementedError`. + You can check whether or not *dir_fd* is supported for a particular function + on your platform using :data:`os.supports_dir_fd`. If it's unavailable, + using it will raise a :exc:`NotImplementedError`. .. _follow_symlinks: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, - the function will operate on the symbolic link itself instead of the file the - link points to. (For POSIX systems, Python will call the ``l...`` version of - the function.) + the function will operate on the symbolic link itself rather than the file + pointed to by the link. (For POSIX systems, Python will call the ``l...`` + variant of the function.) - You can check whether or not *follow_symlinks* is supported on your platform - using :data:`os.supports_follow_symlinks`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. + You can check whether or not *follow_symlinks* is supported for a particular + function on your platform using :data:`os.supports_follow_symlinks`. + If it's unavailable, using it will raise a :exc:`NotImplementedError`. @@ -1566,6 +1599,9 @@ features: This function can support :ref:`specifying a file descriptor `. The descriptor must refer to an opened directory, not an open file. + This function can raise :exc:`OSError` and subclasses such as + :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. + .. versionadded:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -1662,7 +1698,7 @@ features: .. availability:: Unix. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, + Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. .. versionchanged:: 3.6 @@ -1697,6 +1733,11 @@ features: Return a bytestring representing the current working directory. + .. versionchanged:: 3.8 + The function now uses the UTF-8 encoding on Windows, rather than the ANSI + code page: see :pep:`529` for the rationale. The function is no longer + deprecated on Windows. + .. function:: lchflags(path, flags) @@ -1768,6 +1809,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.listdir path os.listdir + .. note:: To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. @@ -1781,7 +1824,7 @@ features: The *path* parameter became optional. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. + Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -1858,8 +1901,8 @@ features: directories you can set the umask before invoking :func:`makedirs`. The file permission bits of existing parent directories are not changed. - If *exist_ok* is ``False`` (the default), an :exc:`OSError` is raised if the - target directory already exists. + If *exist_ok* is ``False`` (the default), an :exc:`FileExistsError` is + raised if the target directory already exists. .. note:: @@ -2012,8 +2055,8 @@ features: .. function:: remove(path, *, dir_fd=None) - Remove (delete) the file *path*. If *path* is a directory, :exc:`OSError` is - raised. Use :func:`rmdir` to remove directories. + Remove (delete) the file *path*. If *path* is a directory, an + :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. This function can support :ref:`paths relative to directory descriptors `. @@ -2050,13 +2093,19 @@ features: .. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - Rename the file or directory *src* to *dst*. If *dst* is a directory, - :exc:`OSError` will be raised. On Unix, if *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail on some - Unix flavors if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). On - Windows, if *dst* already exists, :exc:`OSError` will be raised even if it is a - file. + Rename the file or directory *src* to *dst*. If *dst* exists, the operation + will fail with an :exc:`OSError` subclass in a number of cases: + + On Windows, if *dst* exists a :exc:`FileExistsError` is always raised. + + On Unix, if *src* is a file and *dst* is a directory or vice-versa, an + :exc:`IsADirectoryError` or a :exc:`NotADirectoryError` will be raised + respectively. If both are directories and *dst* is empty, *dst* will be + silently replaced. If *dst* is a non-empty directory, an :exc:`OSError` + is raised. If both are files, *dst* it will be replaced silently if the user + has permission. The operation may fail on some Unix flavors if *src* and + *dst* are on different filesystems. If successful, the renaming will be an + atomic operation (this is a POSIX requirement). This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors `. @@ -2105,9 +2154,10 @@ features: .. function:: rmdir(path, *, dir_fd=None) - Remove (delete) the directory *path*. Only works when the directory is - empty, otherwise, :exc:`OSError` is raised. In order to remove whole - directory trees, :func:`shutil.rmtree` can be used. + Remove (delete) the directory *path*. If the directory does not exist or is + not empty, an :exc:`FileNotFoundError` or an :exc:`OSError` is raised + respectively. In order to remove whole directory trees, + :func:`shutil.rmtree` can be used. This function can support :ref:`paths relative to directory descriptors `. @@ -2145,6 +2195,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.scandir path os.scandir + The :func:`scandir` iterator supports the :term:`context manager` protocol and has the following method: @@ -2593,7 +2645,7 @@ features: The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. + Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.4 The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, @@ -2610,59 +2662,61 @@ features: .. data:: supports_dir_fd - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *dir_fd* parameter. Different platforms - provide different functionality, and an option that might work on one might - be unsupported on another. For consistency's sakes, functions that support - *dir_fd* always allow specifying the parameter, but will raise an exception - if the functionality is not actually available. - - To check whether a particular function permits use of its *dir_fd* - parameter, use the ``in`` operator on ``supports_dir_fd``. As an example, - this expression determines whether the *dir_fd* parameter of :func:`os.stat` - is locally available:: + A :class:`set` object indicating which functions in the :mod:`os` + module accept an open file descriptor for their *dir_fd* parameter. + Different platforms provide different features, and the underlying + functionality Python uses to implement the *dir_fd* parameter is not + available on all platforms Python supports. For consistency's sake, + functions that may support *dir_fd* always allow specifying the + parameter, but will throw an exception if the functionality is used + when it's not locally available. (Specifying ``None`` for *dir_fd* + is always supported on all platforms.) + + To check whether a particular function accepts an open file descriptor + for its *dir_fd* parameter, use the ``in`` operator on ``supports_dir_fd``. + As an example, this expression evaluates to ``True`` if :func:`os.stat` + accepts open file descriptors for *dir_fd* on the local platform:: os.stat in os.supports_dir_fd - Currently *dir_fd* parameters only work on Unix platforms; none of them work - on Windows. + Currently *dir_fd* parameters only work on Unix platforms; + none of them work on Windows. .. versionadded:: 3.3 .. data:: supports_effective_ids - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of the *effective_ids* parameter for - :func:`os.access`. If the local platform supports it, the collection will - contain :func:`os.access`, otherwise it will be empty. + A :class:`set` object indicating whether :func:`os.access` permits + specifying ``True`` for its *effective_ids* parameter on the local platform. + (Specifying ``False`` for *effective_ids* is always supported on all + platforms.) If the local platform supports it, the collection will contain + :func:`os.access`; otherwise it will be empty. - To check whether you can use the *effective_ids* parameter for - :func:`os.access`, use the ``in`` operator on ``supports_effective_ids``, - like so:: + This expression evaluates to ``True`` if :func:`os.access` supports + ``effective_ids=True`` on the local platform:: os.access in os.supports_effective_ids - Currently *effective_ids* only works on Unix platforms; it does not work on - Windows. + Currently *effective_ids* is only supported on Unix platforms; + it does not work on Windows. .. versionadded:: 3.3 .. data:: supports_fd - A :class:`~collections.abc.Set` object indicating which functions in the + A :class:`set` object indicating which functions in the :mod:`os` module permit specifying their *path* parameter as an open file - descriptor. Different platforms provide different functionality, and an - option that might work on one might be unsupported on another. For - consistency's sakes, functions that support *fd* always allow specifying - the parameter, but will raise an exception if the functionality is not - actually available. + descriptor on the local platform. Different platforms provide different + features, and the underlying functionality Python uses to accept open file + descriptors as *path* arguments is not available on all platforms Python + supports. - To check whether a particular function permits specifying an open file + To determine whether a particular function permits specifying an open file descriptor for its *path* parameter, use the ``in`` operator on - ``supports_fd``. As an example, this expression determines whether - :func:`os.chdir` accepts open file descriptors when called on your local + ``supports_fd``. As an example, this expression evaluates to ``True`` if + :func:`os.chdir` accepts open file descriptors for *path* on your local platform:: os.chdir in os.supports_fd @@ -2672,17 +2726,21 @@ features: .. data:: supports_follow_symlinks - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *follow_symlinks* parameter. Different - platforms provide different functionality, and an option that might work on - one might be unsupported on another. For consistency's sakes, functions that - support *follow_symlinks* always allow specifying the parameter, but will - raise an exception if the functionality is not actually available. - - To check whether a particular function permits use of its *follow_symlinks* - parameter, use the ``in`` operator on ``supports_follow_symlinks``. As an - example, this expression determines whether the *follow_symlinks* parameter - of :func:`os.stat` is locally available:: + A :class:`set` object indicating which functions in the :mod:`os` module + accept ``False`` for their *follow_symlinks* parameter on the local platform. + Different platforms provide different features, and the underlying + functionality Python uses to implement *follow_symlinks* is not available + on all platforms Python supports. For consistency's sake, functions that + may support *follow_symlinks* always allow specifying the parameter, but + will throw an exception if the functionality is used when it's not locally + available. (Specifying ``True`` for *follow_symlinks* is always supported + on all platforms.) + + To check whether a particular function accepts ``False`` for its + *follow_symlinks* parameter, use the ``in`` operator on + ``supports_follow_symlinks``. As an example, this expression evaluates + to ``True`` if you may specify ``follow_symlinks=False`` when calling + :func:`os.stat` on the local platform:: os.stat in os.supports_follow_symlinks @@ -2745,6 +2803,8 @@ features: This function can support :ref:`specifying a file descriptor `. + .. audit-event:: os.truncate path,length os.truncate + .. availability:: Unix, Windows. .. versionadded:: 3.3 @@ -2801,7 +2861,7 @@ features: following symlinks `. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, + Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. .. versionchanged:: 3.6 @@ -2963,6 +3023,51 @@ features: Added support for :class:`bytes` paths. +.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC]) + + Create an anonymous file and return a file descriptor that refers to it. + *flags* must be one of the ``os.MFD_*`` constants available on the system + (or a bitwise ORed combination of them). By default, the new file + descriptor is :ref:`non-inheritable `. + + The name supplied in *name* is used as a filename and will be displayed as + the target of the corresponding symbolic link in the directory + ``/proc/self/fd/``. The displayed name is always prefixed with ``memfd:`` + and serves only for debugging purposes. Names do not affect the behavior of + the file descriptor, and as such multiple files can have the same name + without any side effects. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. + + .. versionadded:: 3.8 + + +.. data:: MFD_CLOEXEC + MFD_ALLOW_SEALING + MFD_HUGETLB + MFD_HUGE_SHIFT + MFD_HUGE_MASK + MFD_HUGE_64KB + MFD_HUGE_512KB + MFD_HUGE_1MB + MFD_HUGE_2MB + MFD_HUGE_8MB + MFD_HUGE_16MB + MFD_HUGE_32MB + MFD_HUGE_256MB + MFD_HUGE_512MB + MFD_HUGE_1GB + MFD_HUGE_2GB + MFD_HUGE_16GB + + These flags can be passed to :func:`memfd_create`. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. The + ``MFD_HUGE*`` flags are only available since Linux 4.14. + + .. versionadded:: 3.8 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3162,7 +3267,7 @@ to be ignored. .. availability:: Unix, Windows. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path* + Added support for specifying *path* as an open file descriptor for :func:`execve`. .. versionchanged:: 3.6 @@ -3568,6 +3673,9 @@ written in Python, such as a mail server's external command delivery program. process. On Windows, the process id will actually be the process handle, so can be used with the :func:`waitpid` function. + Note on VxWorks, this function doesn't return ``-signal`` when the new process is + killed. Instead it raises OSError exception. + The "l" and "v" variants of the :func:`spawn\* ` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the @@ -3701,6 +3809,8 @@ written in Python, such as a mail server's external command delivery program. to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. + .. audit-event:: os.system command os.system + .. availability:: Unix, Windows. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 450e8ff378a3a5..166de8de1f062e 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -976,7 +976,7 @@ call fails (for example because the path doesn't exist). is raised. .. versionadded:: 3.6 - The *strict* argument. + The *strict* argument (pre-3.6 behavior is strict). .. method:: Path.rglob(pattern) @@ -1048,11 +1048,27 @@ call fails (for example because the path doesn't exist). otherwise :exc:`FileExistsError` is raised. -.. method:: Path.unlink() +.. method:: Path.unlink(missing_ok=False) Remove this file or symbolic link. If the path points to a directory, use :func:`Path.rmdir` instead. + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. + + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). + + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. + + +.. method:: Path.link_to(target) + + Create a hard link pointing to a path named *target*. + + .. versionchanged:: 3.8 + .. method:: Path.write_bytes(data) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index c7864e9e3f2225..f26b6a8b553b36 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -181,6 +181,8 @@ access further features, you have to do this yourself: import pdb; pdb.Pdb(skip=['django.*']).set_trace() + .. audit-event:: pdb.Pdb "" pdb.Pdb + .. versionadded:: 3.1 The *skip* argument. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 53eb5d39ef9414..e6025aeaf47663 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -195,34 +195,29 @@ The :mod:`pickle` module provides the following constants: The :mod:`pickle` module provides the following functions to make the pickling process more convenient: -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True) +.. function:: dump(obj, file, protocol=None, \*, fix_imports=True, buffer_callback=None) Write a pickled representation of *obj* to the open :term:`file object` *file*. This is equivalent to ``Pickler(file, protocol).dump(obj)``. - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. - - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. + Arguments *file*, *protocol*, *fix_imports* and *buffer_callback* have + the same meaning as in the :class:`Pickler` constructor. - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. -.. function:: dumps(obj, protocol=None, \*, fix_imports=True) +.. function:: dumps(obj, protocol=None, \*, fix_imports=True, buffer_callback=None) Return the pickled representation of the object as a :class:`bytes` object, instead of writing it to a file. - Arguments *protocol* and *fix_imports* have the same meaning as in - :func:`dump`. + Arguments *protocol*, *fix_imports* and *buffer_callback* have the same + meaning as in the :class:`Pickler` constructor. + + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read a pickled object representation from the open :term:`file object` *file* and return the reconstituted object hierarchy specified therein. @@ -232,24 +227,13 @@ process more convenient: protocol argument is needed. Bytes past the pickled object's representation are ignored. - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened for - binary reading, an :class:`io.BytesIO` object, or any other custom object - that meets this interface. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. + + .. versionchanged:: 3.8 + The *buffers* argument was added. -.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read a pickled object hierarchy from a :class:`bytes` object and return the reconstituted object hierarchy specified therein. @@ -258,16 +242,11 @@ process more convenient: protocol argument is needed. Bytes past the pickled object's representation are ignored. - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. + + .. versionchanged:: 3.8 + The *buffers* argument was added. The :mod:`pickle` module defines three exceptions: @@ -295,10 +274,10 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports two classes, :class:`Pickler` and -:class:`Unpickler`: +The :mod:`pickle` module exports three classes, :class:`Pickler`, +:class:`Unpickler` and :class:`PickleBuffer`: -.. class:: Pickler(file, protocol=None, \*, fix_imports=True) +.. class:: Pickler(file, protocol=None, \*, fix_imports=True, buffer_callback=None) This takes a binary file for writing a pickle data stream. @@ -316,6 +295,20 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. + If *buffer_callback* is None (the default), buffer views are + serialized into *file* as part of the pickle stream. + + If *buffer_callback* is not None, then it can be called any number + of times with a buffer view. If the callback returns a false value + (such as None), the given buffer is :ref:`out-of-band `; + otherwise the buffer is serialized in-band, i.e. inside the pickle stream. + + It is an error if *buffer_callback* is not None and *protocol* is + None or smaller than 5. + + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. + .. method:: dump(obj) Write a pickled representation of *obj* to the open file object given in @@ -356,6 +349,18 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and .. versionadded:: 3.3 + .. method:: reducer_override(self, obj) + + Special reducer that can be defined in :class:`Pickler` subclasses. This + method has priority over any reducer in the :attr:`dispatch_table`. It + should conform to the same interface as a :meth:`__reduce__` method, and + can optionally return ``NotImplemented`` to fallback on + :attr:`dispatch_table`-registered reducers to pickle ``obj``. + + For a detailed example, see :ref:`reducer_override`. + + .. versionadded:: 3.8 + .. attribute:: fast Deprecated. Enable fast mode if set to a true value. The fast mode @@ -367,26 +372,43 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and Use :func:`pickletools.optimize` if you need more compact pickles. -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) This takes a binary file for reading a pickle data stream. The protocol version of the pickle is detected automatically, so no protocol argument is needed. - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object + The argument *file* must have three methods, a read() method that takes an + integer argument, a readinto() method that takes a buffer argument + and a readline() method that requires no arguments, as in the + :class:`io.BufferedIOBase` interface. Thus *file* can be an on-disk file opened for binary reading, an :class:`io.BytesIO` object, or any other custom object that meets this interface. - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + The optional arguments *fix_imports*, *encoding* and *errors* are used + to control compatibility support for pickle stream generated by Python 2. + If *fix_imports* is true, pickle will try to map the old Python 2 names + to the new names used in Python 3. The *encoding* and *errors* tell + pickle how to decode 8-bit string instances pickled by Python 2; + these default to 'ASCII' and 'strict', respectively. The *encoding* can be 'bytes' to read these 8-bit string instances as bytes objects. + Using ``encoding='latin1'`` is required for unpickling NumPy arrays and + instances of :class:`~datetime.datetime`, :class:`~datetime.date` and + :class:`~datetime.time` pickled by Python 2. + + If *buffers* is None (the default), then all data necessary for + deserialization must be contained in the pickle stream. This means + that the *buffer_callback* argument was None when a :class:`Pickler` + was instantiated (or when :func:`dump` or :func:`dumps` was called). + + If *buffers* is not None, it should be an iterable of buffer-enabled + objects that is consumed each time the pickle stream references + an :ref:`out-of-band ` buffer view. Such buffers have been + given in order to the *buffer_callback* of a Pickler object. + + .. versionchanged:: 3.8 + The *buffers* argument was added. .. method:: load() @@ -415,6 +437,35 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and how they can be loaded, potentially reducing security risks. Refer to :ref:`pickle-restrict` for details. + .. audit-event:: pickle.find_class module,name pickle.Unpickler.find_class + +.. class:: PickleBuffer(buffer) + + A wrapper for a buffer representing picklable data. *buffer* must be a + :ref:`buffer-providing ` object, such as a + :term:`bytes-like object` or a N-dimensional array. + + :class:`PickleBuffer` is itself a buffer provider, therefore it is + possible to pass it to other APIs expecting a buffer-providing object, + such as :class:`memoryview`. + + :class:`PickleBuffer` objects can only be serialized using pickle + protocol 5 or higher. They are eligible for + :ref:`out-of-band serialization `. + + .. versionadded:: 3.8 + + .. method:: raw() + + Return a :class:`memoryview` of the memory area underlying this buffer. + The returned object is a one-dimensional, C-contiguous memoryview + with format ``B`` (unsigned bytes). :exc:`BufferError` is raised if + the buffer is neither C- nor Fortran-contiguous. + + .. method:: release() + + Release the underlying buffer exposed by the PickleBuffer object. + .. _pickle-picklable: @@ -598,7 +649,7 @@ or both. module; the pickle module searches the module namespace to determine the object's module. This behaviour is typically useful for singletons. - When a tuple is returned, it must be between two and five items long. + When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their value. The semantics of each item are in order: @@ -629,6 +680,15 @@ or both. value``. This is primarily used for dictionary subclasses, but may be used by other classes as long as they implement :meth:`__setitem__`. + * Optionally, a callable with a ``(obj, state)`` signature. This + callable allows the user to programmatically control the state-updating + behavior of a specific object, instead of using ``obj``'s static + :meth:`__setstate__` method. If not ``None``, this callable will have + priority over ``obj``'s :meth:`__setstate__`. + + .. versionadded:: 3.8 + The optional sixth tuple item, ``(obj, state)``, was added. + .. method:: object.__reduce_ex__(protocol) @@ -782,6 +842,184 @@ A sample usage might be something like this:: >>> new_reader.readline() '3: Goodbye!' +.. _reducer_override: + +Custom Reduction for Types, Functions, and Other Objects +-------------------------------------------------------- + +.. versionadded:: 3.8 + +Sometimes, :attr:`~Pickler.dispatch_table` may not be flexible enough. +In particular we may want to customize pickling based on another criterion +than the object's type, or we may want to customize the pickling of +functions and classes. + +For those cases, it is possible to subclass from the :class:`Pickler` class and +implement a :meth:`~Pickler.reducer_override` method. This method can return an +arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +``NotImplemented`` to fallback to the traditional behavior. + +If both the :attr:`~Pickler.dispatch_table` and +:meth:`~Pickler.reducer_override` are defined, then +:meth:`~Pickler.reducer_override` method takes priority. + +.. Note:: + For performance reasons, :meth:`~Pickler.reducer_override` may not be + called for the following objects: ``None``, ``True``, ``False``, and + exact instances of :class:`int`, :class:`float`, :class:`bytes`, + :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` + and :class:`tuple`. + +Here is a simple example where we allow pickling and reconstructing +a given class:: + + import io + import pickle + + class MyClass: + my_attribute = 1 + + class MyPickler(pickle.Pickler): + def reducer_override(self, obj): + """Custom reducer for MyClass.""" + if getattr(obj, "__name__", None) == "MyClass": + return type, (obj.__name__, obj.__bases__, + {'my_attribute': obj.my_attribute}) + else: + # For any other object, fallback to usual reduction + return NotImplemented + + f = io.BytesIO() + p = MyPickler(f) + p.dump(MyClass) + + del MyClass + + unpickled_class = pickle.loads(f.getvalue()) + + assert isinstance(unpickled_class, type) + assert unpickled_class.__name__ == "MyClass" + assert unpickled_class.my_attribute == 1 + + +.. _pickle-oob: + +Out-of-band Buffers +------------------- + +.. versionadded:: 3.8 + +In some contexts, the :mod:`pickle` module is used to transfer massive amounts +of data. Therefore, it can be important to minimize the number of memory +copies, to preserve performance and resource consumption. However, normal +operation of the :mod:`pickle` module, as it transforms a graph-like structure +of objects into a sequential stream of bytes, intrinsically involves copying +data to and from the pickle stream. + +This constraint can be eschewed if both the *provider* (the implementation +of the object types to be transferred) and the *consumer* (the implementation +of the communications system) support the out-of-band transfer facilities +provided by pickle protocol 5 and higher. + +Provider API +^^^^^^^^^^^^ + +The large data objects to be pickled must implement a :meth:`__reduce_ex__` +method specialized for protocol 5 and higher, which returns a +:class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) +for any large data. + +A :class:`PickleBuffer` object *signals* that the underlying buffer is +eligible for out-of-band data transfer. Those objects remain compatible +with normal usage of the :mod:`pickle` module. However, consumers can also +opt-in to tell :mod:`pickle` that they will handle those buffers by +themselves. + +Consumer API +^^^^^^^^^^^^ + +A communications system can enable custom handling of the :class:`PickleBuffer` +objects generated when serializing an object graph. + +On the sending side, it needs to pass a *buffer_callback* argument to +:class:`Pickler` (or to the :func:`dump` or :func:`dumps` function), which +will be called with each :class:`PickleBuffer` generated while pickling +the object graph. Buffers accumulated by the *buffer_callback* will not +see their data copied into the pickle stream, only a cheap marker will be +inserted. + +On the receiving side, it needs to pass a *buffers* argument to +:class:`Unpickler` (or to the :func:`load` or :func:`loads` function), +which is an iterable of the buffers which were passed to *buffer_callback*. +That iterable should produce buffers in the same order as they were passed +to *buffer_callback*. Those buffers will provide the data expected by the +reconstructors of the objects whose pickling produced the original +:class:`PickleBuffer` objects. + +Between the sending side and the receiving side, the communications system +is free to implement its own transfer mechanism for out-of-band buffers. +Potential optimizations include the use of shared memory or datatype-dependent +compression. + +Example +^^^^^^^ + +Here is a trivial example where we implement a :class:`bytearray` subclass +able to participate in out-of-band buffer pickling:: + + class ZeroCopyByteArray(bytearray): + + def __reduce_ex__(self, protocol): + if protocol >= 5: + return type(self)._reconstruct, (PickleBuffer(self),), None + else: + # PickleBuffer is forbidden with pickle protocols <= 4. + return type(self)._reconstruct, (bytearray(self),) + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + # Get a handle over the original buffer object + obj = m.obj + if type(obj) is cls: + # Original buffer object is a ZeroCopyByteArray, return it + # as-is. + return obj + else: + return cls(obj) + +The reconstructor (the ``_reconstruct`` class method) returns the buffer's +providing object if it has the right type. This is an easy way to simulate +zero-copy behaviour on this toy example. + +On the consumer side, we can pickle those objects the usual way, which +when unserialized will give us a copy of the original object:: + + b = ZeroCopyByteArray(b"abc") + data = pickle.dumps(b, protocol=5) + new_b = pickle.loads(data) + print(b == new_b) # True + print(b is new_b) # False: a copy was made + +But if we pass a *buffer_callback* and then give back the accumulated +buffers when unserializing, we are able to get back the original object:: + + b = ZeroCopyByteArray(b"abc") + buffers = [] + data = pickle.dumps(b, protocol=5, buffer_callback=buffers.append) + new_b = pickle.loads(data, buffers=buffers) + print(b == new_b) # True + print(b is new_b) # True: no copy was made + +This example is limited by the fact that :class:`bytearray` allocates its +own memory: you cannot create a :class:`bytearray` instance that is backed +by another object's memory. However, third-party datatypes such as NumPy +arrays do not have this limitation, and allow use of zero-copy pickling +(or making as few copies as possible) when transferring between distinct +processes or systems. + +.. seealso:: :pep:`574` -- Pickle protocol 5 with out-of-band data + .. _pickle-restrict: diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 8bd6b63a8ee50b..d84fcac0ef23d4 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -36,6 +36,10 @@ or :class:`datetime.datetime` objects. .. versionchanged:: 3.4 New API, old API deprecated. Support for binary format plists added. +.. versionchanged:: 3.8 + Support added for reading and writing :class:`UID` tokens in binary plists as used + by NSKeyedArchiver and NSKeyedUnarchiver. + .. seealso:: `PList manual page `_ @@ -179,6 +183,16 @@ The following classes are available: .. deprecated:: 3.4 Use a :class:`bytes` object instead. +.. class:: UID(data) + + Wraps an :class:`int`. This is used when reading or writing NSKeyedArchiver + encoded data, which contains UID (see PList manual). + + It has one attribute, :attr:`data` which can be used to retrieve the int value + of the UID. :attr:`data` must be in the range `0 <= data <= 2**64`. + + .. versionadded:: 3.8 + The following constants are available: diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index d72b660d6ea9f6..28b42fa60c1859 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -39,6 +39,14 @@ The :mod:`poplib` module provides two classes: connection attempt (if not specified, the global default timeout setting will be used). + .. audit-event:: poplib.connect self,host,port poplib.POP3 + + .. audit-event:: poplib.putline self,line poplib.POP3 + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) @@ -54,6 +62,14 @@ The :mod:`poplib` module provides two classes: point to PEM-formatted private key and certificate chain files, respectively, for the SSL connection. + .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL + + .. audit-event:: poplib.putline self,line popplib.POP3_SSL + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.2 *context* parameter added. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 9abf2865ef6143..207c3f8bca928b 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -25,6 +25,9 @@ width constraint. Dictionaries are sorted by key before the display is computed. +.. versionchanged:: 3.9 + Added support for pretty-printing :class:`types.SimpleNamespace`. + The :mod:`pprint` module defines one class: .. First the implementation class: diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index d8039fd28e0d8a..8d589d247b7747 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -309,7 +309,7 @@ functions: Profile the cmd via :func:`exec` with the specified global and local environment. - .. method:: runcall(func, *args, **kwargs) + .. method:: runcall(func, /, *args, **kwargs) Profile ``func(*args, **kwargs)`` diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 0ab766065d6e81..12268437d07e98 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -43,11 +43,32 @@ The :mod:`pty` module defines the following functions: Spawn a process, and connect its controlling terminal with the current process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. + reading from the controlling terminal. It is expected that the process + spawned behind the pty will eventually terminate, and when it does *spawn* + will return. + + The functions *master_read* and *stdin_read* are passed a file descriptor + which they should read from, and they should always return a byte string. In + order to force spawn to return before the child process exits an + :exc:`OSError` should be thrown. + + The default implementation for both functions will read and return up to 1024 + bytes each time the function is called. The *master_read* callback is passed + the pseudoterminal’s master file descriptor to read output from the child + process, and *stdin_read* is passed file descriptor 0, to read from the + parent process's standard input. + + Returning an empty byte string from either callback is interpreted as an + end-of-file (EOF) condition, and that callback will not be called after + that. If *stdin_read* signals EOF the controlling terminal can no longer + communicate with the parent process OR the child process. Unless the child + process will quit without any input, *spawn* will then loop forever. If + *master_read* signals EOF the same behavior results (on linux at least). + + If both callbacks signal EOF then *spawn* will probably never return, unless + *select* throws an error on your platform when passed three empty lists. This + is a bug, documented in `issue 26228 `_. - The functions *master_read* and *stdin_read* should be functions which read from - a file descriptor. The defaults try to read 1024 bytes each time they are - called. .. versionchanged:: 3.4 :func:`spawn` now returns the status value from :func:`os.waitpid` diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 8cb5a4d546c874..3824353abda1f8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -42,6 +42,13 @@ byte-code cache files in the directory containing the source code. is raised. This function returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + The *doraise* and *quiet* arguments determine how errors are handled while + compiling file. If *quiet* is 0 or 1, and *doraise* is false, the default + behaviour is enabled: an error string is written to ``sys.stderr``, and the + function returns ``None`` instead of a path. If *doraise* is true, + a :exc:`PyCompileError` is raised instead. However if *quiet* is 2, + no message is written, and *doraise* has no effect. + If the path that *cfile* becomes (either explicitly specified or computed) is a symlink or non-regular file, :exc:`FileExistsError` will be raised. This is to act as a warning that import will turn those paths into regular @@ -82,6 +89,9 @@ byte-code cache files in the directory containing the source code. overrides the value of the *invalidation_mode* argument, and determines its default value instead. + .. versionchanged:: 3.8 + The *quiet* parameter was added. + .. class:: PycInvalidationMode diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index a70c8df6a7b119..b80a2faed9b424 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -44,7 +44,7 @@ modules. .. versionadded:: 3.7 Descriptors for nested definitions. They are accessed through the - new children attibute. Each has a new parent attribute. + new children attribute. Each has a new parent attribute. The descriptors returned by these functions are instances of Function and Class classes. Users are not expected to create instances diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 5ef72b535ce8d4..158248c3d14794 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -942,7 +942,7 @@ form. >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) /|\-|\+|\*\*|\* - This functions must not be used for the replacement string in :func:`sub` + This function must not be used for the replacement string in :func:`sub` and :func:`subn`, only backslashes should be escaped. For example:: >>> digits_re = r'\d+' diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 16c28cf7d02fb9..eae0a6df45f30e 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -19,7 +19,7 @@ function. Readline keybindings may be configured via an initialization file, typically ``.inputrc`` in your home directory. See `Readline Init File -`_ +`_ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index ad96dbc95b0cd2..fab16f52c46218 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -20,8 +20,7 @@ scheduler: The :class:`scheduler` class defines a generic interface to scheduling events. It needs two functions to actually deal with the "outside world" --- *timefunc* should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one + units whatsoever). The *delayfunc* function should be callable with one argument, compatible with the output of *timefunc*, and should delay that many time units. *delayfunc* will also be called with the argument ``0`` after each event is run to allow other threads an opportunity to run in multi-threaded diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 733a91e20b684d..8f5a2cea9257cc 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -480,13 +480,14 @@ Kqueue Objects Create a kqueue object from a given file descriptor. -.. method:: kqueue.control(changelist, max_events[, timeout=None]) -> eventlist +.. method:: kqueue.control(changelist, max_events[, timeout]) -> eventlist Low level interface to kevent - - changelist must be an iterable of kevent object or ``None`` + - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible) + - timeout in seconds (floats possible); the default is ``None``, + to wait forever .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index fb335c69006816..a8421fdb7008ce 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -37,6 +37,21 @@ The :mod:`shlex` module defines the following functions: standard input. +.. function:: join(split_command) + + Concatenate the tokens of the list *split_command* and return a string. + This function is the inverse of :func:`split`. + + >>> from shlex import join + >>> print(join(['echo', '-n', 'Multiple words'])) + echo -n 'Multiple words' + + The returned value is shell-escaped to protect against injection + vulnerabilities (see :func:`quote`). + + .. versionadded:: 3.8 + + .. function:: quote(s) Return a shell-escaped version of the string *s*. The returned value is a @@ -210,7 +225,8 @@ variables which either control lexical analysis or can be used for debugging: appear in filename specifications and command line parameters, will also be included in this attribute, and any characters which appear in ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. + there. If :attr:`whitespace_split` is set to ``True``, this will have no + effect. .. attribute:: shlex.whitespace @@ -243,11 +259,13 @@ variables which either control lexical analysis or can be used for debugging: If ``True``, tokens will only be split in whitespaces. This is useful, for example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. If this attribute is ``True``, - :attr:`punctuation_chars` will have no effect, and splitting will happen - only on whitespaces. When using :attr:`punctuation_chars`, which is - intended to provide parsing closer to that implemented by shells, it is - advisable to leave ``whitespace_split`` as ``False`` (the default value). + tokens in a similar way to shell arguments. When used in combination with + :attr:`punctuation_chars`, tokens will be split on whitespace in addition to + those characters. + + .. versionchanged:: 3.8 + The :attr:`punctuation_chars` attribute was made compatible with the + :attr:`whitespace_split` attribute. .. attribute:: shlex.infile @@ -383,12 +401,15 @@ otherwise. To illustrate, you can see the difference in the following snippet: >>> import shlex >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> list(shlex.shlex(text)) - ['a', '&', '&', 'b', ';', 'c', '&', '&', 'd', '|', '|', 'e', ';', 'f', '>', - "'abc'", ';', '(', 'def', '"ghi"', ')'] - >>> list(shlex.shlex(text, punctuation_chars=True)) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', "'abc'", - ';', '(', 'def', '"ghi"', ')'] + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -413,6 +434,11 @@ which characters constitute punctuation. For example:: >>> list(s) ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] + However, to match the shell as closely as possible, it is recommended to + always use ``posix`` and :attr:`~shlex.whitespace_split` when using + :attr:`~shlex.punctuation_chars`, which will negate + :attr:`~shlex.wordchars` entirely. + For best effect, ``punctuation_chars`` should be set in conjunction with ``posix=True``. (Note that ``posix=False`` is the default for :class:`~shlex.shlex`.) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2dc872fd077757..eaeee8df81b8e6 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -54,7 +54,7 @@ Directory and files operations *dst* and return *dst* in the most efficient way possible. *src* and *dst* are path names given as strings. - *dst* must be the complete target file name; look at :func:`shutil.copy` + *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* specify the same file, :exc:`SameFileError` is raised. @@ -218,7 +218,7 @@ Directory and files operations already exists. Permissions and times of directories are copied with :func:`copystat`, - individual files are copied using :func:`shutil.copy2`. + individual files are copied using :func:`~shutil.copy2`. If *symlinks* is true, symbolic links in the source tree are represented as symbolic links in the new tree and the metadata of the original links will @@ -246,8 +246,10 @@ Directory and files operations If *copy_function* is given, it must be a callable that will be used to copy each file. It will be called with the source path and the destination path - as arguments. By default, :func:`shutil.copy2` is used, but any function - that supports the same signature (like :func:`shutil.copy`) can be used. +   as arguments. By default, :func:`~shutil.copy2` is used, but any function +   that supports the same signature (like :func:`~shutil.copy`) can be used. + + .. audit-event:: shutil.copytree src,dst shutil.copytree .. versionchanged:: 3.3 Copy metadata when *symlinks* is false. @@ -296,6 +298,8 @@ Directory and files operations *excinfo*, will be the exception information returned by :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. + .. audit-event:: shutil.rmtree path shutil.rmtree + .. versionchanged:: 3.3 Added a symlink attack resistant version that is used automatically if platform supports fd-based functions. @@ -420,8 +424,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports -copies between 2 regular file descriptors :func:`os.sendfile` is used. +On Linux :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of @@ -559,6 +562,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The *verbose* argument is unused and deprecated. + .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive + .. versionchanged:: 3.8 The modern pax (POSIX.1-2001) format is now used instead of the legacy GNU format for archives created with ``format="tar"``. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index ac6cad9aff8e97..01200b4df8803e 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -16,7 +16,8 @@ The :func:`signal.signal` function allows defining custom handlers to be executed when a signal is received. A small number of default handlers are installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception. +translated into a :exc:`KeyboardInterrupt` exception if the parent process +has not changed it. A handler for a particular signal, once set, remains installed until it is explicitly reset (Python emulates the BSD style interface regardless of the diff --git a/Doc/library/site.rst b/Doc/library/site.rst index dfc40d17944332..9e4d402c1f5e70 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -8,7 +8,7 @@ -------------- -.. highlightlang:: none +.. highlight:: none **This module is automatically imported during initialization.** The automatic import can be suppressed using the interpreter's :option:`-S` option. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 2c3a5f0c6f7296..6176c35a0e4a50 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -55,6 +55,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). (250, b'Ok') >>> + .. audit-event:: smtplib.send self,data smtplib.SMTP + + All commands will raise an :ref:`auditing event ` + ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, + where ``data`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -242,6 +248,8 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect + .. method:: SMTP.helo(name='') diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 379633a3b6052b..e3ddebdcbc0c26 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -200,6 +200,23 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.8 +- :const:`IPPROTO_UDPLITE` is a variant of UDP which allows you to specify + what portion of a packet is covered with the checksum. It adds two socket + options that you can change. + ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_SEND_CSCOV, length)`` will + change what portion of outgoing packets are covered by the checksum and + ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_RECV_CSCOV, length)`` will + filter out packets which cover too little of their data. In both cases + ``length`` should be in ``range(8, 2**16, 8)``. + + Such a socket should be constructed with + ``socket(AF_INET, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv4 or + ``socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv6. + + .. availability:: Linux >= 2.6.20, FreeBSD >= 10.1-RELEASE + + .. versionadded:: 3.9 + If you use a hostname in the *host* portion of IPv4/v6 socket address, the program may show a nondeterministic behavior, as Python uses the first address returned from the DNS resolution. The socket address will be resolved @@ -380,7 +397,7 @@ Constants Enables CAN FD support in a CAN_RAW socket. This is disabled by default. This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. + you must accept both CAN and CAN FD frames when reading from the socket. This constant is documented in the Linux documentation. @@ -526,6 +543,8 @@ The following functions all create :ref:`socket objects `. The newly created socket is :ref:`non-inheritable `. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket + .. versionchanged:: 3.3 The AF_CAN family was added. The AF_RDS family was added. @@ -718,6 +737,8 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo + The following example fetches address information for a hypothetical TCP connection to ``example.org`` on port 80 (results may differ on your system if IPv6 isn't enabled):: @@ -753,6 +774,8 @@ The :mod:`socket` module also offers various network-related services: interface. :func:`gethostbyname` does not support IPv6 name resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname + .. function:: gethostbyname_ex(hostname) @@ -765,12 +788,16 @@ The :mod:`socket` module also offers various network-related services: resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname_ex + .. function:: gethostname() Return a string containing the hostname of the machine where the Python interpreter is currently executing. + .. audit-event:: socket.gethostname "" socket.gethostname + Note: :func:`gethostname` doesn't always return the fully qualified domain name; use :func:`getfqdn` for that. @@ -785,6 +812,8 @@ The :mod:`socket` module also offers various network-related services: domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports both IPv4 and IPv6. + .. audit-event:: socket.gethostbyaddr ip_address socket.gethostbyaddr + .. function:: getnameinfo(sockaddr, flags) @@ -798,6 +827,8 @@ The :mod:`socket` module also offers various network-related services: For more information about *flags* you can consult :manpage:`getnameinfo(3)`. + .. audit-event:: socket.getnameinfo sockaddr socket.getnameinfo + .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant @@ -813,6 +844,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyname servicename,protocolname socket.getservbyname + .. function:: getservbyport(port[, protocolname]) @@ -820,6 +853,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyport port,protocolname socket.getservbyport + .. function:: ntohl(x) @@ -1003,6 +1038,8 @@ The :mod:`socket` module also offers various network-related services: Set the machine's hostname to *name*. This will raise an :exc:`OSError` if you don't have enough rights. + .. audit-event:: socket.sethostname name socket.sethostname + .. availability:: Unix. .. versionadded:: 3.3 @@ -1014,10 +1051,13 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_nametoindex(if_name) @@ -1025,10 +1065,13 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_indextoname(if_index) @@ -1036,10 +1079,13 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. _socket-objects: @@ -1078,6 +1124,7 @@ to sockets. Bind the socket to *address*. The socket must not already be bound. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.bind self,address socket.socket.bind .. method:: socket.close() @@ -1115,6 +1162,8 @@ to sockets. :exc:`InterruptedError` exception if the connection is interrupted by a signal (or the exception raised by the signal handler). + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 The method now waits until the connection completes instead of raising an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -1131,6 +1180,7 @@ to sockets. :c:data:`errno` variable. This is useful to support, for example, asynchronous connects. + .. audit-event:: socket.connect self,address socket.socket.connect_ex .. method:: socket.detach() @@ -1472,6 +1522,8 @@ to sockets. bytes sent. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.sendto self,address socket.socket.sendto + .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an exception, the method now retries the system call instead of raising @@ -1511,6 +1563,8 @@ to sockets. .. availability:: most Unix platforms, possibly others. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionadded:: 3.3 .. versionchanged:: 3.5 diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 37087ac5af492a..67ea2b1d776638 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -224,6 +224,8 @@ Module functions and constants More information about this feature, including a list of recognized options, can be found in the `SQLite URI documentation `_. + .. audit-event:: sqlite3.connect database sqlite3.connect + .. versionchanged:: 3.4 Added the *uri* parameter. @@ -537,6 +539,7 @@ Connection Objects with open('dump.sql', 'w') as f: for line in con.iterdump(): f.write('%s\n' % line) + con.close() .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) @@ -573,8 +576,11 @@ Connection Objects print(f'Copied {total-remaining} of {total} pages...') con = sqlite3.connect('existing_db.db') - with sqlite3.connect('backup.db') as bck: + bck = sqlite3.connect('backup.db') + with bck: con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() Example 2, copy an existing database into a transient copy:: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 6a441983f8884e..279af5728913d9 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -139,6 +139,10 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. + When :attr:`~SSLContext.keylog_filename` is supported and the environment + variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` + enables key logging. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +176,10 @@ purposes. 3DES was dropped from the default cipher string. + .. versionchanged:: 3.8 + + Support for key logging to :envvar:`SSLKEYLOGFILE` was added. + Exceptions ^^^^^^^^^^ @@ -665,7 +673,7 @@ Constants .. data:: PROTOCOL_SSLv23 - Alias for data:`PROTOCOL_TLS`. + Alias for :data:`PROTOCOL_TLS`. .. deprecated:: 3.6 @@ -1056,6 +1064,7 @@ Constants SSL 3.0 to TLS 1.3. + SSL Sockets ----------- @@ -1821,7 +1830,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.sslsocket_class - The return type of :meth:`SSLContext.wrap_sockets`, defaults to + The return type of :meth:`SSLContext.wrap_socket`, defaults to :class:`SSLSocket`. The attribute can be overridden on instance of class in order to return a custom subclass of :class:`SSLSocket`. @@ -1831,7 +1840,7 @@ to speed up repeated connections from the same clients. server_hostname=None, session=None) Wrap the BIO objects *incoming* and *outgoing* and return an instance of - attr:`SSLContext.sslobject_class` (default :class:`SSLObject`). The SSL + :attr:`SSLContext.sslobject_class` (default :class:`SSLObject`). The SSL routines will read input data from the incoming BIO and write data to the outgoing BIO. @@ -1901,6 +1910,20 @@ to speed up repeated connections from the same clients. This features requires OpenSSL 0.9.8f or newer. +.. attribute:: SSLContext.keylog_filename + + Write TLS keys to a keylog file, whenever key material is generated or + received. The keylog file is designed for debugging purposes only. The + file format is specified by NSS and used by many traffic analyzers such + as Wireshark. The log file is opened in append-only mode. Writes are + synchronized between threads, but not between processes. + + .. versionadded:: 3.8 + + .. note:: + + This features requires OpenSSL 1.1.1 or newer. + .. attribute:: SSLContext.maximum_version A :class:`TLSVersion` enum member representing the highest supported @@ -1936,6 +1959,19 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 +.. attribute:: SSLContext.num_tickets + + Control the number of TLS 1.3 session tickets of a + :attr:`TLS_PROTOCOL_SERVER` context. The setting has no impact on TLS + 1.0 to 1.2 connections. + + .. note:: + + This attribute is not available unless the ssl module is compiled + with OpenSSL 1.1.1 or newer. + + .. versionadded:: 3.8 + .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index fb7df4e7188a07..bc841fda72f887 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -511,22 +511,33 @@ However, for reading convenience, most of the examples show sorted sequences. is not least 1. The *dist* can be any iterable containing sample data or it can be an - instance of a class that defines an :meth:`~inv_cdf` method. + instance of a class that defines an :meth:`~inv_cdf` method. For meaningful + results, the number of data points in *dist* should be larger than *n*. Raises :exc:`StatisticsError` if there are not at least two data points. For sample data, the cut points are linearly interpolated from the two nearest data points. For example, if a cut point falls one-third of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. Other selection methods may be - offered in the future (for example choose ``100`` as the nearest - value or compute ``106`` as the midpoint). This might matter if - there are too few samples for a given number of cut points. - - If *method* is set to *inclusive*, *dist* is treated as population data. - The minimum value is treated as the 0th percentile and the maximum - value is treated as the 100th percentile. If *dist* is an instance of - a class that defines an :meth:`~inv_cdf` method, setting *method* - has no effect. + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the data in *dist* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* data points is computed as ``i / (m + 1)``. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that include the extreme points. The minimum + value in *dist* is treated as the 0th percentile and the maximum + value is treated as the 100th percentile. The portion of the + population falling below the *i-th* of *m* data points is computed as + ``(i - 1) / (m - 1)``. + + If *dist* is an instance of a class that defines an + :meth:`~inv_cdf` method, setting *method* has no effect. .. doctest:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0a6bb149075f47..9dd557fabaae2e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -351,7 +351,7 @@ Notes: The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See http://www.unicode.org/Public/12.0.0/ucd/extracted/DerivedNumericType.txt + See http://www.unicode.org/Public/12.1.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. @@ -1114,7 +1114,7 @@ Notes: item is removed and returned. (3) - ``remove`` raises :exc:`ValueError` when *x* is not found in *s*. + :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. (4) The :meth:`reverse` method modifies the sequence in place for economy of @@ -1124,7 +1124,9 @@ Notes: (5) :meth:`clear` and :meth:`!copy` are included for consistency with the interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`) + (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the + :class:`collections.abc.MutableSequence` ABC, but most concrete + mutable sequence classes provide it. .. versionadded:: 3.3 :meth:`clear` and :meth:`!copy` methods. @@ -1557,9 +1559,16 @@ expression support in the :mod:`re` module). :func:`codecs.register_error`, see section :ref:`error-handlers`. For a list of possible encodings, see section :ref:`standard-encodings`. + By default, the *errors* argument is not checked for best performances, but + only used at the first encoding error. Enable the development mode + (:option:`-X` ``dev`` option), or use a debug build, to check *errors*. + .. versionchanged:: 3.1 Support for keyword arguments added. + .. versionchanged:: 3.9 + The *errors* is now checked in development mode and in debug mode. + .. method:: str.endswith(suffix[, start[, end]]) @@ -2402,8 +2411,26 @@ data and are closely related to string objects in a variety of other ways. >>> b'\xf0\xf1\xf2'.hex() 'f0f1f2' + If you want to make the hex string easier to read, you can specify a + single character separator *sep* parameter to include in the output. + By default between each byte. A second optional *bytes_per_sep* + parameter controls the spacing. Positive values calculate the + separator position from the right, negative values from the left. + + >>> value = b'\xf0\xf1\xf2' + >>> value.hex('-') + 'f0-f1-f2' + >>> value.hex('_', 2) + 'f0_f1f2' + >>> b'UUDDLRLRAB'.hex(' ', -4) + '55554444 4c524c52 4142' + .. versionadded:: 3.5 + .. versionchanged:: 3.8 + :meth:`bytes.hex` now supports optional *sep* and *bytes_per_sep* + parameters to insert separators between bytes in the hex output. + Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes object of length 1. (This contrasts with text strings, where both indexing @@ -2555,6 +2582,10 @@ arbitrary binary data. :func:`codecs.register_error`, see section :ref:`error-handlers`. For a list of possible encodings, see section :ref:`standard-encodings`. + By default, the *errors* argument is not checked for best performances, but + only used at the first decoding error. Enable the development mode + (:option:`-X` ``dev`` option), or use a debug build, to check *errors*. + .. note:: Passing the *encoding* argument to :class:`str` allows decoding any @@ -2564,6 +2595,9 @@ arbitrary binary data. .. versionchanged:: 3.1 Added support for keyword arguments. + .. versionchanged:: 3.9 + The *errors* is now checked in development mode and in debug mode. + .. method:: bytes.endswith(suffix[, start[, end]]) bytearray.endswith(suffix[, start[, end]]) @@ -2716,8 +2750,8 @@ arbitrary binary data. The prefix(es) to search for may be any :term:`bytes-like object`. -.. method:: bytes.translate(table, delete=b'') - bytearray.translate(table, delete=b'') +.. method:: bytes.translate(table, /, delete=b'') + bytearray.translate(table, /, delete=b'') Return a copy of the bytes or bytearray object where all bytes occurring in the optional argument *delete* are removed, and the remaining bytes have @@ -3781,7 +3815,7 @@ copying. >>> z.nbytes 48 - Cast 1D/unsigned char to 2D/unsigned long:: + Cast 1D/unsigned long to 2D/unsigned long:: >>> buf = struct.pack("L"*6, *list(range(6))) >>> x = memoryview(buf) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c2f65224bc8daf..af8b9b358cc388 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -88,7 +88,7 @@ implementation as the built-in :meth:`~str.format` method. The :class:`Formatter` class has the following public methods: - .. method:: format(format_string, *args, **kwargs) + .. method:: format(format_string, /, *args, **kwargs) The primary API method. It takes a format string and an arbitrary set of positional and keyword arguments. @@ -146,7 +146,7 @@ implementation as the built-in :meth:`~str.format` method. keyword arguments. For compound field names, these functions are only called for the first - component of the field name; Subsequent components are handled through + component of the field name; subsequent components are handled through normal attribute and indexing operations. So for example, the field expression '0.name' would cause @@ -720,7 +720,7 @@ these rules. The methods of :class:`Template` are: The constructor takes a single argument which is the template string. - .. method:: substitute(mapping, **kwds) + .. method:: substitute(mapping={}, /, **kwds) Performs the template substitution, returning a new string. *mapping* is any dictionary-like object with keys that match the placeholders in the @@ -729,7 +729,7 @@ these rules. The methods of :class:`Template` are: and there are duplicates, the placeholders from *kwds* take precedence. - .. method:: safe_substitute(mapping, **kwds) + .. method:: safe_substitute(mapping={}, /, **kwds) Like :meth:`substitute`, except that if placeholders are missing from *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 1a0fd73c6758c7..a06d90344ba34e 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -70,7 +70,7 @@ The module defines the following exception and functions: size required by the format, as reflected by :func:`calcsize`. -.. function:: unpack_from(format, buffer, offset=0) +.. function:: unpack_from(format, /, buffer, offset=0) Unpack from *buffer* starting at position *offset*, according to the format string *format*. The result is a tuple even if it contains exactly one diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 3280c95cacbbc3..af338795dae340 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -55,7 +55,9 @@ compatibility with older versions, see the :ref:`call-function-trio` section. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. + not be supplied at the same time as *capture_output*. If you wish to capture + and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` + instead of *capture_output*. The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout expires, the child process will be killed and waited for. The @@ -345,7 +347,8 @@ functions. the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. - *args* should be a sequence of program arguments or else a single string. + *args* should be a sequence of program arguments or else a single string + or :term:`path-like object`. By default, the program to execute is the first item in *args* if *args* is a sequence. If *args* is a string, the interpretation is platform-dependent and described below. See the *shell* and *executable* @@ -379,6 +382,15 @@ functions. manner described in :ref:`converting-argument-sequence`. This is because the underlying ``CreateProcess()`` operates on strings. + .. versionchanged:: 3.6 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing path-like objects on POSIX. + + .. versionchanged:: 3.8 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing bytes and path-like objects + on Windows. + The *shell* argument (which defaults to ``False``) specifies whether to use the shell as the program to execute. If *shell* is ``True``, it is recommended to pass *args* as a string rather than as a sequence. @@ -434,6 +446,13 @@ functions. :program:`ps`. If ``shell=True``, on POSIX the *executable* argument specifies a replacement shell for the default :file:`/bin/sh`. + .. versionchanged:: 3.6 + *executable* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.8 + *executable* parameter accepts a bytes and :term:`path-like object` + on Windows. + *stdin*, *stdout* and *stderr* specify the executed program's standard input, standard output and standard error file handles, respectively. Valid values are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive @@ -490,13 +509,19 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. *cwd* can be a :class:`str` and + *cwd* before executing the child. *cwd* can be a string, bytes or :term:`path-like ` object. In particular, the function looks for *executable* (or for the first item in *args*) relative to *cwd* if the executable path is a relative path. .. versionchanged:: 3.6 - *cwd* parameter accepts a :term:`path-like object`. + *cwd* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.7 + *cwd* parameter accepts a :term:`path-like object` on Windows. + + .. versionchanged:: 3.8 + *cwd* parameter accepts a bytes object on Windows. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. @@ -560,6 +585,13 @@ functions. with Popen(["ifconfig"], stdout=PIPE) as proc: log.write(proc.stdout.read()) + .. audit-event:: subprocess.Popen executable,args,cwd,env subprocess.Popen + + Popen and the other functions in this module that use it raise an + :ref:`auditing event ` ``subprocess.Popen`` with arguments + ``executable``, ``args``, ``cwd``, ``env``. The value for ``args`` + may be a single string or a list of strings, depending on platform. + .. versionchanged:: 3.2 Added context manager support. diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 2064fd7e20bfc2..aad6f93b6bff18 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -59,13 +59,6 @@ The :mod:`sunau` module defines the following functions: or ``'wb'`` returns an :class:`AU_write` object. -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - .. deprecated-removed:: 3.7 3.9 - - The :mod:`sunau` module defines the following exception: .. exception:: Error diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 591972e9b783a0..acd54421a370aa 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -16,9 +16,37 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionchanged:: 3.8 + Default flags became an empty string (``m`` flag for pymalloc has been + removed). + .. versionadded:: 3.2 +.. function:: addaudithook(hook) + + Adds the callable *hook* to the collection of active auditing hooks for the + current interpreter. + + When an auditing event is raised through the :func:`sys.audit` function, each + hook will be called in the order it was added with the event name and the + tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are + called first, followed by hooks added in the current interpreter. + + Calling this function will trigger an event for all existing hooks, and if + any raise an exception derived from :class:`Exception`, the add will be + silently ignored. As a result, callers cannot assume that their hook has been + added unless they control all existing hooks. + + .. versionadded:: 3.8 + + .. impl-detail:: + + When tracing is enabled, Python hooks are only traced if the callable has + a ``__cantrace__`` member that is set to a true value. Otherwise, trace + functions will not see the hook. + + .. data:: argv The list of command line arguments passed to a Python script. ``argv[0]`` is the @@ -37,6 +65,33 @@ always available. ``[os.fsencode(arg) for arg in sys.argv]``. +.. _auditing: + +.. function:: audit(event, *args) + + .. index:: single: auditing + + Raises an auditing event with any active hooks. The event name is a string + identifying the event and its associated schema, which is the number and + types of arguments. The schema for a given event is considered public and + stable API and should not be modified between releases. + + This function will raise the first exception raised by any hook. In general, + these errors should not be handled and should terminate the process as + quickly as possible. + + Hooks are added using the :func:`sys.addaudithook` or + :c:func:`PySys_AddAuditHook` functions. + + The native equivalent of this function is :c:func:`PySys_Audit`. Using the + native function is preferred when possible. + + See the :ref:`audit events table ` for all events raised by + ``CPython``. + + .. versionadded:: 3.8 + + .. data:: base_exec_prefix Set during Python startup, before ``site.py`` is run, to the same value as @@ -114,6 +169,8 @@ always available. This function should be used for internal and specialized purposes only. + .. audit-event:: sys._current_frames "" sys._current_frames + .. function:: breakpointhook() @@ -248,16 +305,23 @@ always available. before the program exits. The handling of such top-level exceptions can be customized by assigning another three-argument function to ``sys.excepthook``. + .. seealso:: + + The :func:`sys.unraisablehook` function handles unraisable exceptions + and the :func:`threading.excepthook` function handles exception raised + by :func:`threading.Thread.run`. + .. data:: __breakpointhook__ __displayhook__ __excepthook__ + __unraisablehook__ These objects contain the original values of ``breakpointhook``, - ``displayhook``, and ``excepthook`` at the start of the program. They are - saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be - restored in case they happen to get replaced with broken or alternative - objects. + ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the + program. They are saved so that ``breakpointhook``, ``displayhook`` and + ``excepthook``, ``unraisablehook`` can be restored in case they happen to + get replaced with broken or alternative objects. .. versionadded:: 3.7 __breakpointhook__ @@ -490,14 +554,6 @@ always available. .. versionadded:: 3.7 -.. function:: getcheckinterval() - - Return the interpreter's "check interval"; see :func:`setcheckinterval`. - - .. deprecated:: 3.2 - Use :func:`getswitchinterval` instead. - - .. function:: getdefaultencoding() Return the name of the current default string encoding used by the Unicode @@ -614,6 +670,8 @@ always available. that is deeper than the call stack, :exc:`ValueError` is raised. The default for *depth* is zero, returning the frame at the top of the call stack. + .. audit-event:: sys._getframe "" sys._getframe + .. impl-detail:: This function should be used for internal and specialized purposes only. @@ -722,22 +780,6 @@ always available. for details.) Use it only for debugging purposes. -.. function:: get_coroutine_wrapper() - - Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - - .. data:: hash_info A :term:`struct sequence` giving parameters of the numeric hash @@ -863,6 +905,12 @@ always available. read, so that you can set this hook there. The :mod:`site` module :ref:`sets this `. + .. audit-event:: cpython.run_interactivehook hook sys.__interactivehook__ + + Raises an :ref:`auditing event ` + ``cpython.run_interactivehook`` with the hook object as the argument when + the hook is called on startup. + .. versionadded:: 3.4 @@ -1094,21 +1142,6 @@ always available. implement a dynamic prompt. -.. function:: setcheckinterval(interval) - - Set the interpreter's "check interval". This integer value determines how often - the interpreter checks for periodic things such as thread switches and signal - handlers. The default is ``100``, meaning the check is performed every 100 - Python virtual instructions. Setting it to a larger value may increase - performance for programs using threads. Setting it to a value ``<=`` 0 checks - every virtual instruction, maximizing responsiveness as well as overhead. - - .. deprecated:: 3.2 - This function doesn't have an effect anymore, as the internal logic for - thread switching and asynchronous tasks has been rewritten. Use - :func:`setswitchinterval` instead. - - .. function:: setdlopenflags(n) Set the flags used by the interpreter for :c:func:`dlopen` calls, such as when @@ -1143,6 +1176,8 @@ always available. ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. + .. audit-event:: sys.setprofile "" sys.setprofile + The events have the following meaning: ``'call'`` @@ -1204,8 +1239,8 @@ always available. Set the system's trace function, which allows you to implement a Python source code debugger in Python. The function is thread-specific; for a - debugger to support multiple threads, it must be registered using - :func:`settrace` for each thread being debugged. + debugger to support multiple threads, it must register a trace function using + :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, @@ -1263,6 +1298,8 @@ always available. For more information on code and frame objects, refer to :ref:`types`. + .. audit-event:: sys.settrace "" sys.settrace + .. impl-detail:: The :func:`settrace` function is intended only for implementing debuggers, @@ -1283,6 +1320,13 @@ always available. first time. The *finalizer* will be called when an asynchronous generator is about to be garbage collected. + .. audit-event:: sys.set_asyncgen_hooks_firstiter "" sys.set_asyncgen_hooks + + .. audit-event:: sys.set_asyncgen_hooks_finalizer "" sys.set_asyncgen_hooks + + Two auditing events are raised because the underlying API consists of two + calls, each of which must raise its own event. + .. versionadded:: 3.6 See :pep:`525` for more details, and for a reference example of a *finalizer* method see the implementation of @@ -1314,49 +1358,6 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) Use it only for debugging purposes. -.. function:: set_coroutine_wrapper(wrapper) - - Allows intercepting creation of :term:`coroutine` objects (only ones that - are created by an :keyword:`async def` function; generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine` will not be - intercepted). - - The *wrapper* argument must be either: - - * a callable that accepts one argument (a coroutine object); - * ``None``, to reset the wrapper. - - If called twice, the new wrapper replaces the previous one. The function - is thread-specific. - - The *wrapper* callable cannot define new coroutines directly or indirectly:: - - def wrapper(coro): - async def wrap(coro): - return await coro - return wrap(coro) - sys.set_coroutine_wrapper(wrapper) - - async def foo(): - pass - - # The following line will fail with a RuntimeError, because - # ``wrapper`` creates a ``wrap(coro)`` coroutine: - foo() - - See also :func:`get_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - .. function:: _enablelegacywindowsfsencoding() Changes the default filesystem encoding and errors mode to 'mbcs' and @@ -1487,6 +1488,41 @@ always available. is suppressed and only the exception type and value are printed. +.. function:: unraisablehook(unraisable, /) + + Handle an unraisable exception. + + Called when an exception has occurred but there is no way for Python to + handle it. For example, when a destructor raises an exception or during + garbage collection (:func:`gc.collect`). + + The *unraisable* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *err_msg*: Error message, can be ``None``. + * *object*: Object causing the exception, can be ``None``. + + The default hook formats *err_msg* and *object* as: + ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message + if *err_msg* is ``None``. + + :func:`sys.unraisablehook` can be overridden to control how unraisable + exceptions are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + + See also :func:`excepthook` which handles uncaught exceptions. + + .. versionadded:: 3.8 + .. data:: version A string containing the version number of the Python interpreter plus additional diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 4ba426425277f9..0262a9b084108b 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -141,6 +141,8 @@ Telnet Objects Do not try to reopen an already connected instance. + .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open + .. method:: Telnet.msg(msg, *args) @@ -176,6 +178,8 @@ Telnet Objects block if the connection is blocked. May raise :exc:`OSError` if the connection is closed. + .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write + .. versionchanged:: 3.3 This method used to raise :exc:`socket.error`, which is now an alias of :exc:`OSError`. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index daa6f621f1378e..0793e43df0145c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -62,6 +62,8 @@ The module defines the following user-callable items: The :py:data:`os.O_TMPFILE` flag is used if it is available and works (Linux-specific, requires Linux kernel 3.11 or later). + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + .. versionchanged:: 3.5 The :py:data:`os.O_TMPFILE` flag is now used if available. @@ -85,6 +87,8 @@ The module defines the following user-callable items: attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + .. versionchanged:: 3.8 Added *errors* parameter. @@ -130,6 +134,8 @@ The module defines the following user-callable items: The directory can be explicitly cleaned up by calling the :func:`cleanup` method. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + .. versionadded:: 3.2 @@ -177,6 +183,8 @@ The module defines the following user-callable items: file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. @@ -198,6 +206,8 @@ The module defines the following user-callable items: :func:`mkdtemp` returns the absolute pathname of the new directory. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 054521dcc5d3eb..7d62a94d839bcd 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1081,6 +1081,67 @@ The :mod:`test.support` module defines the following functions: :exc:`PermissionError` is raised. +.. function:: catch_threading_exception() + + Context manager catching :class:`threading.Thread` exception using + :func:`threading.excepthook`. + + Attributes set when an exception is catched: + + * ``exc_type`` + * ``exc_value`` + * ``exc_traceback`` + * ``thread`` + + See :func:`threading.excepthook` documentation. + + These attributes are deleted at the context manager exit. + + Usage:: + + with support.catch_threading_exception() as cm: + # code spawning a thread which raises an exception + ... + + # check the thread exception, use cm attributes: + # exc_type, exc_value, exc_traceback, thread + ... + + # exc_type, exc_value, exc_traceback, thread attributes of cm no longer + # exists at this point + # (to avoid reference cycles) + + .. versionadded:: 3.8 + + +.. function:: catch_unraisable_exception() + + Context manager catching unraisable exception using + :func:`sys.unraisablehook`. + + Storing the exception value (``cm.unraisable.exc_value``) creates a + reference cycle. The reference cycle is broken explicitly when the context + manager exits. + + Storing the object (``cm.unraisable.object``) can resurrect it if it is set + to an object which is being finalized. Exiting the context manager clears + the stored object. + + Usage:: + + with support.catch_unraisable_exception() as cm: + # code creating an "unraisable exception" + ... + + # check the unraisable exception: use cm.unraisable + ... + + # cm.unraisable attribute no longer exists at this point + # (to break a reference cycle) + + .. versionadded:: 3.8 + + .. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) Returns an unused port that should be suitable for binding. This is diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index d254466c9a320d..0f11ef401569de 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -77,6 +77,9 @@ functions should be good enough; otherwise, you should use an instance of equal: the lines ``" hello"`` and ``"\thello"`` are considered to have no common leading whitespace. + Lines containing only whitespace are ignored in the input and normalized to a + single newline character in the output. + For example:: def test(): diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index c58a6ad75d08fd..f80eb22e18fca4 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -38,6 +38,40 @@ This module defines the following functions: returned. +.. function:: excepthook(args, /) + + Handle uncaught exception raised by :func:`Thread.run`. + + The *args* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *thread*: Thread which raised the exception, can be ``None``. + + If *exc_type* is :exc:`SystemExit`, the exception is silently ignored. + Otherwise, the exception is printed out on :data:`sys.stderr`. + + If this function raises an exception, :func:`sys.excepthook` is called to + handle it. + + :func:`threading.excepthook` can be overridden to control how uncaught + exceptions raised by :func:`Thread.run` are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + + .. seealso:: + :func:`sys.excepthook` handles uncaught exceptions. + + .. versionadded:: 3.8 + + .. function:: get_ident() Return the 'thread identifier' of the current thread. This is a nonzero @@ -49,6 +83,18 @@ This module defines the following functions: .. versionadded:: 3.3 +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + .. function:: enumerate() Return a list of all :class:`Thread` objects currently alive. The list @@ -179,6 +225,10 @@ called is terminated. A thread has a name. The name can be passed to the constructor, and read or changed through the :attr:`~Thread.name` attribute. +If the :meth:`~Thread.run` method raises an exception, +:func:`threading.excepthook` is called to handle it. By default, +:func:`threading.excepthook` ignores silently :exc:`SystemExit`. + A thread can be flagged as a "daemon thread". The significance of this flag is that the entire Python program exits when only daemon threads are left. The initial value is inherited from the creating thread. The flag can be set @@ -230,6 +280,8 @@ since it is impossible to detect the termination of alien threads. base class constructor (``Thread.__init__()``) before doing anything else to the thread. + Daemon threads must not be used in subinterpreters. + .. versionchanged:: 3.3 Added the *daemon* argument. @@ -244,6 +296,12 @@ since it is impossible to detect the termination of alien threads. This method will raise a :exc:`RuntimeError` if called more than once on the same thread object. + Raise a :exc:`RuntimeError` if the thread is a daemon thread and the + method is called from a subinterpreter. + + .. versionchanged:: 3.9 + In a subinterpreter, spawning a daemon thread now raises an exception. + .. method:: run() Method representing the thread's activity. @@ -297,6 +355,26 @@ since it is impossible to detect the termination of alien threads. another thread is created. The identifier is available even after the thread has exited. + .. attribute:: native_id + + The native integral thread ID of this thread. + This is a non-negative integer, or ``None`` if the thread has not + been started. See the :func:`get_native_id` function. + This represents the Thread ID (``TID``) as assigned to the + thread by the OS (kernel). Its value may be used to uniquely identify + this particular thread system-wide (until the thread terminates, + after which the value may be recycled by the OS). + + .. note:: + + Similar to Process IDs, Thread IDs are only valid (guaranteed unique + system-wide) from the time the thread is created until the thread + has been terminated. + + .. availability:: Require :func:`get_native_id` function. + + .. versionadded:: 3.8 + .. method:: is_alive() Return whether the thread is alive. @@ -937,7 +1015,7 @@ As an example, here is a simple way to synchronize a client and server thread:: Return the barrier to the default, empty state. Any threads waiting on it will receive the :class:`BrokenBarrierError` exception. - Note that using this function may can require some external + Note that using this function may require some external synchronization if there are other threads whose state is unknown. If a barrier is broken it may be better to just leave it and create a new one. @@ -945,7 +1023,7 @@ As an example, here is a simple way to synchronize a client and server thread:: Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. Use - this for example if one of the needs to abort, to avoid deadlocking the + this for example if one of the threads needs to abort, to avoid deadlocking the application. It may be preferable to simply create the barrier with a sensible diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 170f8dc629bfe1..6d0ceafa522ad6 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -136,30 +136,6 @@ Functions Unlike the C function of the same name, :func:`asctime` does not add a trailing newline. - -.. function:: clock() - - .. index:: - single: CPU time - single: processor time - single: benchmarking - - On Unix, return the current processor time as a floating point number expressed - in seconds. The precision, and in fact the very definition of the meaning of - "processor time", depends on that of the C function of the same name. - - On Windows, this function returns wall-clock seconds elapsed since the first - call to this function, as a floating point number, based on the Win32 function - :c:func:`QueryPerformanceCounter`. The resolution is typically better than one - microsecond. - - .. availability:: Windows, Unix. Not available on VxWorks. - - .. deprecated:: 3.3 - The behaviour of this function depends on the platform: use - :func:`perf_counter` or :func:`process_time` instead, depending on your - requirements, to have a well defined behaviour. - .. function:: pthread_getcpuclockid(thread_id) Return the *clk_id* of the thread-specific CPU-time clock for the specified *thread_id*. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 8ca37034f79b7c..ef7a4e40be6590 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -44,8 +44,12 @@ This can be achieved from the :ref:`python-interface` with:: >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) 0.23702679807320237 +A callable can also be passed from the :ref:`python-interface`:: -Note however that :mod:`timeit` will automatically determine the number of + >>> timeit.timeit(lambda: "-".join(map(str, range(100))), number=10000) + 0.19665591977536678 + +Note however that :func:`.timeit` will automatically determine the number of repetitions only when the command-line interface is used. In the :ref:`timeit-examples` section you can find more advanced examples. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 60cf892e0888b7..e1fc051d9597c6 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -31,7 +31,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `TKDocs `_ Extensive tutorial plus friendlier widget pages for some of the widgets. - `Tkinter reference: a GUI for Python `_ + `Tkinter 8.5 reference: a GUI for Python `_ On-line reference material. `Tkinter docs from effbot `_ @@ -41,7 +41,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. Book by Mark Lutz, has excellent coverage of Tkinter. `Modern Tkinter for Busy Python Developers `_ - Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. + Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. `Python and Tkinter Programming `_ Book by John Grayson (ISBN 1-884777-81-3). @@ -55,7 +55,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `Tcl/Tk recent man pages `_ Recent Tcl/Tk manuals on www.tcl.tk. - `ActiveState Tcl Home Page `_ + `ActiveState Tcl Home Page `_ The Tk/Tcl development is largely taking place at ActiveState. `Tcl and the Tk Toolkit `_ diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 111289c767f35c..b208ba46d17d99 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -39,8 +39,8 @@ The primary entry point is a :term:`generator`: column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed (the last tuple item) - is the *logical* line; continuation lines are included. The 5 tuple is - returned as a :term:`named tuple` with the field names: + is the *physical* line. The 5 tuple is returned as a :term:`named tuple` + with the field names: ``type string start end line``. The returned :term:`named tuple` has an additional property named diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 5cb7029adf5e9e..c2732d900bc138 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -42,6 +42,9 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. +.. versionadded:: 3.8 + Added ``--module`` option that allows to run an executable module. + Main options ^^^^^^^^^^^^ @@ -163,7 +166,7 @@ Programmatic Interface environments. If not defined, *globals* and *locals* default to empty dictionaries. - .. method:: runfunc(func, *args, **kwds) + .. method:: runfunc(func, /, *args, **kwds) Call *func* with the given arguments under control of the :class:`Trace` object with the current tracing parameters. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 3d90d3cbd974f5..7f9f0c34386799 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -20,8 +20,8 @@ Introduction ============ Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzig and -Seymour Papert in 1966. +part of the original Logo programming language developed by Wally Feurzeig, +Seymour Papert and Cynthia Solomon in 1967. Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 07c3a2e7f68276..a21fb44dda5dec 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -98,6 +98,9 @@ the types that arise only incidentally during processing such as the Typical use of these names is for :func:`isinstance` or :func:`issubclass` checks. + +If you instantiate any of these types, note that signatures may vary between Python versions. + Standard names are defined for the following types: .. data:: FunctionType @@ -327,7 +330,7 @@ Additional Utility Classes and Functions The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, **kwargs): + def __init__(self, /, **kwargs): self.__dict__.update(kwargs) def __repr__(self): diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c2523ed5296003..d2dd03d50fc6e7 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -17,7 +17,8 @@ -------------- -This module supports type hints as specified by :pep:`484` and :pep:`526`. +This module provides runtime support for type hints as specified by +:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, and :pep:`591`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For @@ -392,6 +393,48 @@ it as a return value) of a more specialized type is a type error. For example:: Use :class:`object` to indicate that a value could be any type in a typesafe manner. Use :data:`Any` to indicate that a value is dynamically typed. + +Nominal vs structural subtyping +------------------------------- + +Initially :pep:`484` defined Python static type system as using +*nominal subtyping*. This means that a class ``A`` is allowed where +a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. + +This requirement previously also applied to abstract base classes, such as +:class:`Iterable`. The problem with this approach is that a class had +to be explicitly marked to support them, which is unpythonic and unlike +what one would normally do in idiomatic dynamically typed Python code. +For example, this conforms to the :pep:`484`:: + + from typing import Sized, Iterable, Iterator + + class Bucket(Sized, Iterable[int]): + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + +:pep:`544` allows to solve this problem by allowing users to write +the above code without explicit base classes in the class definition, +allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` +and ``Iterable[int]`` by static type checkers. This is known as +*structural subtyping* (or static duck-typing):: + + from typing import Iterator, Iterable + + class Bucket: # Note: no base classes + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + + def collect(items: Iterable[int]) -> int: ... + result = collect(Bucket()) # Passes type check + +Moreover, by subclassing a special class :class:`Protocol`, a user +can define new custom protocols to fully enjoy structural subtyping +(see examples below). + + Classes, functions, and decorators ---------------------------------- @@ -459,6 +502,39 @@ The module defines the following classes, functions and decorators: except KeyError: return default +.. class:: Protocol(Generic) + + Base class for protocol classes. Protocol classes are defined like this:: + + class Proto(Protocol): + def meth(self) -> int: + ... + + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: + + class C: + def meth(self) -> int: + return 0 + + def func(x: Proto) -> int: + return x.meth() + + func(C()) # Passes static type check + + See :pep:`544` for details. Protocol classes decorated with + :func:`runtime_checkable` (described later) act as simple-minded runtime + protocols that check only the presence of given attributes, ignoring their + type signatures. + + Protocol classes can be generic, for example:: + + class GenProto(Protocol[T]): + def meth(self) -> T: + ... + + .. versionadded:: 3.8 + .. class:: Type(Generic[CT_co]) A variable annotated with ``C`` may accept a value of type ``C``. In @@ -529,6 +605,12 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__bytes__``. +.. class:: SupportsIndex + + An ABC with one abstract method ``__index__``. + + .. versionadded:: 3.8 + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -555,7 +637,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Collection` - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. class:: AbstractSet(Sized, Collection[T_co]) @@ -599,6 +681,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: List(list, MutableSequence[T]) @@ -648,6 +731,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Awaitable`. + .. versionadded:: 3.5.2 + .. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. @@ -661,25 +746,33 @@ The module defines the following classes, functions and decorators: async def bar() -> None: x = await c # type: int + .. versionadded:: 3.5.3 + .. class:: AsyncIterable(Generic[T_co]) A generic version of :class:`collections.abc.AsyncIterable`. + .. versionadded:: 3.5.2 + .. class:: AsyncIterator(AsyncIterable[T_co]) A generic version of :class:`collections.abc.AsyncIterator`. + .. versionadded:: 3.5.2 + .. class:: ContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 .. class:: AsyncContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -708,12 +801,14 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) A generic version of :class:`collections.ChainMap`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -778,7 +873,7 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 .. class:: Text @@ -872,6 +967,46 @@ The module defines the following classes, functions and decorators: The ``_field_types`` and ``__annotations__`` attributes are now regular dictionaries instead of instances of ``OrderedDict``. +.. class:: TypedDict(dict) + + A simple typed namespace. At runtime it is equivalent to + a plain :class:`dict`. + + ``TypedDict`` creates a dictionary type that expects all of its + instances to have a certain set of keys, where each key is + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by type checkers. + Usage:: + + class Point2D(TypedDict): + x: int + y: int + label: str + + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info for introspection can be accessed via ``Point2D.__annotations__`` + and ``Point2D.__total__``. To allow using this feature with older versions + of Python that do not support :pep:`526`, ``TypedDict`` supports two additional + equivalent syntactic forms:: + + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + + See :pep:`589` for more examples and detailed rules of using ``TypedDict`` + with type checkers. + + .. versionadded:: 3.8 + +.. class:: ForwardRef + + A class used for internal typing representation of string forward references. + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + a user, but may be used by introspection tools. .. function:: NewType(typ) @@ -906,6 +1041,25 @@ The module defines the following classes, functions and decorators: a dictionary constructed by merging all the ``__annotations__`` along ``C.__mro__`` in reverse order. +.. function:: get_origin(typ) +.. function:: get_args(typ) + + Provide basic introspection for generic types and special typing forms. + + For a typing object of the form ``X[Y, Z, ...]`` these functions return + ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or + :mod:`collections` class, it gets normalized to the original class. + For unsupported objects return ``None`` and ``()`` correspondingly. + Examples:: + + assert get_origin(Dict[str, int]) is dict + assert get_args(Dict[int, str]) == (int, str) + + assert get_origin(Union[int, str]) is Union + assert get_args(Union[int, str]) == (int, str) + + .. versionadded:: 3.8 + .. decorator:: overload The ``@overload`` decorator allows describing functions and methods @@ -934,6 +1088,31 @@ The module defines the following classes, functions and decorators: See :pep:`484` for details and comparison with other typing semantics. +.. decorator:: final + + A decorator to indicate to type checkers that the decorated method + cannot be overridden, and the decorated class cannot be subclassed. + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... + + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + .. decorator:: no_type_check Decorator to indicate that annotations are not type hints. @@ -969,6 +1148,26 @@ The module defines the following classes, functions and decorators: Note that returning instances of private classes is not recommended. It is usually preferable to make such classes public. +.. decorator:: runtime_checkable + + Mark a protocol class as a runtime protocol. + + Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. + This raises :exc:`TypeError` when applied to a non-protocol class. This + allows a simple-minded structural check, very similar to "one trick ponies" + in :mod:`collections.abc` such as :class:`Iterable`. For example:: + + @runtime_checkable + class Closable(Protocol): + def close(self): ... + + assert isinstance(open('/some/file'), Closable) + + **Warning:** this will check only the presence of the required methods, + not their type signatures! + + .. versionadded:: 3.8 + .. data:: Any Special type indicating an unconstrained type. @@ -987,6 +1186,7 @@ The module defines the following classes, functions and decorators: raise RuntimeError('no way') .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. data:: Union @@ -1072,6 +1272,28 @@ The module defines the following classes, functions and decorators: ``Callable[..., Any]``, and in turn to :class:`collections.abc.Callable`. +.. data:: Literal + + A type that can be used to indicate to type checkers that the + corresponding variable or function parameter has a value equivalent to + the provided literal (or one of several literals). For example:: + + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... + + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... + + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker + + ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value + is allowed as type argument to ``Literal[...]``, but type checkers may + impose restrictions. See :pep:`586` for more details about literal types. + + .. versionadded:: 3.8 + .. data:: ClassVar Special type construct to mark class variables. @@ -1098,6 +1320,25 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.5.3 +.. data:: Final + + A special typing construct to indicate to type checkers that a name + cannot be re-assigned or overridden in a subclass. For example:: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker + + class Connection: + TIMEOUT: Final[int] = 10 + + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + .. data:: AnyStr ``AnyStr`` is a type variable defined as diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index b019fa5b3bc9c2..ee790c0cd00bbc 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -17,8 +17,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 12.0.0 -`_. +this database is compiled from the `UCD version 12.1.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/12.0.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/12.1.0/ucd/NameAliases.txt -.. [#] http://www.unicode.org/Public/12.0.0/ucd/NamedSequences.txt +.. [#] http://www.unicode.org/Public/12.1.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 16690f349822ec..811f0fb1ce9397 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -848,7 +848,7 @@ Here's an example implementation: >>> from copy import deepcopy >>> class CopyingMock(MagicMock): - ... def __call__(self, *args, **kwargs): + ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) ... return super(CopyingMock, self).__call__(*args, **kwargs) @@ -1042,7 +1042,7 @@ that it takes arbitrary keyword arguments (``**kwargs``) which are then passed onto the mock constructor: >>> class Subclass(MagicMock): - ... def _get_child_mock(self, **kwargs): + ... def _get_child_mock(self, /, **kwargs): ... return MagicMock(**kwargs) ... >>> mymock = Subclass() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ed00ee6d0c2d83..46e8ef38ab11bd 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -201,9 +201,11 @@ The Mock Class .. testsetup:: + import asyncio + import inspect import unittest from unittest.mock import sentinel, DEFAULT, ANY - from unittest.mock import patch, call, Mock, MagicMock, PropertyMock + from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock from unittest.mock import mock_open :class:`Mock` is a flexible mock object intended to replace the use of stubs and @@ -851,6 +853,217 @@ object:: >>> p.assert_called_once_with() +.. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) + + An asynchronous version of :class:`Mock`. The :class:`AsyncMock` object will + behave so the object is recognized as an async function, and the result of a + call is an awaitable. + + >>> mock = AsyncMock() + >>> asyncio.iscoroutinefunction(mock) + True + >>> inspect.isawaitable(mock()) # doctest: +SKIP + True + + The result of ``mock()`` is an async function which will have the outcome + of ``side_effect`` or ``return_value``: + + - if ``side_effect`` is a function, the async function will return the + result of that function, + - if ``side_effect`` is an exception, the async function will raise the + exception, + - if ``side_effect`` is an iterable, the async function will return the + next value of the iterable, however, if the sequence of result is + exhausted, ``StopIteration`` is raised immediately, + - if ``side_effect`` is not defined, the async function will return the + value defined by ``return_value``, hence, by default, the async function + returns a new :class:`AsyncMock` object. + + + Setting the *spec* of a :class:`Mock` or :class:`MagicMock` to an async function + will result in a coroutine object being returned after calling. + + >>> async def async_func(): pass + ... + >>> mock = MagicMock(async_func) + >>> mock + + >>> mock() # doctest: +SKIP + + + .. method:: assert_awaited() + + Assert that the mock was awaited at least once. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.assert_awaited() + >>> mock_2 = AsyncMock() + >>> mock_2.assert_awaited() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited. + + .. method:: assert_awaited_once() + + Assert that the mock was awaited exactly once. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.assert_awaited_once() + >>> asyncio.run(main()) + >>> mock.method.assert_awaited_once() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_awaited_with(*args, **kwargs) + + Assert that the last await was with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_with('foo', bar='bar') + >>> mock.assert_awaited_with('other') + Traceback (most recent call last): + ... + AssertionError: expected call not found. + Expected: mock('other') + Actual: mock('foo', bar='bar') + + .. method:: assert_awaited_once_with(*args, **kwargs) + + Assert that the mock was awaited exactly once and with the specified + arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_any_await(*args, **kwargs) + + Assert the mock has ever been awaited with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> asyncio.run(main('hello')) + >>> mock.assert_any_await('foo', bar='bar') + >>> mock.assert_any_await('other') + Traceback (most recent call last): + ... + AssertionError: mock('other') await not found + + .. method:: assert_has_awaits(calls, any_order=False) + + Assert the mock has been awaited with the specified calls. + The :attr:`await_args_list` list is checked for the awaits. + + If *any_order* is False (the default) then the awaits must be + sequential. There can be extra calls before or after the + specified awaits. + + If *any_order* is True then the awaits can be in any order, but + they must all appear in :attr:`await_args_list`. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> calls = [call("foo"), call("bar")] + >>> mock.assert_has_calls(calls) + Traceback (most recent call last): + ... + AssertionError: Calls not found. + Expected: [call('foo'), call('bar')] + >>> asyncio.run(main('foo')) + >>> asyncio.run(main('bar')) + >>> mock.assert_has_calls(calls) + + .. method:: assert_not_awaited() + + Assert that the mock was never awaited. + + >>> mock = AsyncMock() + >>> mock.assert_not_awaited() + + .. method:: reset_mock(*args, **kwargs) + + See :func:`Mock.reset_mock`. Also sets :attr:`await_count` to 0, + :attr:`await_args` to None, and clears the :attr:`await_args_list`. + + .. attribute:: await_count + + An integer keeping track of how many times the mock object has been awaited. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.await_count + 1 + >>> asyncio.run(main()) + >>> mock.await_count + 2 + + .. attribute:: await_args + + This is either ``None`` (if the mock hasn’t been awaited), or the arguments that + the mock was last awaited with. Functions the same as :attr:`Mock.call_args`. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args + >>> asyncio.run(main('foo')) + >>> mock.await_args + call('foo') + >>> asyncio.run(main('bar')) + >>> mock.await_args + call('bar') + + + .. attribute:: await_args_list + + This is a list of all the awaits made to the mock object in sequence (so the + length of the list is the number of times it has been awaited). Before any + awaits have been made it is an empty list. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args_list + [] + >>> asyncio.run(main('foo')) + >>> mock.await_args_list + [call('foo')] + >>> asyncio.run(main('bar')) + >>> mock.await_args_list + [call('foo'), call('bar')] + + Calling ~~~~~~~ @@ -1343,15 +1556,24 @@ patch.dict decorator. When used as a class decorator :func:`patch.dict` honours ``patch.TEST_PREFIX`` for choosing which methods to wrap. + .. versionchanged:: 3.8 + + :func:`patch.dict` now returns the patched dictionary when used as a context + manager. + :func:`patch.dict` can be used to add members to a dictionary, or simply let a test change a dictionary, and ensure the dictionary is restored when the test ends. >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}): + >>> with patch.dict(foo, {'newkey': 'newvalue'}) as patched_foo: ... assert foo == {'newkey': 'newvalue'} + ... assert patched_foo == {'newkey': 'newvalue'} + ... # You can add, update or delete keys of foo (or patched_foo, it's the same dict) + ... patched_foo['spam'] = 'eggs' ... >>> assert foo == {} + >>> assert patched_foo == {} >>> import os >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): @@ -1732,7 +1954,7 @@ The full list of supported magic methods is: * Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` and ``__missing__`` -* Context manager: ``__enter__`` and ``__exit__`` +* Context manager: ``__enter__``, ``__exit__``, ``__aenter__`` and ``__aexit__`` * Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` * The numeric methods (including right hand and in-place variants): ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, @@ -1744,10 +1966,14 @@ The full list of supported magic methods is: * Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` * File system path representation: ``__fspath__`` +* Asynchronous iteration methods: ``__aiter__`` and ``__anext__`` .. versionchanged:: 3.8 Added support for :func:`os.PathLike.__fspath__`. +.. versionchanged:: 3.8 + Added support for ``__aenter__``, ``__aexit__``, ``__aiter__`` and ``__anext__``. + The following methods exist but are *not* supported as they are either in use by mock, can't be set dynamically, or can cause problems: @@ -1810,6 +2036,7 @@ Methods and their defaults: * ``__len__``: 0 * ``__iter__``: iter([]) * ``__exit__``: False +* ``__aexit__``: False * ``__complex__``: 1j * ``__float__``: 1.0 * ``__bool__``: True diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 8ad2abd3d89a44..320d898fc8d4ca 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -510,7 +510,8 @@ that is broken and will fail, but shouldn't be counted as a failure on a :class:`TestResult`. Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants. +or one of its conditional variants, calling :meth:`TestCase.skipTest` within a +:meth:`~TestCase.setUp` or test method, or raising :exc:`SkipTest` directly. Basic skipping looks like this:: @@ -531,16 +532,23 @@ Basic skipping looks like this:: # windows specific testing code pass + def test_maybe_skipped(self): + if not external_resource_available(): + self.skipTest("external resource not available") + # test code that depends on the external resource + pass + This is the output of running the example above in verbose mode:: test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' + test_maybe_skipped (__main__.MyTestCase) ... skipped 'external resource not available' test_windows_support (__main__.MyTestCase) ... skipped 'requires Windows' ---------------------------------------------------------------------- - Ran 3 tests in 0.005s + Ran 4 tests in 0.005s - OK (skipped=3) + OK (skipped=4) Classes can be skipped just like methods:: @@ -568,7 +576,7 @@ the test unless the passed object has a certain attribute:: return lambda func: func return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) -The following decorators implement test skipping and expected failures: +The following decorators and exception implement test skipping and expected failures: .. decorator:: skip(reason) @@ -1419,7 +1427,7 @@ Test cases :class:`TextTestResult` in Python 3.2. - .. method:: addCleanup(function, *args, **kwargs) + .. method:: addCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDown` to cleanup resources used during the test. Functions will be called in reverse order to the @@ -1448,7 +1456,7 @@ Test cases .. versionadded:: 3.1 - .. classmethod:: addClassCleanup(function, *args, **kwargs) + .. classmethod:: addClassCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDownClass` to cleanup resources used during the test class. Functions will be called in reverse @@ -2305,7 +2313,7 @@ To add cleanup code that must be run even in the case of an exception, use ``addModuleCleanup``: -.. function:: addModuleCleanup(function, *args, **kwargs) +.. function:: addModuleCleanup(function, /, *args, **kwargs) Add a function to be called after :func:`tearDownModule` to cleanup resources used during the test class. Functions will be called in reverse diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index f9936288fd42cd..49276daa7ff43f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -370,6 +370,13 @@ or on combining URL components into a URL string. .. versionchanged:: 3.2 Result is a structured object rather than a simple 2-tuple. +.. function:: unwrap(url) + + Extract the url from a wrapped URL (that is, a string formatted as + ````, ````, ``URL:scheme://host/path`` + or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned + without changes. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 14fa27bb08af20..3b750896669149 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -95,6 +95,12 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. + .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen + + The default opener raises an :ref:`auditing event ` + ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, + ``method`` taken from the request object. + .. versionchanged:: 3.2 *cafile* and *capath* were added. @@ -118,6 +124,7 @@ The :mod:`urllib.request` module defines the following functions: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 412808ad44866d..d3d5ae2b007d5f 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -97,7 +97,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None) + prompt=None, upgrade_deps=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -123,12 +123,17 @@ creation according to their needs, the :class:`EnvBuilder` class. (defaults to ``None`` which means directory name of the environment would be used). + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + .. versionchanged:: 3.4 Added the ``with_pip`` parameter .. versionadded:: 3.6 Added the ``prompt`` parameter + .. versionadded:: 3.8 + Added the ``upgrade_deps`` parameter + Creators of third-party virtual environment tools will be free to use the provided ``EnvBuilder`` class as a base class. @@ -234,14 +239,19 @@ creation according to their needs, the :class:`EnvBuilder` class. There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False) + symlinks=False, with_pip=False, prompt=None) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. + .. versionadded:: 3.3 + .. versionchanged:: 3.4 Added the ``with_pip`` parameter + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index d121f320d6a3eb..a481a3509d4ec8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -19,10 +19,10 @@ Python programmers issue warnings by calling the :func:`warn` function defined in this module. (C programmers use :c:func:`PyErr_WarnEx`; see :ref:`exceptionhandling` for details). -Warning messages are normally written to ``sys.stderr``, but their disposition +Warning messages are normally written to :data:`sys.stderr`, but their disposition can be changed flexibly, from ignoring all warnings to turning them into -exceptions. The disposition of warnings can vary based on the warning category -(see below), the text of the warning message, and the source location where it +exceptions. The disposition of warnings can vary based on the :ref:`warning category +`, the text of the warning message, and the source location where it is issued. Repetitions of a particular warning for the same source location are typically suppressed. @@ -31,7 +31,7 @@ determination is made whether a message should be issued or not; next, if a message is to be issued, it is formatted and printed using a user-settable hook. The determination whether to issue a warning message is controlled by the -warning filter, which is a sequence of matching rules and actions. Rules can be +:ref:`warning filter `, which is a sequence of matching rules and actions. Rules can be added to the filter by calling :func:`filterwarnings` and reset to its default state by calling :func:`resetwarnings`. @@ -181,9 +181,9 @@ Describing Warning Filters The warnings filter is initialized by :option:`-W` options passed to the Python interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. The interpreter saves the arguments for all supplied entries without -interpretation in ``sys.warnoptions``; the :mod:`warnings` module parses these +interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these when it is first imported (invalid options are ignored, after printing a -message to ``sys.stderr``). +message to :data:`sys.stderr`). Individual warnings filters are specified as a sequence of fields separated by colons:: @@ -192,7 +192,7 @@ colons:: The meaning of each of these fields is as described in :ref:`warning-filter`. When listing multiple filters on a single line (as for -:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas,and +:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas and the filters listed later take precedence over those listed before them (as they're applied left-to-right, and the most recently applied filters take precedence over earlier ones). @@ -395,12 +395,12 @@ Available Functions .. function:: warn(message, category=None, stacklevel=1, source=None) Issue a warning, or maybe ignore it or raise an exception. The *category* - argument, if given, must be a warning category class (see above); it defaults to - :exc:`UserWarning`. Alternatively *message* can be a :exc:`Warning` instance, + argument, if given, must be a :ref:`warning category class `; it + defaults to :exc:`UserWarning`. Alternatively, *message* can be a :exc:`Warning` instance, in which case *category* will be ignored and ``message.__class__`` will be used. - In this case the message text will be ``str(message)``. This function raises an + In this case, the message text will be ``str(message)``. This function raises an exception if the particular warning issued is changed into an error by the - warnings filter see above. The *stacklevel* argument can be used by wrapper + :ref:`warnings filter `. The *stacklevel* argument can be used by wrapper functions written in Python, like this:: def deprecation(message): @@ -444,7 +444,7 @@ Available Functions Write a warning to a file. The default implementation calls ``formatwarning(message, category, filename, lineno, line)`` and writes the - resulting string to *file*, which defaults to ``sys.stderr``. You may replace + resulting string to *file*, which defaults to :data:`sys.stderr`. You may replace this function with any callable by assigning to ``warnings.showwarning``. *line* is a line of source code to be included in the warning message; if *line* is not supplied, :func:`showwarning` will diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 60d19a8d5f7dc0..f63e0d3dce19c6 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -47,13 +47,6 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - .. deprecated-removed:: 3.7 3.9 - - .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 80a908bbd83b0a..c3519e45beb6b1 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. @@ -240,7 +241,7 @@ objects. .. versionadded:: 3.4 -.. class:: finalize(obj, func, *args, **kwargs) +.. class:: finalize(obj, func, /, *args, **kwargs) Return a callable finalizer object which will be called when *obj* is garbage collected. Unlike an ordinary weak reference, a finalizer @@ -396,7 +397,7 @@ the referent is accessed:: import weakref class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, **annotations): + def __init__(self, ob, callback=None, /, **annotations): super(ExtendedRef, self).__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 9dc5551398ee82..b7bfb655a71579 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -64,6 +64,8 @@ The following functions are defined: may work and start the operating system's associated program. However, this is neither supported nor portable. + .. audit-event:: webbrowser.open url webbrowser.open + .. function:: open_new(url) diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index ec5136742fa2c4..6edd0714b9df3b 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -767,7 +767,7 @@ This is a working "Hello World" WSGI application:: # use a function (note that you're not limited to a function, you can # use a class for example). The first argument passed to the function # is a dictionary containing CGI-style environment variables and the - # second variable is the callable object (see PEP 333). + # second variable is the callable object. def hello_world_app(environ, start_response): status = '200 OK' # HTTP Status headers = [('Content-type', 'text/plain; charset=utf-8')] # HTTP Headers diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2423a0c15691f4..8711242d95d741 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -134,11 +134,12 @@ module documentation. This section lists the differences between the API and .. method:: Node.writexml(writer, indent="", addindent="", newl="") - Write XML to the writer object. The writer should have a :meth:`write` method - which matches that of the file object interface. The *indent* parameter is the - indentation of the current node. The *addindent* parameter is the incremental - indentation to use for subnodes of the current one. The *newl* parameter - specifies the string to use to terminate newlines. + Write XML to the writer object. The writer receives texts but not bytes as input, + it should have a :meth:`write` method which matches that of the file object + interface. The *indent* parameter is the indentation of the current node. + The *addindent* parameter is the incremental indentation to use for subnodes + of the current one. The *newl* parameter specifies the string to use to + terminate newlines. For the :class:`Document` node, an additional keyword argument *encoding* can be used to specify the encoding field of the XML header. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index ef74d0c852cd75..c4667315793e4c 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -399,6 +399,12 @@ module. We'll be using the ``countrydata`` XML document from the # All 'neighbor' nodes that are the second child of their parent root.findall(".//neighbor[2]") +For XML with namespaces, use the usual qualified ``{namespace}tag`` notation:: + + # All dublin-core "title" tags in the document + root.findall(".//{http://purl.org/dc/elements/1.1/}title") + + Supported XPath syntax ^^^^^^^^^^^^^^^^^^^^^^ @@ -411,9 +417,16 @@ Supported XPath syntax | | For example, ``spam`` selects all child elements | | | named ``spam``, and ``spam/egg`` selects all | | | grandchildren named ``egg`` in all children named | -| | ``spam``. | +| | ``spam``. ``{namespace}*`` selects all tags in the | +| | given namespace, ``{*}spam`` selects tags named | +| | ``spam`` in any (or no) namespace, and ``{}*`` | +| | only selects tags that are not in a namespace. | +| | | +| | .. versionchanged:: 3.8 | +| | Support for star-wildcards was added. | +-----------------------+------------------------------------------------------+ -| ``*`` | Selects all child elements. For example, ``*/egg`` | +| ``*`` | Selects all child elements, including comments and | +| | processing instructions. For example, ``*/egg`` | | | selects all grandchildren named ``egg``. | +-----------------------+------------------------------------------------------+ | ``.`` | Selects the current node. This is mostly useful | diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4e9edff270143d..9db9697105d6b5 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -52,6 +52,15 @@ The module defines the following items: :ref:`zipfile-objects` for constructor details. +.. class:: Path + :noindex: + + A pathlib-compatible wrapper for zip files. See section + :ref:`path-objects` for details. + + .. versionadded:: 3.8 + + .. class:: PyZipFile :noindex: @@ -456,6 +465,64 @@ The following data attributes are also available: truncated. +.. _path-objects: + +Path Objects +------------ + +.. class:: Path(root, at='') + + Construct a Path object from a ``root`` zipfile (which may be a + :class:`ZipFile` instance or ``file`` suitable for passing to + the :class:`ZipFile` constructor). + + ``at`` specifies the location of this Path within the zipfile, + e.g. 'dir/file.txt', 'dir/', or ''. Defaults to the empty string, + indicating the root. + +Path objects expose the following features of :mod:`pathlib.Path` +objects: + +Path objects are traversable using the ``/`` operator. + +.. attribute:: Path.name + + The final path component. + +.. method:: Path.open(*, **) + + Invoke :meth:`ZipFile.open` on the current path. Accepts + the same arguments as :meth:`ZipFile.open`. + +.. method:: Path.listdir() + + Enumerate the children of the current directory. + +.. method:: Path.is_dir() + + Return ``True`` if the current context references a directory. + +.. method:: Path.is_file() + + Return ``True`` if the current context references a file. + +.. method:: Path.exists() + + Return ``True`` if the current context references a file or + directory in the zip file. + +.. method:: Path.read_text(*, **) + + Read the current file as unicode text. Positional and + keyword arguments are passed through to + :class:`io.TextIOWrapper` (except ``buffer``, which is + implied by the context). + +.. method:: Path.read_bytes() + + Read the current file as bytes. + + .. _pyzipfile-objects: PyZipFile Objects diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index aa1831dbc60de9..2138697ff06415 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -6,6 +6,8 @@ .. moduleauthor:: Just van Rossum +**Source code:** :source:`Lib/zipimport.py` + -------------- This module adds the ability to import Python modules (:file:`\*.py`, diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index aa61278e099ac6..339acfd0e5786f 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -47,7 +47,7 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data, level=-1) +.. function:: compress(data, /, level=-1) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; @@ -132,7 +132,7 @@ The available exception and functions in this module are: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: decompress(data, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) +.. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) Decompresses the bytes in *data*, returning a bytes object containing the uncompressed data. The *wbits* parameter depends on diff --git a/Doc/license.rst b/Doc/license.rst index d3733f53a116f4..d877f45677549b 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _history-and-license: @@ -561,7 +561,7 @@ SipHash24 --------- The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. The contains the following note:: +Dan Bernstein's SipHash24 algorithm. It contains the following note:: Copyright (c) 2013 Marek Majkowski diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea916b..dfc622f66615df 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 42fa8647623935..988eec6d254e10 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -483,8 +483,10 @@ A function definition defines a user-defined function object (see section decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` + parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] + : | `parameter_list_no_posonly` + parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] + : | `parameter_list_starargs` parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1683d25db9247a..fa47bf1c1619b5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -890,6 +890,8 @@ Internal types .. index:: single: co_argcount (code object attribute) + single: co_posonlyargcount (code object attribute) + single: co_kwonlyargcount (code object attribute) single: co_code (code object attribute) single: co_consts (code object attribute) single: co_filename (code object attribute) @@ -905,21 +907,26 @@ Internal types single: co_freevars (code object attribute) Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_argcount` is the number of positional arguments (including arguments - with default values); :attr:`co_nlocals` is the number of local variables used - by the function (including arguments); :attr:`co_varnames` is a tuple containing + :attr:`co_argcount` is the total number of positional arguments + (including positional-only arguments and arguments with default values); + :attr:`co_posonlyargcount` is the number of positional-only arguments + (including arguments with default values); :attr:`co_kwonlyargcount` is + the number of keyword-only arguments (including arguments with default + values); :attr:`co_nlocals` is the number of local variables used by the + function (including arguments); :attr:`co_varnames` is a tuple containing the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables that are - referenced by nested functions; :attr:`co_freevars` is a tuple containing the - names of free variables; :attr:`co_code` is a string representing the sequence - of bytecode instructions; :attr:`co_consts` is a tuple containing the literals - used by the bytecode; :attr:`co_names` is a tuple containing the names used by - the bytecode; :attr:`co_filename` is the filename from which the code was - compiled; :attr:`co_firstlineno` is the first line number of the function; - :attr:`co_lnotab` is a string encoding the mapping from bytecode offsets to - line numbers (for details see the source code of the interpreter); - :attr:`co_stacksize` is the required stack size (including local variables); - :attr:`co_flags` is an integer encoding a number of flags for the interpreter. + :attr:`co_cellvars` is a tuple containing the names of local variables + that are referenced by nested functions; :attr:`co_freevars` is a tuple + containing the names of free variables; :attr:`co_code` is a string + representing the sequence of bytecode instructions; :attr:`co_consts` is + a tuple containing the literals used by the bytecode; :attr:`co_names` is + a tuple containing the names used by the bytecode; :attr:`co_filename` is + the filename from which the code was compiled; :attr:`co_firstlineno` is + the first line number of the function; :attr:`co_lnotab` is a string + encoding the mapping from bytecode offsets to line numbers (for details + see the source code of the interpreter); :attr:`co_stacksize` is the + required stack size (including local variables); :attr:`co_flags` is an + integer encoding a number of flags for the interpreter. .. index:: object: generator @@ -1311,9 +1318,9 @@ Basic customization Called by the :func:`format` built-in function, and by extension, evaluation of :ref:`formatted string literals ` and the :meth:`str.format` method, to produce a "formatted" - string representation of an object. The ``format_spec`` argument is + string representation of an object. The *format_spec* argument is a string that contains a description of the formatting options desired. - The interpretation of the ``format_spec`` argument is up to the type + The interpretation of the *format_spec* argument is up to the type implementing :meth:`__format__`, however most classes will either delegate formatting to one of the built-in types, or use a similar formatting option syntax. @@ -1804,7 +1811,7 @@ class defining the method. class, as in:: class Philosopher: - def __init_subclass__(cls, default_name, **kwargs): + def __init_subclass__(cls, /, default_name, **kwargs): super().__init_subclass__(**kwargs) cls.default_name = default_name @@ -2387,11 +2394,9 @@ left undefined. functions). Presence of this method indicates that the numeric object is an integer type. Must return an integer. - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. + If :meth:`__int__`, :meth:`__float__` and :meth:`__complex__` are not + defined then corresponding built-in functions :func:`int`, :func:`float` + and :func:`complex` fall back to :meth:`__index__`. .. method:: object.__round__(self, [,ndigits]) @@ -2680,13 +2685,13 @@ Asynchronous context managers can be used in an :keyword:`async with` statement. .. method:: object.__aenter__(self) - This method is semantically similar to the :meth:`__enter__`, with only - difference that it must return an *awaitable*. + Semantically similar to :meth:`__enter__`, the only + difference being that it must return an *awaitable*. .. method:: object.__aexit__(self, exc_type, exc_value, traceback) - This method is semantically similar to the :meth:`__exit__`, with only - difference that it must return an *awaitable*. + Semantically similar to :meth:`__exit__`, the only + difference being that it must return an *awaitable*. An example of an asynchronous context manager class:: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index ba7130d6362163..49cb86b56084d3 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -243,7 +243,7 @@ re-entering the offending piece of code from the top). When an exception is not handled at all, the interpreter terminates execution of the program, or returns to its interactive main loop. In either case, it prints -a stack backtrace, except when the exception is :exc:`SystemExit`. +a stack traceback, except when the exception is :exc:`SystemExit`. Exceptions are identified by class instances. The :keyword:`except` clause is selected depending on the class of the instance: it must reference the class of diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index cf7d05eef6746a..432327a87c3748 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -144,7 +144,7 @@ the single expression that makes up the expression list. .. index:: pair: empty; tuple An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the rules for literals apply (i.e., two occurrences of the empty +immutable, the same rules as for literals apply (i.e., two occurrences of the empty tuple may or may not yield the same object). .. index:: @@ -196,7 +196,7 @@ the comprehension is executed in a separate implicitly nested scope. This ensure that names assigned to in the target list don't "leak" into the enclosing scope. The iterable expression in the leftmost :keyword:`!for` clause is evaluated -directly in the enclosing scope and then passed as an argument to the implictly +directly in the enclosing scope and then passed as an argument to the implicitly nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as they may depend on the values obtained from the leftmost iterable. For example: @@ -337,6 +337,12 @@ all mutable objects.) Clashes between duplicate keys are not detected; the last datum (textually rightmost in the display) stored for a given key value prevails. +.. versionchanged:: 3.8 + Prior to Python 3.8, in dict comprehensions, the evaluation order of key + and value was not well-defined. In CPython, the value was evaluated before + the key. Starting with 3.8, the key is evaluated before the value, as + proposed by :pep:`572`. + .. _genexpr: @@ -479,8 +485,8 @@ will raise :exc:`AttributeError` or :exc:`TypeError`, while When the underlying iterator is complete, the :attr:`~StopIteration.value` attribute of the raised :exc:`StopIteration` instance becomes the value of the yield expression. It can be either set explicitly when raising -:exc:`StopIteration`, or automatically when the sub-iterator is a generator -(by returning a value from the sub-generator). +:exc:`StopIteration`, or automatically when the subiterator is a generator +(by returning a value from the subgenerator). .. versionchanged:: 3.3 Added ``yield from `` to delegate control flow to a subiterator. @@ -499,7 +505,7 @@ on the right hand side of an assignment statement. :pep:`380` - Syntax for Delegating to a Subgenerator The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. + to subgenerators easy. :pep:`525` - Asynchronous Generators The proposal that expanded on :pep:`492` by adding generator capabilities to @@ -608,7 +614,7 @@ Asynchronous generator functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The presence of a yield expression in a function or method defined using -:keyword:`async def` further defines the function as a +:keyword:`async def` further defines the function as an :term:`asynchronous generator` function. When an asynchronous generator function is called, it returns an @@ -673,13 +679,13 @@ which are used to control the execution of a generator function. Returns an awaitable which when run starts to execute the asynchronous generator or resumes it at the last executed yield expression. When an - asynchronous generator function is resumed with a :meth:`~agen.__anext__` + asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield expression. The value of the :token:`expression_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without - yielding another value, the awaitable instead raises an + yielding another value, the awaitable instead raises a :exc:`StopAsyncIteration` exception, signalling that the asynchronous iteration has completed. @@ -707,7 +713,7 @@ which are used to control the execution of a generator function. where the asynchronous generator was paused, and returns the next value yielded by the generator function as the value of the raised :exc:`StopIteration` exception. If the asynchronous generator exits - without yielding another value, an :exc:`StopAsyncIteration` exception is + without yielding another value, a :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or raises a different exception, then when the awaitable is run that exception @@ -1563,14 +1569,15 @@ y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is -produced while iterating over ``y``. If an exception is raised during the -iteration, it is as if :keyword:`in` raised that exception. +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +expression ``x is z or x == z`` is true, is produced while iterating over ``y``. +If an exception is raised during the iteration, it is as if :keyword:`in` raised +that exception. Lastly, the old-style iteration protocol is tried: if a class defines :meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x == y[i]``, and all lower integer indices do not -raise :exc:`IndexError` exception. (If any other exception is raised, it is as +integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index +raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). .. index:: @@ -1579,7 +1586,7 @@ if :keyword:`in` raised that exception). pair: membership; test object: sequence -The operator :keyword:`not in` is defined to have the inverse true value of +The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. .. index:: @@ -1594,8 +1601,8 @@ The operator :keyword:`not in` is defined to have the inverse true value of Identity comparisons -------------------- -The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x -is y`` is true if and only if *x* and *y* are the same object. Object identity +The operators :keyword:`is` and :keyword:`is not` test for an object's identity: ``x +is y`` is true if and only if *x* and *y* are the same object. An Object's identity is determined using the :meth:`id` function. ``x is not y`` yields the inverse truth value. [#]_ diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index fb04ccc839aaa3..cc1b2f57a70e3b 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -70,7 +70,7 @@ Comments A comment starts with a hash character (``#``) that is not part of a string literal, and ends at the end of the physical line. A comment signifies the end of the logical line unless the implicit line joining rules are invoked. Comments -are ignored by the syntax; they are not tokens. +are ignored by the syntax. .. _encodings: @@ -316,7 +316,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -680,11 +680,12 @@ with a closing curly bracket ``'}'``. Expressions in formatted string literals are treated like regular Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and a :keyword:`lambda` expression -must be surrounded by explicit parentheses. Replacement expressions -can contain line breaks (e.g. in triple-quoted strings), but they -cannot contain comments. Each expression is evaluated in the context -where the formatted string literal appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses. +Replacement expressions can contain line breaks (e.g. in triple-quoted +strings), but they cannot contain comments. Each expression is evaluated +in the context where the formatted string literal appears, in order from +left to right. If a conversion is specified, the result of evaluating the expression is converted before formatting. Conversion ``'!s'`` calls :func:`str` on diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 207057cbc12414..0a043a90050c4e 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -169,12 +169,12 @@ Assignment of an object to a single target is recursively defined as follows. .. _attr-target-note: Note: If the object is a class instance and the attribute reference occurs on - both sides of the assignment operator, the RHS expression, ``a.x`` can access + both sides of the assignment operator, the right-hand side expression, ``a.x`` can access either an instance attribute or (if no instance attribute exists) a class - attribute. The LHS target ``a.x`` is always set as an instance attribute, + attribute. The left-hand side target ``a.x`` is always set as an instance attribute, creating it if necessary. Thus, the two occurrences of ``a.x`` do not - necessarily refer to the same attribute: if the RHS expression refers to a - class attribute, the LHS creates a new instance attribute as the target of the + necessarily refer to the same attribute: if the right-hand side expression refers to a + class attribute, the left-hand side creates a new instance attribute as the target of the assignment:: class Cls: @@ -329,10 +329,10 @@ Annotated assignment statements statement, of a variable or attribute annotation and an optional assignment statement: .. productionlist:: - annotated_assignment_stmt: `augtarget` ":" `expression` ["=" `expression`] + annotated_assignment_stmt: `augtarget` ":" `expression` + : ["=" (`starred_expression` | `yield_expression`)] -The difference from normal :ref:`assignment` is that only single target and -only single right hand side value is allowed. +The difference from normal :ref:`assignment` is that only single target is allowed. For simple names as assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module @@ -366,6 +366,11 @@ target, then the interpreter evaluates the target except for the last syntax for type annotations that can be used in static analysis tools and IDEs. +.. versionchanged:: 3.8 + Now annotated assignments allow same expressions in the right hand side as + the regular assignments. Previously, some expressions (like un-parenthesized + tuple expressions) caused a syntax error. + .. _assert: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 66317430881b44..8839033b983c47 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -23,7 +23,7 @@ from sphinx import addnodes from sphinx.builders import Builder from sphinx.locale import translators -from sphinx.util import status_iterator +from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter, TextTranslator @@ -151,6 +151,109 @@ def run(self): return [pnode] +# Support for documenting audit event + +class AuditEvent(Directive): + + has_content = True + required_arguments = 1 + optional_arguments = 2 + final_argument_whitespace = True + + _label = [ + "Raises an :ref:`auditing event ` {name} with no arguments.", + "Raises an :ref:`auditing event ` {name} with argument {args}.", + "Raises an :ref:`auditing event ` {name} with arguments {args}.", + ] + + @property + def logger(self): + cls = type(self) + return logging.getLogger(cls.__module__ + "." + cls.__name__) + + def run(self): + name = self.arguments[0] + if len(self.arguments) >= 2 and self.arguments[1]: + args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) + args = [a for a in args if a] + else: + args = [] + + label = translators['sphinx'].gettext(self._label[min(2, len(args))]) + text = label.format(name="``{}``".format(name), + args=", ".join("``{}``".format(a) for a in args if a)) + + env = self.state.document.settings.env + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + + new_info = { + 'source': [], + 'args': args + } + info = env.all_audit_events.setdefault(name, new_info) + if info is not new_info: + if not self._do_args_match(info['args'], new_info['args']): + self.logger.warn( + "Mismatched arguments for audit-event {}: {!r} != {!r}" + .format(name, info['args'], new_info['args']) + ) + + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = "audit_event_{}_{}".format( + re.sub(r'\W', '_', name), + len(info['source']), + ) + ids.append(target) + + info['source'].append((env.docname, target)) + + pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) + if self.content: + self.state.nested_parse(self.content, self.content_offset, pnode) + else: + n, m = self.state.inline_text(text, self.lineno) + pnode.extend(n + m) + + return [pnode] + + # This list of sets are allowable synonyms for event argument names. + # If two names are in the same set, they are treated as equal for the + # purposes of warning. This won't help if number of arguments is + # different! + _SYNONYMS = [ + {"file", "path", "fd"}, + ] + + def _do_args_match(self, args1, args2): + if args1 == args2: + return True + if len(args1) != len(args2): + return False + for a1, a2 in zip(args1, args2): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in self._SYNONYMS): + continue + return False + return True + + +class audit_event_list(nodes.General, nodes.Element): + pass + + +class AuditEventListDirective(Directive): + + def run(self): + return [audit_event_list('')] + + # Support for documenting decorators class PyDecoratorMixin(object): @@ -271,7 +374,7 @@ def run(self): translatable=False) node.append(para) env = self.state.document.settings.env - env.note_versionchange('deprecated', version[0], node, self.lineno) + env.get_domain('changeset').note_changeset(node) return [node] + messages @@ -355,7 +458,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) + self.env.logger.warn('label %r not in documentation' % label) continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -419,11 +522,73 @@ def parse_pdb_command(env, sig, signode): return fullname +def process_audit_events(app, doctree, fromdocname): + for node in doctree.traverse(audit_event_list): + break + else: + return + + env = app.builder.env + + table = nodes.table(cols=3) + group = nodes.tgroup( + '', + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + row = nodes.row() + row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) + head += row + + for name in sorted(getattr(env, "all_audit_events", ())): + audit_event = env.all_audit_events[name] + + row = nodes.row() + node = nodes.paragraph('', nodes.Text(name)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, a in enumerate(audit_event['args']): + if i: + node += nodes.Text(", ") + node += nodes.literal(a, nodes.Text(a)) + row += nodes.entry('', node) + + node = nodes.paragraph() + backlinks = enumerate(sorted(set(audit_event['source'])), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) + ref['refuri'] = "{}#{}".format( + app.builder.get_relative_uri(fromdocname, doc), + label, + ) + node += ref + row += nodes.entry('', node) + + body += row + + for node in doctree.traverse(audit_event_list): + node.replace_self(table) + + def setup(app): app.add_role('issue', issue_role) app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) + app.add_directive('audit-event', AuditEvent) + app.add_directive('audit-event-table', AuditEventListDirective) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) @@ -438,4 +603,5 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('doctree-resolved', process_audit_events) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 494efabc46234e..34a0112f5a007b 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -115,8 +115,8 @@ def write_doc(self, docname, doctree): def finish(self): unused_rules = [rule for rule in self.rules if not rule.used] if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) + self.logger.warn('Found %s/%s unused rules:' % + (len(unused_rules), len(self.rules))) for rule in unused_rules: self.logger.info(repr(rule)) return @@ -151,10 +151,10 @@ def report_issue(self, text, lineno, issue): self.any_issue = True self.write_log_entry(lineno, issue, text) if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) + self.logger.warn('[%s:%d] "%s" found in "%-.120s"' % + (self.docname, lineno, issue, text)) else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( + self.logger.warn('[%s:%d] "%s" found in "%-.120s"' % ( self.docname.encode(sys.getdefaultencoding(),'replace'), lineno, issue.encode(sys.getdefaultencoding(),'replace'), diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 346b31494e60f9..fa298a76b0fe10 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.9': 'dev (3.9)', + '3.8': 'pre (3.8)', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 3672955bf55b11..fcf556ec0daf2e 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -4,7 +4,7 @@ c-api/sequence,,:i2,del o[i1:i2] c-api/sequence,,:i2,o[i1:i2] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,267,`,This is the description of the ``foobar`` package. +distutils/examples,274,`,This is the description of the ``foobar`` package. distutils/setupscript,,::, extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" @@ -81,6 +81,7 @@ howto/ipaddress,,::,IPv6Address('2001:db8::ffff:ffff') howto/ipaddress,,:ffff,IPv6Address('2001:db8::ffff:ffff') howto/logging,,:And,"WARNING:And this, too" howto/logging,,:And,"WARNING:root:And this, too" +howto/logging,,:And,"ERROR:root:And non-ASCII stuff, too, like " howto/logging,,:Doing,INFO:root:Doing something howto/logging,,:Finished,INFO:root:Finished howto/logging,,:logger,severity:logger name:message @@ -90,6 +91,7 @@ howto/logging,,:root,DEBUG:root:This message should go to the log file howto/logging,,:root,INFO:root:Doing something howto/logging,,:root,INFO:root:Finished howto/logging,,:root,INFO:root:So should this +howto/logging,,:root,"ERROR:root:And non-ASCII stuff, too, like " howto/logging,,:root,INFO:root:Started howto/logging,,:root,"WARNING:root:And this, too" howto/logging,,:root,WARNING:root:Look before you leap! @@ -236,6 +238,8 @@ library/urllib.request,,:close,Connection:close library/urllib.request,,:port,:port library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" library/urllib.request,,:password,"""joe:password@python.org""" +library/urllib.parse,,:scheme, +library/urllib.parse,,:scheme,URL:scheme://host/path library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" @@ -335,7 +339,6 @@ library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" library/zipapp,,:fn,"pkg.mod:fn" library/zipapp,,:callable,"pkg.module:callable" library/stdtypes,,::,>>> m[::2].tolist() -library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' whatsnew/3.5,,:warning,'WARNING:root:warning\n' whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') @@ -350,3 +353,5 @@ whatsnew/3.7,,::,error::BytesWarning whatsnew/changelog,,::,error::BytesWarning whatsnew/changelog,,::,default::BytesWarning whatsnew/changelog,,::,default::DeprecationWarning +library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')" +library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path). diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 1a99b18bbb268e..d9364d6ced729f 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -12,8 +12,7 @@

Download Python {{ release }} Documentation

{% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %}

To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table. The numbers -in the table are the size of the download files in megabytes.

+Python in one of various formats, follow one of links in this table.

diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 3666af92f0da47..4fd7423430ca81 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,7 +2,8 @@

{% trans %}Download{% endtrans %}

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

FormatPacked as .zipPacked as .tar.bz2