Add _dbg support for Windows, add more in-build testing (needs a new …

…conda I fear due to using conda run there)

recipe/bld.bat

@@ -30,32 +30,66 @@ if "%PY_INTERP_DEBUG%" neq "" (
 )
 
 
-if "%DEBUG_C%"=="yes" (
+if "%PY_INTERP_DEBUG%"=="yes" (
   set PGO=
 ) else (
-  set PGO=--pgo
+  if "%DEBUG_C%"=="yes" (
+    set PGO=
+  ) else (
+    set PGO=--pgo
+  )
 )
 
-:: AP doesn't support PGO atm?
-set PGO=
-
 cd PCbuild
 
+:: Doesn't avoid the SDK problem.
+:: devenv /upgrade pcbuild.sln
+:: set __VCVARS_VERSION=%WindowsSDKVer%
+:: 14.16.27023
+:: echo ^<?xml version="1.0"?^> > my_props.props
+:: echo ^<PropertyGroup Label="Configuration"^> >> my_props.props
+:: echo    ^<VCToolsVersion /^> >> my_props.props
+:: echo    ^<PlatformToolset^>v141^</PlatformToolset^> >> my_props.props
+:: echo ^</PropertyGroup^> >> my_props.props
+:: type my_props.props
+:: call build.bat %PGO% %CONFIG% -m -e -v -p %PLATFORM% "/p:ForceImportBeforeCppTargets=%CD%\my_props.props" "/p:ForceImportAfterCppTargets=%CD%\my_props.props"
+
+:: Twice because I am changing zipimport ATM.
+call build.bat %PGO% %CONFIG% -m -e -v -p %PLATFORM%
 call build.bat %PGO% %CONFIG% -m -e -v -p %PLATFORM%
 if errorlevel 1 exit 1
 cd ..
 
 :: Populate the root package directory
 for %%x in (python38%_D%.dll python3%_D%.dll python%_D%.exe pythonw%_D%.exe venvlauncher%_D%.exe venvwlauncher%_D%.exe) do (
+    echo Copying: %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x to %PREFIX%
     copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x %PREFIX%
     if errorlevel 1 exit 1
 )
 
+:: If _d appears anywhere other than at the end of the filename then this will break.
+if "%_D%"=="_d" (
+  for %%x in (python38%_D%.dll python3%_D%.dll python%_D%.exe pythonw%_D%.exe venvlauncher%_D%.exe venvwlauncher%_D%.exe) do (
+    set _TMP=%%x
+    call set _DST=%%_TMP:_D=%%
+    echo Copying: %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x to !_DST!
+    copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x %PREFIX%\!_DST!
+    if errorlevel 1 exit 1
+  )
+)
+
 for %%x in (python%_D%.pdb python38%_D%.pdb pythonw%_D%.pdb) do (
+    echo Copying: %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x %PREFIX%
     copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x %PREFIX%
     if errorlevel 1 exit 1
 )
 
+for %%x in (*.pdb) do (
+    echo Copying PDB: %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x to %PREFIX%\DLLs
+    copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\%%x %PREFIX%\DLLs
+    if errorlevel 1 exit 1
+)
+
 copy %SRC_DIR%\LICENSE %PREFIX%\LICENSE_PYTHON.txt
 if errorlevel 1 exit 1
 
@@ -139,7 +173,14 @@ copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\python3%_D%.lib %PREFIX%\libs\
 if errorlevel 1 exit 1
 copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\_tkinter%_D%.lib %PREFIX%\libs\
 if errorlevel 1 exit 1
-
+if "%_D%"=="_d" (
+  copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\python38%_D%.lib %PREFIX%\libs\python38.lib
+  if errorlevel 1 exit 1
+  copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\python3%_D%.lib %PREFIX%\libs\python3.lib
+  if errorlevel 1 exit 1
+  copy /Y %SRC_DIR%\PCbuild\%BUILD_PATH%\_tkinter%_D%.lib %PREFIX%\libs\_tkinter.lib
+  if errorlevel 1 exit 1
+)
 
 :: Populate the Lib directory
 del %PREFIX%\libs\libpython*.a
@@ -166,30 +207,39 @@ if errorlevel 1 exit 1
 rd /s /q %PREFIX%\Lib\lib2to3\tests\
 if errorlevel 1 exit 1
 
-:: We need our Python to be found!
+:: We need our Python to be found
 if "%_D%" neq "" copy %PREFIX%\python%_D%.exe %PREFIX%\python.exe
+if "%_D%" neq "" copy %PREFIX%\python%_D%.pdb %PREFIX%\python.pdb
 
 %PREFIX%\python.exe -Wi %PREFIX%\Lib\compileall.py -f -q -x "bad_coding|badsyntax|py2_" %PREFIX%\Lib
 if errorlevel 1 exit 1
 
 :: Pickle lib2to3 Grammar
 %PREFIX%\python.exe -m lib2to3 --help
 
-:: Some quick tests for common failures
-echo "Testing print() does not print: Hello"
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe -c "print()" 2>&1 | findstr /r /c:"Hello"
+echo CONDA_EXE is %CONDA_EXE%
+echo where conda is
+where conda
+echo where python is
+where python
+
+echo "Testing print() does not print Hello"
+conda run -p %PREFIX% python -c "print()" 2>&1 | findstr /r /c:"Hello"
 if %errorlevel% neq 1 exit /b 1
 
-echo "Testing print('Hello') prints: Hello"
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe "print('Hello')" 2>&1 | findstr /r /c:"Hello"
+echo "Testing print('Hello') prints Hello"
+conda run -p %PREFIX% python -c "print('Hello')" 2>&1 | findstr /r /c:"Hello"
 if %errorlevel% neq 0 exit /b 1
 
-echo "Testing import of os (no DLL needed) does not print: The specified module could not be found"
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe -v -c "import os" 2>&1
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe -v -c "import os" 2>&1 | findstr /r /c:"The specified module could not be found"
+echo "Testing import of os does not print The specified module could not be found"
+conda run -p %PREFIX% python -v -c "import os" 2>&1
+conda run -p %PREFIX% python -v -c "import os" 2>&1 | findstr /r /c:"The specified module could not be found"
 if %errorlevel% neq 1 exit /b 1
 
-echo "Testing import of _sqlite3 (DLL located via PATH needed) does not print: The specified module could not be found"
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe -v -c "import _sqlite3" 2>&1
-%CONDA_EXE% run -p %PREFIX% cd %PREFIX% & %PREFIX%\python.exe -v -c "import _sqlite3" 2>&1 | findstr /r /c:"The specified module could not be found"
+:: echo "Waiting for 60 seconds. Recommend you run procmon to figure out why the impeding import of _sqlite3 fails (on Win 32, python 3.7 building 3.8)"
+:: waitfor SomethingThatIsNeverHappening /t 60 2>NUL
+
+echo "Testing import of _sqlite3 prints The specified module could not be found"
+conda run -p %PREFIX% python -v -c "import _sqlite3" 2>&1
+conda run -p %PREFIX% python -v -c "import _sqlite3" 2>&1 | findstr /r /c:"The specified module could not be found"
 if %errorlevel% neq 1 exit /b 1

recipe/build.sh

@@ -224,11 +224,13 @@ if [[ ${_OPTIMIZED} == yes ]]; then
   _MAKE_TARGET=profile-opt
   # To speed up build times during testing (1):
   if [[ ${QUICK_BUILD} == yes ]]; then
-    # TODO :: It seems this is just profiling everything, on Windows, only 40 odd tests are
-    #         run while on Unix, all 400+ are run, making this slower and less well curated
     _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo")
   else
-    _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo-extended")
+    # TODO :: Run some benchmarks on these to see which is better.
+    #      :: We decided to go with the faster-to-build option for now
+    #      :: --pgo-extended runs 10 times as many tests as --pgo does.
+    # _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo-extended")
+    _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo")
   fi
   if [[ ${CC} =~ .*gcc.* ]]; then
     LTO_CFLAGS+=(-fuse-linker-plugin)

recipe/meta.yaml

@@ -1,12 +1,12 @@
 {% set base_version = "3.8.1" %}
 {% set version = base_version %}
 {% set linkage_nature = os.environ.get('PY_INTERP_LINKAGE_NATURE', '') %}
-{% set debug = os.environ.get('PY_INTERP_DEBUG', '') %}
 {% if linkage_nature != '' %}
   {% set linkage_nature = "_" ~ linkage_nature %}
 {% endif %}
+{% set debug = os.environ.get('PY_INTERP_DEBUG', '') %}
 {% if debug != '' %}
-  {% set debug = "_d" ~ debug %}
+  {% set debug = "_dbg" %}
 {% endif %}
 
 package:
@@ -44,6 +44,8 @@ source:
       - patches/0021-roll-back-2e33ecd-for-py_compile.main.patch
       # Python 3.8 is pretty broken without this.
       - patches/0022-Add-CondaEcosystemModifyDllSearchPath.patch
+      - patches/0025-egg-debugging-with-Windows-debug-builds.patch
+      - patches/0026-Revert-part-of-https-bugs.python.org-issue33895-http.patch
 
   # TODO :: Depend on our own packages for these:
   - url: https://github.com/python/cpython-source-deps/archive/xz-5.2.2.zip          # [win]
@@ -78,7 +80,7 @@ source:
 
 
 build:
-  number: 3
+  number: 8
   # Windows has issues updating python if conda is using files itself.
   # Copy rather than link.
   no_link:
@@ -99,10 +101,9 @@ build:
   #   - lib/libpython3.dylib  # [osx]
   # match python.org compiler standard
   skip: True            # [win and int(float(vc)) < 14]
-# Would like to be able to append to the calculated build string here, ping @msarahan, is this possible?
-  string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_cpython
+  string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}_cpython{{ debug }}
   run_exports:
-   - python_abi 3.8.* cp38
+   - python_abi 3.8.* cp38{{ debug }}
   script_env:
    - PY_INTERP_LINKAGE_NATURE
    - PY_INTERP_DEBUG
@@ -123,12 +124,12 @@ requirements:
     - make  # [not win]
     - pkg-config  # [not win]
     # configure script looks for llvm-ar for lto
+{% if 'conda-forge' in channel_targets %}
     - llvm-tools  # [osx]
+{% endif %}
     - patch  # [not win]
     - m2-patch  # [win]
-    - posix  # [win]
-    - ld_impl_{{target_platform}}  # [linux]
-    - binutils_impl_{{target_platform}} >=2.33.1  # [linux]
+    - m2-gcc-libs  # [win]
   host:
     - bzip2  # [not win]
     - sqlite
@@ -137,13 +138,21 @@ requirements:
     - openssl
     - readline  # [not win]
     - tk  # [not win]
-    - ncurses  # [linux]
+    - ncurses  # [not win]
     - libffi  # [not win]
+{% if 'conda-forge' not in channel_targets %}
+    - binutils_impl_{{target_platform}}  # [linux and aarch64]
+    - ld_impl_{{target_platform}}  # [linux and not aarch64]
+{% else %}
     - ld_impl_{{target_platform}}  # [linux]
+{% endif %}
   run:
+{% if 'conda-forge' not in channel_targets %}
+    - binutils_impl_{{target_platform}}  # [linux and aarch64]
+    - ld_impl_{{target_platform}}  # [linux and not aarch64]
+{% else %}
     - ld_impl_{{target_platform}}  # [linux]
-  run_constrained:
-    - python_abi 3.8.* cp38
+{% endif %}
 
 test:
   requires:
@@ -178,15 +187,18 @@ test:
     -     python -c "import foobar"
     -   popd
     -   pushd distutils.cext
-    -     python setup.py install -v -v
-    -     python -c "import greet; greet.greet('Python user')" | rg "Hello Python"
+    # --single-version-externally-managed because .egg files with compiled code do not work on Windows (Python 3.8) when Py_DEBUG (this should be investigated)
+    -     python setup.py install -v -v  # [(not win) or environ.get('PY_INTERP_DEBUG') == '']
+    -     python setup.py install --single-version-externally-managed --record=record.txt -v -v  # [win and environ.get('PY_INTERP_DEBUG') != '']
+    -     python -v -v -v -c "import greet"
+    -     python -v -v -v -c "import greet; greet.greet('Python user')" | rg "Hello Python"
     -   popd
     -   pushd prefix-replacement  # [unix]
     -     bash build-and-test.sh  # [unix]
     -   popd  # [unix]
     -   pushd cmake
-    -     cmake -GNinja -DPY_VER={{ version }}
-          # --trace --debug-output --debug-trycompile .
+    -     cmake -GNinja -DPY_VER={{ version }} .
+          # --trace --debug-output --debug-trycompile
     -   popd
     - popd
 

recipe/patches/0022-Add-CondaEcosystemModifyDllSearchPath.patch

@@ -1,21 +1,23 @@
-From 6c67f1126dc91ad94f735b20e2dda4bb7c54f472 Mon Sep 17 00:00:00 2001
+From bd510e850256e4711b3e7d9a2ba089f32d3a0eea Mon Sep 17 00:00:00 2001
 From: Ray Donnelly <mingw.android@gmail.com>
 Date: Tue, 24 Dec 2019 18:37:17 +0100
-Subject: [PATCH 22/22] Add CondaEcosystemModifyDllSearchPath()
+Subject: [PATCH 22/24] Add CondaEcosystemModifyDllSearchPath()
 
 This changes the DLL search order so that C:\Windows\System32 does not
 get searched in before entries in PATH.
 
 Reviewed by Kai Tietz 7.2.2019
 
 Updated a bit to include other directories.
+
+Made fwprintfs breakpointable
 ---
- Modules/main.c       | 384 +++++++++++++++++++++++++++++++++++++++++++
+ Modules/main.c       | 391 +++++++++++++++++++++++++++++++++++++++++++
  Python/dynload_win.c |   7 +-
- 2 files changed, 389 insertions(+), 2 deletions(-)
+ 2 files changed, 396 insertions(+), 2 deletions(-)
 
 diff --git a/Modules/main.c b/Modules/main.c
-index 0d2cf3ab82..a8cd6dd89a 100644
+index 0d2cf3ab82..fcf9811aaf 100644
 --- a/Modules/main.c
 +++ b/Modules/main.c
 @@ -20,6 +20,10 @@
@@ -38,7 +40,7 @@ index 0d2cf3ab82..a8cd6dd89a 100644
  /* --- pymain_init() ---------------------------------------------- */
 
  static PyStatus
-@@ -658,10 +664,388 @@ Py_RunMain(void)
+@@ -658,10 +664,395 @@ Py_RunMain(void)
      return exitcode;
  }
 
@@ -275,7 +277,8 @@ index 0d2cf3ab82..a8cd6dd89a 100644
 +            {
 +                if (_wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_NEVER_CACHE") == NULL)
 +                {
-+                    if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: INFO :: Values unchanged\n");
++                    if (debug_it)
++                        fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: INFO :: Values unchanged\n");
 +                    return 0;
 +                }
 +            }
@@ -295,7 +298,8 @@ index 0d2cf3ab82..a8cd6dd89a 100644
 +    {
 +        size_t len = wcslen(path_env);
 +        wchar_t *path = (wchar_t *)alloca((len + 1) * sizeof(wchar_t));
-+        if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: PATH=%ls\n\b", path_env);
++        if (debug_it)
++            fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: PATH=%ls\n\b", path_env);
 +        memcpy(path, path_env, (len + 1) * sizeof(wchar_t));
 +        /* Convert any / to \ */
 +        /* Replace slash with backslash */
@@ -364,7 +368,8 @@ index 0d2cf3ab82..a8cd6dd89a 100644
 +                }
 +            }
 +#endif /* defined(HARDCODE_CONDA_PATHS) */
-+            if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls)\n", path_entries[i]);
++            if (debug_it)
++                fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls)\n", path_entries[i]);
 +            pAddDllDirectory(path_entries[i]);
 +        }
 +    }
@@ -374,10 +379,12 @@ index 0d2cf3ab82..a8cd6dd89a 100644
 +    {
 +        for (j = NUM_CONDA_PATHS-1, p_conda_path = &condaPaths[NUM_CONDA_PATHS-1]; j > -1; --j, --p_conda_path)
 +        {
-+            if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: p_conda_path->p_name = %ls, foundCondaPath[%zd] = %d\n", p_conda_path->p_name, j, foundCondaPath[j]);
++            if (debug_it)
++                fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: p_conda_path->p_name = %ls, foundCondaPath[%zd] = %d\n", p_conda_path->p_name, j, foundCondaPath[j]);
 +            if (!foundCondaPath[j])
 +            {
-+                if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - ExePrefix)\n", p_conda_path->p_name);
++                if (debug_it)
++                    fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - ExePrefix)\n", p_conda_path->p_name);
 +                pAddDllDirectory(p_conda_path->p_name);
 +            }
 +        }
@@ -391,14 +398,16 @@ index 0d2cf3ab82..a8cd6dd89a 100644
 +            free(sv_added_cwd);
 +        }
 +        sv_added_cwd = _wcsdup(p_cwd);
-+        if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - CWD)\n", sv_added_cwd);
++        if (debug_it)
++            fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - CWD)\n", sv_added_cwd);
 +        pAddDllDirectory(sv_added_cwd);
 +    }
 +
 +    if (add_windows_directory)
 +    {
 +        sv_added_windows_directory = &sv_windows_directory[0];
-+        if (debug_it) fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - WinDir)\n", sv_windows_directory);
++        if (debug_it)
++            fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - WinDir)\n", sv_windows_directory);
 +        pAddDllDirectory(sv_windows_directory);
 +    }
 +    else
@@ -448,5 +457,5 @@ index 6deba1134e..56d8913455 100644
  #if HAVE_SXS
          _Py_DeactivateActCtx(cookie);
 -- 
-2.24.0
+2.25.0