From 2aa8cf82b91f52fde1083ff87f254efaec433255 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Sat, 22 Jul 2023 17:33:51 +0300 Subject: [PATCH] Long path and Unicode support on Windows 10.0.1607/10.0.1903+ (#316) Long path support is still not universal, even after an app has explicitly opted-in via manifest it also requires registry modification or group policies, see https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later though Python and Git installers nudges to enable it by their installers for a while. `-X utf8` forces Python to use UTF-8 when its outputs are piped (will be default in 3.15 https://peps.python.org/pep-0686/) Also had to fix clang-linux and msvc toolsets manifest embedding issues. --- .ci/azp-windows-test.yml | 2 +- .github/workflows/core_tests.yml | 1 + Jamroot.jam | 4 ++ appveyor.yml | 3 +- src/engine/b2.exe.manifest | 22 +++++++++++ src/engine/build.sh | 4 ++ src/engine/config_toolset.bat | 37 +++++++++++------ src/engine/jam.cpp | 43 ++++++++++++++++++++ src/engine/res.rc | 1 + src/tools/clang-linux.jam | 18 +++++++++ src/tools/msvc.jam | 1 + test/BoostBuild.py | 24 +++++++---- test/path_specials.py | 68 ++++++++++++++++++++++++++++++++ test/space_in_path.py | 51 ------------------------ test/test_all.py | 2 +- 15 files changed, 208 insertions(+), 73 deletions(-) create mode 100644 src/engine/b2.exe.manifest create mode 100644 src/engine/res.rc create mode 100644 test/path_specials.py delete mode 100755 test/space_in_path.py diff --git a/.ci/azp-windows-test.yml b/.ci/azp-windows-test.yml index c718287d50..037aeccdb4 100644 --- a/.ci/azp-windows-test.yml +++ b/.ci/azp-windows-test.yml @@ -14,7 +14,7 @@ steps: $env:path += ';' + $env:CXX_PATH cd test echo "using" $env:TEST_TOOLSET ":" ":" $env:CXX ";" > ${env:HOME}/user-config.jam - py test_all.py $env:TEST_TOOLSET + py -X utf8 test_all.py $env:TEST_TOOLSET cd .. displayName: Test - powershell: | diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index 53a5f333f2..ca53249c88 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -65,6 +65,7 @@ jobs: - name: Test working-directory: test + env: { PYTHONUTF8: 1 } run: ./test_all.py ${{matrix.toolset}} - name: 'No Warnings' diff --git a/Jamroot.jam b/Jamroot.jam index 3ad5426a89..5aef6865f9 100644 --- a/Jamroot.jam +++ b/Jamroot.jam @@ -150,6 +150,10 @@ exe b2 clang-win:kernel32 clang-win:advapi32 clang-win:user32 + windows,gcc:src/engine/res.rc + windows,clang:src/engine/res.rc + msvc:src/engine/b2.exe.manifest + clang-win:src/engine/b2.exe.manifest ; explicit b2 ; diff --git a/appveyor.yml b/appveyor.yml index e8a9af2c7c..6b8233e978 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,6 +32,7 @@ environment: job_group: 'Test' TOOLSET: clang-win TEST_TOOLSET: clang-win + B2_DONT_EMBED_MANIFEST: true # lld-link: error: unable to find mt.exe in PATH: no such file or directory APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - job_name: 'Cygwin 3.1.7 x64, Test' job_group: 'TestCygwin' @@ -61,7 +62,7 @@ for: echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> TEST" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" cd test - py test_all.py %TEST_TOOLSET% + py -X utf8 test_all.py %TEST_TOOLSET% cd .. - cmd: | echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" diff --git a/src/engine/b2.exe.manifest b/src/engine/b2.exe.manifest new file mode 100644 index 0000000000..778e24f072 --- /dev/null +++ b/src/engine/b2.exe.manifest @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + true + UTF-8 + + + diff --git a/src/engine/build.sh b/src/engine/build.sh index 2d332f1484..36535e522f 100755 --- a/src/engine/build.sh +++ b/src/engine/build.sh @@ -503,6 +503,10 @@ mod_version.cpp \ if test_true ${B2_DEBUG_OPT} ; then B2_CXXFLAGS="${B2_CXXFLAGS_DEBUG}" else B2_CXXFLAGS="${B2_CXXFLAGS_RELEASE} -DNDEBUG" fi + if [ -z "$B2_DONT_EMBED_MANIFEST" ] && [ -x "$(command -v windres)" ] ; then + B2_CXXFLAGS="${B2_CXXFLAGS} -Wl,res.o" + ( B2_VERBOSE_OPT=${TRUE} echo_run windres --input res.rc --output res.o ) + fi ( B2_VERBOSE_OPT=${TRUE} echo_run ${B2_CXX} ${B2_CXXFLAGS} ${B2_SOURCES} -o b2 ) } diff --git a/src/engine/config_toolset.bat b/src/engine/config_toolset.bat index c6afbb6c76..4ba577cacd 100644 --- a/src/engine/config_toolset.bat +++ b/src/engine/config_toolset.bat @@ -39,7 +39,7 @@ if not "_%B2_TOOLSET_ROOT%_" == "__" ( set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC11 if not defined CXX ( set "CXX=cl" ) @@ -54,7 +54,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" ( set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC12 if not defined CXX ( set "CXX=cl" ) @@ -73,7 +73,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" ( set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC14 if not defined CXX ( set "CXX=cl" ) @@ -93,7 +93,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" ( set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC141 if not defined CXX ( set "CXX=cl" ) @@ -115,7 +115,7 @@ popd set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC142 if not defined CXX ( set "CXX=cl" ) @@ -137,7 +137,7 @@ popd set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VC143 if not defined CXX ( set "CXX=cl" ) @@ -160,7 +160,7 @@ popd set "B2_CXX="%CXX%" /nologo -TP /wd4996 /wd4675 /EHs /GR /Zc:throwingNew /O2 /Ob2 /W3 /MD /Zc:forScope /Zc:wchar_t /Zc:inline /Gw /favor:blend /Feb2" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_VCUNK if NOT "_%B2_TOOLSET%_" == "_vcunk_" goto Skip_VCUNK @@ -182,7 +182,7 @@ popd set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_BORLAND if not defined CXX ( set "CXX=bcc32c" ) @@ -209,13 +209,13 @@ goto :eof if not defined CXX ( set "CXX=g++" ) set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -o b2.exe -D_GNU_SOURCE" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Windres :Config_CLANG if not defined CXX ( set "CXX=clang++" ) set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -o b2.exe" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Windres :Config_CLANG_WIN if not defined CXX ( set "CXX=clang-cl" ) @@ -224,10 +224,10 @@ if "_%ProgramFiles(x86)%_" == "__" ( ) else ( set "PATH=%PATH%;%ProgramFiles%\LLVM\bin;%ProgramFiles(x86)%\LLVM\bin" ) -set "B2_CXX="%CXX%" /MT /TP /Feb2 /wd4996 /O2 /EHsc /Gw /Zc:inline" +set "B2_CXX="%CXX%" /MT /TP /Feb2 /wd4996 /O2 /EHsc /Gw /Zc:inline -fuse-ld=lld" set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" set "_known_=1" -goto :eof +goto :Embed_Minafest_Via_Link :Config_GCC_NOCYGWIN if not defined CXX ( set "CXX=g++" ) @@ -249,4 +249,17 @@ if not "_%B2_TOOLSET_ROOT%_" == "__" ( for /F "delims=" %%I in ("%CXX%") do set "PATH=%PATH%;%%~dpI" set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -static -o b2.exe" set "_known_=1" +goto :Embed_Minafest_Via_Windres + +:Embed_Minafest_Via_Link +if not defined B2_DONT_EMBED_MANIFEST ( + set "B2_CXX_LINK=%B2_CXX_LINK% /MANIFEST:EMBED /MANIFESTINPUT:b2.exe.manifest" +) +goto :eof + +:Embed_Minafest_Via_Windres +if not defined B2_DONT_EMBED_MANIFEST ( + where windres >NUL 2>NUL + if %ERRORLEVEL% NEQ 0 ( call; ) else ( set "B2_CXX=windres --input res.rc --output res.o && %B2_CXX% -Wl,res.o" ) +) goto :eof diff --git a/src/engine/jam.cpp b/src/engine/jam.cpp index 4d428d7168..10afcdd600 100644 --- a/src/engine/jam.cpp +++ b/src/engine/jam.cpp @@ -152,6 +152,11 @@ # include #endif +#ifdef WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + struct globs globs = { 0, /* noexec */ @@ -614,6 +619,44 @@ int guarded_main( int argc, char * * argv ) return status ? EXITBAD : EXITOK; } +#ifdef WIN32 +namespace { + +struct SetConsoleCodepage +{ + SetConsoleCodepage() + { + // Check whether UTF-8 is actually the default encoding for this process + if (GetACP() != CP_UTF8) + return; + + orig_console_cp = GetConsoleCP(); + if (orig_console_cp != 0 && orig_console_cp != CP_UTF8) + SetConsoleCP(CP_UTF8); + orig_console_output_cp = GetConsoleOutputCP(); + if (orig_console_output_cp != 0 && orig_console_output_cp != CP_UTF8) + SetConsoleOutputCP(CP_UTF8); + } + + ~SetConsoleCodepage() + { + // Restore original console codepage + if (orig_console_cp != 0 && orig_console_cp != CP_UTF8) + SetConsoleCP(orig_console_cp); + if (orig_console_output_cp != 0 && orig_console_output_cp != CP_UTF8) + SetConsoleOutputCP(orig_console_output_cp); + } + +private: + UINT orig_console_cp = 0; + UINT orig_console_output_cp = 0; +}; + +static const SetConsoleCodepage g_console_codepage_setter{}; + +} +#endif + int main( int argc, char * * argv ) { BJAM_MEM_INIT(); diff --git a/src/engine/res.rc b/src/engine/res.rc new file mode 100644 index 0000000000..6e07915793 --- /dev/null +++ b/src/engine/res.rc @@ -0,0 +1 @@ +1 24 b2.exe.manifest diff --git a/src/tools/clang-linux.jam b/src/tools/clang-linux.jam index b389ff3364..26891f4b60 100644 --- a/src/tools/clang-linux.jam +++ b/src/tools/clang-linux.jam @@ -21,6 +21,7 @@ import type ; import numbers ; import os ; import property ; +import rc ; import set ; feature.extend-subfeature toolset clang : platform : linux ; @@ -86,6 +87,23 @@ rule init ( version ? : command * : options * ) { archiver = $(root)/bin/ar ; } toolset.flags clang-linux.archive .AR $(condition) : $(archiver[1]) ; + + # - Resource compiler. + local rc = [ common.get-invocation-command-nodefault clang-linux : windres : + [ feature.get-values : $(options) ] : $(bin) : search-path ] ; + local rc-type = [ feature.get-values : $(options) ] ; + rc-type ?= windres ; + if ! $(rc) + { + # If we can not find an RC compiler we fallback to a null one that + # creates empty object files. This allows the same Jamfiles to work + # across the board. The null RC uses assembler to create the empty + # objects, so configure that. + rc = [ common.get-invocation-command clang-linux : as : : $(bin) : search-path ] + ; + rc-type = null ; + } + rc.configure $(rc) : $(condition) : $(rc-type) ; } rule get-full-version ( command-string ) diff --git a/src/tools/msvc.jam b/src/tools/msvc.jam index 2e481d4f06..ee8637a414 100644 --- a/src/tools/msvc.jam +++ b/src/tools/msvc.jam @@ -2000,6 +2000,7 @@ local rule register-toolset-really ( ) toolset.flags msvc.link LINKFLAGS on : /DEBUG ; toolset.flags msvc.link DEF_FILE ; toolset.flags msvc.link MANIFEST_FILE linker : ; + toolset.flags msvc.link EMBED_MANIFEST_FILE mt : ; # The linker disables the default optimizations when using /DEBUG so we # have to enable them manually for release builds with debug symbols. diff --git a/test/BoostBuild.py b/test/BoostBuild.py index ea72da5a27..74239d6aaf 100644 --- a/test/BoostBuild.py +++ b/test/BoostBuild.py @@ -332,6 +332,12 @@ def __init__(self, arguments=None, executable=None, os.chdir(self.workdir) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.cleanup() + def cleanup(self): try: TestCmd.TestCmd.cleanup(self) @@ -383,11 +389,13 @@ def write(self, file, content, wait=True): self.__makedirs(os.path.dirname(nfile), wait) if not type(content) == bytes: content = content.encode() - f = open(nfile, "wb") try: - f.write(content) - finally: - f.close() + with open(nfile, "wb") as f: + f.write(content) + except Exception as e: + annotation("failure", f"Could not create file '{nfile}': {e}") + annotate_stack_trace(level=3) + self.fail_test(1) self.__ensure_newer_than_last_build(nfile) def rename(self, src, dst): @@ -1049,9 +1057,11 @@ def __makedirs(self, path, wait): os.mkdir(path) self.__ensure_newer_than_last_build(path) else: - os.makedirs(path) - except Exception: - pass + os.makedirs(path, exist_ok=True) + except Exception as e: + annotation("failure", f"Could not create path '{path}': {e}") + annotate_stack_trace(level=3) + self.fail_test(1) def __python_timestamp_resolution(self, path, minimum_resolution): """ diff --git a/test/path_specials.py b/test/path_specials.py new file mode 100644 index 0000000000..b317790d5a --- /dev/null +++ b/test/path_specials.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2012 Steven Watanabe +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt) + +# Test that paths containing spaces, unicode, and very long paths +# are handled correctly by actions. + +import BoostBuild +import os +import os.path +import sys +import tempfile +from unittest.mock import patch + + +def test_dir(dir_name): + with BoostBuild.Tester(use_test_config=False) as t: + do_compile_test = True + tmpdir = t.workpath(dir_name) + tmp = {'TMP': tmpdir, 'TMPDIR': tmpdir} + # cmd.exe hangs in a busy loop when TMP is a long path + if os.name == 'nt' and len(tmpdir) > 256: + tmp = {} + # cl.exe and link.exe still does not support long paths + # clang-cl does support long path, but it uses link.exe by default + if t.toolset.startswith('msvc') or t.toolset.startswith('clang-win'): + do_compile_test = False + # on windows gcc doesn't support long path, ld doesn't support neither unicode nor long path + if os.environ.get('MSYSTEM') in ['UCRT64', 'MINGW64', 'MINGW32'] and t.toolset in ['gcc', 'clang']: + do_compile_test = False + + t.write(f'{dir_name}/jamroot.jam', '''\ +import testing ; +actions write-file +{ + @(STDOUT:E=okay) >"$(<)" +} +make test.txt : : @write-file ; +''' + ('''\ +unit-test test : test.cpp ; +''' if do_compile_test else '')) + t.write(f'{dir_name}/test.cpp', 'int main() {}\n') + + with patch.dict(os.environ, tmp): + t.run_build_system([dir_name]) + t.expect_addition(f'{dir_name}/bin/test.txt') + if do_compile_test: + t.expect_addition(f'{dir_name}/bin/$toolset/debug*/test.passed') + + +test_dir('has space') +# Windows versions are a huge mess, like in any other Microsoft product: +# Windows 10, version 1903 codename 19H1, build number 10.0.18362 +# Windows 10, version 1607 codename Redstone 1, build number 10.0.14393 +if not hasattr(sys, 'getwindowsversion') or sys.getwindowsversion()[:3] >= (10, 0, 18362): + test_dir('uni\u2665code') +if not hasattr(sys, 'getwindowsversion') or sys.getwindowsversion()[:3] >= (10, 0, 14393): + long_path = '/'.join(['a_very_very_long_path'] * (260 // 21 + 1)) + try: + with tempfile.TemporaryDirectory() as path: + os.makedirs(os.path.join(path, long_path)) + except WindowsError as e: + if e.winerror != 206: # ERROR_FILENAME_EXCED_RANGE + raise + else: + test_dir(long_path) diff --git a/test/space_in_path.py b/test/space_in_path.py deleted file mode 100755 index e929778e7b..0000000000 --- a/test/space_in_path.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2012 Steven Watanabe -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt) - -# Test that paths containing spaces are handled correctly by actions. - -import BoostBuild -import os - -t = BoostBuild.Tester(use_test_config=False) - -t.write("has space/jamroot.jam", """\ -import testing ; -unit-test test : test.cpp ; -actions write-file -{ - @(STDOUT:E=okay) >"$(<)" -} -make test.txt : : @write-file ; -""") -t.write("has space/test.cpp", "int main() {}\n") - -tmpdir = t.workpath("has space") -try: - oldtmp = os.environ["TMP"] -except: - oldtmp = None -try: - oldtmpdir = os.environ["TMPDIR"] -except: - oldtmpdir = None -os.environ["TMP"] = tmpdir; # Windows -os.environ["TMPDIR"] = tmpdir; # *nix - -try: - t.run_build_system(["has space"]) - t.expect_addition("has space/bin/test.txt") - t.expect_addition("has space/bin/$toolset/debug*/test.passed") -finally: - if oldtmp is not None: - os.environ["TMP"] = oldtmp - else: - del os.environ["TMP"] - if oldtmpdir is not None: - os.environ["TMPDIR"] = oldtmpdir - else: - del os.environ["TMPDIR"] - -t.cleanup() diff --git a/test/test_all.py b/test/test_all.py index 43b71361cf..73e8a7225c 100755 --- a/test/test_all.py +++ b/test/test_all.py @@ -337,6 +337,7 @@ def reorder_tests(tests, first_test): "package", "param", "path_features", + "path_specials", "prebuilt", "preprocessor", "print", @@ -362,7 +363,6 @@ def reorder_tests(tests, first_test): "sort_rule", "source_locations", "source_order", - "space_in_path", "stage", "standalone", "static_and_shared_library",