diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0a370e6..16850583 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: - name: Checkout source uses: actions/checkout@v4.1.1 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3.0.0 if: runner.os == 'Linux' && matrix.arch != 'auto' with: platforms: all @@ -183,7 +183,7 @@ jobs: arch: auto - python-version: '3.12' install-extras: tests - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.12' install-extras: tests @@ -191,67 +191,67 @@ jobs: arch: auto - python-version: '3.6' install-extras: tests,optional - os: windows-latest + os: ubuntu-20.04 arch: auto - python-version: '3.7' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.8' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.9' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.10' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.11' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.12' install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - - python-version: pypy-3.7 + - python-version: pypy-3.9 install-extras: tests,optional - os: windows-latest + os: ubuntu-latest arch: auto - python-version: '3.6' install-extras: tests,optional - os: windows-latest + os: macos-13 arch: auto - python-version: '3.7' install-extras: tests,optional - os: windows-latest + os: macos-13 arch: auto - python-version: '3.8' install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.9' install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.10' install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.11' install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.12' install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - - python-version: pypy-3.7 + - python-version: pypy-3.9 install-extras: tests,optional - os: windows-latest + os: macOS-latest arch: auto - python-version: '3.6' install-extras: tests,optional @@ -281,19 +281,7 @@ jobs: install-extras: tests,optional os: windows-latest arch: auto - - python-version: pypy-3.7 - install-extras: tests,optional - os: windows-latest - arch: auto - - python-version: pypy-3.7 - install-extras: tests,optional - os: windows-latest - arch: auto - - python-version: pypy-3.7 - install-extras: tests,optional - os: windows-latest - arch: auto - - python-version: pypy-3.7 + - python-version: pypy-3.9 install-extras: tests,optional os: windows-latest arch: auto @@ -304,7 +292,7 @@ jobs: uses: ilammy/msvc-dev-cmd@v1 if: matrix.os == 'windows-latest' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3.0.0 if: runner.os == 'Linux' && matrix.arch != 'auto' with: platforms: all diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fbd1ffb..bc038112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,21 @@ This project (loosely) adheres to [Semantic Versioning](https://semver.org/spec/ ## Version 1.3.6 - +### Added: +* Add `ub.IndexableWalker.diff` + ### Fixed: * `ub.import_module_from_path` now correctly accepts `PathLike` objects. +* `ub.modname_to_modpath` fixed in cases where editable installs use type + annotations in their MAPPING definition. + +### Added +* Support for UNIX special permission (suid/sgid/svtx) codes in `Path.chmod`. + +### Changed +* Moved windows dependencies from requires to optional. Windows users that make + use of these will need to update their ubelt install or explicitly depend on + them as well. ## Version 1.3.5 - Released 2024-03-20 diff --git a/requirements/optional.txt b/requirements/optional.txt index 80d85cbc..aa2ca0a4 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -26,3 +26,8 @@ colorama>=0.4.3;platform_system=="Windows" python_dateutil>=2.8.1 packaging>=21.0 + +jaraco.windows>=3.9.1;platform_system=="Windows" + +# Transative dependency from pydantic>=1.9.1->inflect->jaraco.text->jaraco.windows->ubelt +pydantic<2.0;platform_system=="Windows" and platform_python_implementation == "PyPy" diff --git a/requirements/runtime.txt b/requirements/runtime.txt index f0f13da5..e69de29b 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -1,4 +0,0 @@ -jaraco.windows>=3.9.1;platform_system=="Windows" - -# Transative dependency from pydantic>=1.9.1->inflect->jaraco.text->jaraco.windows->ubelt -pydantic<2.0;platform_system=="Windows" and platform_python_implementation == "PyPy" diff --git a/run_tests.py b/run_tests.py index 5cfaa993..5ac0609e 100755 --- a/run_tests.py +++ b/run_tests.py @@ -1,18 +1,47 @@ #!/usr/bin/env python -if __name__ == '__main__': +import sys + + +def get_this_script_fpath(): + import pathlib + try: + fpath = pathlib.Path(__file__) + except NameError: + # This is not being run from a script, thus the developer is doing some + # IPython hacking, so we will assume a path on the developer machine. + fpath = pathlib.Path('~/code/ubelt/run_tests.py').expanduser() + if not fpath.exists(): + raise Exception( + 'Unable to determine the file path that this script ' + 'should correspond to') + return fpath + + +def main(): import pytest - import sys + import os + + repo_dpath = get_this_script_fpath().parent + package_name = 'ubelt' - mod_dpath = 'ubelt' - test_dpath = 'tests' + mod_dpath = repo_dpath / 'ubelt' + test_dpath = repo_dpath / 'tests' + config_fpath = repo_dpath / 'pyproject.toml' + pytest_args = [ - '--cov-config', 'pyproject.toml', + '--cov-config', os.fspath(config_fpath), '--cov-report', 'html', '--cov-report', 'term', '--durations', '100', '--xdoctest', '--cov=' + package_name, - mod_dpath, test_dpath + os.fspath(mod_dpath), + os.fspath(test_dpath) ] pytest_args = pytest_args + sys.argv[1:] - sys.exit(pytest.main(pytest_args)) + ret = pytest.main(pytest_args) + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py index 07498bc8..ebd31e1c 100755 --- a/setup.py +++ b/setup.py @@ -206,25 +206,18 @@ def gen_packages_items(): "requirements/runtime.txt", versions="loose" ) setupkw["extras_require"] = { - "all": parse_requirements("requirements.txt", versions="loose"), - "tests": parse_requirements("requirements/tests.txt", versions="loose"), - "optional": parse_requirements("requirements/optional.txt", versions="loose"), - "all": parse_requirements("requirements.txt", versions="loose"), - "runtime": parse_requirements("requirements/runtime.txt", versions="loose"), - "tests": parse_requirements("requirements/tests.txt", versions="loose"), - "optional": parse_requirements("requirements/optional.txt", versions="loose"), - "docs": parse_requirements("requirements/docs.txt", versions="loose"), - "types": parse_requirements("requirements/types.txt", versions="loose"), - "all-strict": parse_requirements("requirements.txt", versions="strict"), - "runtime-strict": parse_requirements( - "requirements/runtime.txt", versions="strict" - ), - "tests-strict": parse_requirements("requirements/tests.txt", versions="strict"), - "optional-strict": parse_requirements( - "requirements/optional.txt", versions="strict" - ), - "docs-strict": parse_requirements("requirements/docs.txt", versions="strict"), - "types-strict": parse_requirements("requirements/types.txt", versions="strict"), + "all" : parse_requirements("requirements.txt", versions="loose"), + "all-strict" : parse_requirements("requirements.txt", versions="strict"), + "docs" : parse_requirements("requirements/docs.txt", versions="loose"), + "docs-strict" : parse_requirements("requirements/docs.txt", versions="strict"), + "optional" : parse_requirements("requirements/optional.txt", versions="loose"), + "optional-strict" : parse_requirements("requirements/optional.txt", versions="strict"), + "runtime" : parse_requirements("requirements/runtime.txt", versions="loose"), + "runtime-strict" : parse_requirements("requirements/runtime.txt", versions="strict"), + "tests" : parse_requirements("requirements/tests.txt", versions="loose"), + "tests-strict" : parse_requirements("requirements/tests.txt", versions="strict"), + "types" : parse_requirements("requirements/types.txt", versions="loose"), + "types-strict" : parse_requirements("requirements/types.txt", versions="strict"), } setupkw["name"] = NAME setupkw["version"] = VERSION diff --git a/tests/test_editable_modules.py b/tests/test_editable_modules.py index 8bf9fe30..2d476a85 100644 --- a/tests/test_editable_modules.py +++ b/tests/test_editable_modules.py @@ -467,9 +467,20 @@ def teardown_module(module): def test_import_of_editable_install(): _check_skip_editable_module_tests() + print('Testing ediable installs') import ubelt as ub for PROJ in GLOBAL_PROJECTS: result = ub.modname_to_modpath(PROJ.mod_name) print(f'result={result}') assert result is not None assert PROJ.mod_dpath == ub.Path(result) + + +if __name__ == '__main__': + """ + CommandLine: + UBELT_DO_EDITABLE_TESTS=1 python ~/code/ubelt/tests/test_editable_modules.py + """ + setup_module(None) + test_import_of_editable_install() + teardown_module(None) diff --git a/tests/test_links.py b/tests/test_links.py index a8d5e7bf..85411311 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -9,12 +9,25 @@ import pytest import os from ubelt import util_links +import sys + + +if sys.platform.startswith('win32'): + try: + import jaraco.windows.filesystem as jwfs + except ImportError: + jwfs = None def test_rel_dir_link(): """ xdoctest ~/code/ubelt/tests/test_links.py test_rel_dir_link """ + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. + dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_rel_dir_link').ensuredir() ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) @@ -64,6 +77,10 @@ def test_rel_dir_link(): def test_rel_file_link(): + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_rel_file_link').ensuredir() ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) @@ -119,6 +136,10 @@ def test_delete_symlinks(): CommandLine: python -m ubelt.tests.test_links test_delete_symlinks """ + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. # TODO: test that we handle broken links dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_delete_links').ensuredir() @@ -224,6 +245,10 @@ def assert_broken_link(path, want=True): def test_modify_directory_symlinks(): + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_modify_symlinks').ensuredir() ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) @@ -283,6 +308,10 @@ def test_modify_file_symlinks(): CommandLine: python -m ubelt.tests.test_links test_modify_symlinks """ + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. # TODO: test that we handle broken links dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_modify_symlinks').ensuredir() happy_fpath = dpath / 'happy_fpath.txt' @@ -306,6 +335,10 @@ def test_broken_link(): CommandLine: python -m ubelt.tests.test_links test_broken_link """ + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_broken_link').ensuredir() ub.delete(dpath, verbose=2) @@ -370,6 +403,10 @@ def test_overwrite_symlink(): CommandLine: python ~/code/ubelt/tests/test_links.py test_overwrite_symlink """ + import pytest + import ubelt as ub + if ub.WIN32 and jwfs is None: + pytest.skip() # hack for windows for now. # TODO: test that we handle broken links dpath = ub.Path.appdir('ubelt/tests/test_links', 'test_overwrite_symlink').ensuredir() diff --git a/tests/test_pathlib.py b/tests/test_pathlib.py index 3e020173..05c2de14 100644 --- a/tests/test_pathlib.py +++ b/tests/test_pathlib.py @@ -113,6 +113,10 @@ def test_move_to_nested_non_existing(): base = _demo_directory_structure() root = base / 'root' + import platform + if ub.WIN32 and platform.python_implementation() == 'PyPy': + ub.util_path._patch_win32_stats_on_pypy() + if ub.LINUX: root2 = root.copy(root.augment(tail='2')) root3 = root.copy(root.augment(tail='3')) diff --git a/ubelt/_win32_jaraco.py b/ubelt/_win32_jaraco.py new file mode 100644 index 00000000..717027ab --- /dev/null +++ b/ubelt/_win32_jaraco.py @@ -0,0 +1,261 @@ +""" +Liberated portions of :mod:`jaraco.windows.filesystem`. + +Ignore: + + cat ~/code/ubelt/ubelt/_win32_links.py | grep -o jwfs.api + cat ~/code/ubelt/ubelt/_win32_links.py | grep -o "jwfs\\.[^ ]*" | sort + + git clone git@github.com:jaraco/jaraco.windows.git $HOME/code + cd ~/code/jaraco.windows + touch jaraco/__init__.py + + --- + + Notes: + liberator does not handle the ctypes attributes nicely where + the definition is then modified with argtypes and restypes + + But it does help get a good start on the file. + + --- + + import liberator + import ubelt as ub + repo_dpath = ub.Path('~/code/jaraco.windows').expand() + jwfs_modpath = repo_dpath / 'jaraco/windows/filesystem/__init__.py' + jw_api_filesystem_modpath = repo_dpath / 'jaraco/windows/api/filesystem.py' + jw_reparse_modpath = repo_dpath / 'jaraco/windows/reparse.py' + + lib = liberator.Liberator() + lib.add_static('link', modpath=jwfs_modpath) + lib.add_static('handle_nonzero_success', modpath=jwfs_modpath) + lib.add_static('is_reparse_point', modpath=jwfs_modpath) + + # FIXME: argtypes / restypes + lib.add_static('CreateFile', modpath=jw_api_filesystem_modpath) + lib.add_static('CloseHandle', modpath=jw_api_filesystem_modpath) + + lib.add_static('REPARSE_DATA_BUFFER', modpath=jw_api_filesystem_modpath) + lib.add_static('OPEN_EXISTING', modpath=jw_api_filesystem_modpath) + lib.add_static('FILE_FLAG_OPEN_REPARSE_POINT', modpath=jw_api_filesystem_modpath) + lib.add_static('FILE_FLAG_BACKUP_SEMANTICS', modpath=jw_api_filesystem_modpath) + lib.add_static('FSCTL_GET_REPARSE_POINT', modpath=jw_api_filesystem_modpath) + lib.add_static('INVALID_HANDLE_VALUE', modpath=jw_api_filesystem_modpath) + lib.add_static('IO_REPARSE_TAG_SYMLINK', modpath=jw_api_filesystem_modpath) + lib.add_static('BY_HANDLE_FILE_INFORMATION', modpath=jw_api_filesystem_modpath) + lib.add_static('GetFileInformationByHandle', modpath=jw_api_filesystem_modpath) + + #lib.add_static('DeviceIoControl', modpath=jw_reparse_modpath) + + lib.expand(['jaraco']) + + print(lib.current_sourcecode()) +""" + + +import ctypes.wintypes +import ctypes + +# Makes mypy happy +import sys +assert sys.platform == "win32" + + +def handle_nonzero_success(result): + if (result == 0): + raise ctypes.WinError() + + +class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): + _fields_ = [ + ('file_attributes', ctypes.wintypes.DWORD), + ('creation_time', ctypes.wintypes.FILETIME), + ('last_access_time', ctypes.wintypes.FILETIME), + ('last_write_time', ctypes.wintypes.FILETIME), + ('volume_serial_number', ctypes.wintypes.DWORD), + ('file_size_high', ctypes.wintypes.DWORD), + ('file_size_low', ctypes.wintypes.DWORD), + ('number_of_links', ctypes.wintypes.DWORD), + ('file_index_high', ctypes.wintypes.DWORD), + ('file_index_low', ctypes.wintypes.DWORD) + ] + + @property + def file_size(self): + return ((self.file_size_high << 32) + self.file_size_low) + + @property + def file_index(self): + return ((self.file_index_high << 32) + self.file_index_low) + + +class REPARSE_DATA_BUFFER(ctypes.Structure): + _fields_ = [ + ('tag', ctypes.c_ulong), + ('data_length', ctypes.c_ushort), + ('reserved', ctypes.c_ushort), + ('substitute_name_offset', ctypes.c_ushort), + ('substitute_name_length', ctypes.c_ushort), + ('print_name_offset', ctypes.c_ushort), + ('print_name_length', ctypes.c_ushort), + ('flags', ctypes.c_ulong), + ('path_buffer', (ctypes.c_byte * 1)) + ] + + def get_print_name(self): + wchar_size = ctypes.sizeof(ctypes.wintypes.WCHAR) + arr_typ = (ctypes.wintypes.WCHAR * (self.print_name_length // wchar_size)) + data = ctypes.byref(self.path_buffer, self.print_name_offset) + return ctypes.cast(data, ctypes.POINTER(arr_typ)).contents.value + + def get_substitute_name(self): + wchar_size = ctypes.sizeof(ctypes.wintypes.WCHAR) + arr_typ = (ctypes.wintypes.WCHAR * (self.substitute_name_length // wchar_size)) + data = ctypes.byref(self.path_buffer, self.substitute_name_offset) + return ctypes.cast(data, ctypes.POINTER(arr_typ)).contents.value + + +class SECURITY_ATTRIBUTES(ctypes.Structure): + _fields_ = ( + ('length', ctypes.wintypes.DWORD), + ('p_security_descriptor', ctypes.wintypes.LPVOID), + ('inherit_handle', ctypes.wintypes.BOOLEAN), + ) + +LPSECURITY_ATTRIBUTES = ctypes.POINTER(SECURITY_ATTRIBUTES) + + +IO_REPARSE_TAG_SYMLINK = 0xA000000C +INVALID_HANDLE_VALUE = ctypes.wintypes.HANDLE((- 1)).value +FSCTL_GET_REPARSE_POINT = 0x900A8 +FILE_FLAG_BACKUP_SEMANTICS = 0x2000000 +FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +FILE_SHARE_READ = 1 +OPEN_EXISTING = 3 + + +FILE_ATTRIBUTE_REPARSE_POINT = 0x400 +GENERIC_READ = 0x80000000 +INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF + +GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW +GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,) +GetFileAttributes.restype = ctypes.wintypes.DWORD + +CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW +CreateHardLink.argtypes = ( + ctypes.wintypes.LPWSTR, + ctypes.wintypes.LPWSTR, + ctypes.wintypes.LPVOID, # reserved for LPSECURITY_ATTRIBUTES +) +CreateHardLink.restype = ctypes.wintypes.BOOLEAN + +GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle +GetFileInformationByHandle.restype = ctypes.wintypes.BOOL +GetFileInformationByHandle.argtypes = ( + ctypes.wintypes.HANDLE, + ctypes.POINTER(BY_HANDLE_FILE_INFORMATION), +) + +CloseHandle = ctypes.windll.kernel32.CloseHandle +CloseHandle.argtypes = (ctypes.wintypes.HANDLE,) +CloseHandle.restype = ctypes.wintypes.BOOLEAN + +CreateFile = ctypes.windll.kernel32.CreateFileW +CreateFile.argtypes = ( + ctypes.wintypes.LPWSTR, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + LPSECURITY_ATTRIBUTES, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.HANDLE, +) +CreateFile.restype = ctypes.wintypes.HANDLE + +LPDWORD = ctypes.POINTER(ctypes.wintypes.DWORD) +LPOVERLAPPED = ctypes.wintypes.LPVOID + +DeviceIoControl = ctypes.windll.kernel32.DeviceIoControl +DeviceIoControl.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.LPVOID, + ctypes.wintypes.DWORD, + ctypes.wintypes.LPVOID, + ctypes.wintypes.DWORD, + LPDWORD, + LPOVERLAPPED, +] +DeviceIoControl.restype = ctypes.wintypes.BOOL + + +def is_reparse_point(path): + """ + Determine if the given path is a reparse point. + Return False if the file does not exist or the file attributes cannot + be determined. + """ + res = GetFileAttributes(path) + return ((res != INVALID_FILE_ATTRIBUTES) and bool((res & FILE_ATTRIBUTE_REPARSE_POINT))) + + +def link(target, link): + """ + Establishes a hard link between an existing file and a new file. + """ + handle_nonzero_success(CreateHardLink(link, target, None)) + + +def _reparse_DeviceIoControl(device, io_control_code, in_buffer, out_buffer, overlapped=None): + # ubelt note: name is overloaded, so we mangle it here. + if overlapped is not None: + raise NotImplementedError("overlapped handles not yet supported") + + if isinstance(out_buffer, int): + out_buffer = ctypes.create_string_buffer(out_buffer) + + in_buffer_size = len(in_buffer) if in_buffer is not None else 0 + out_buffer_size = len(out_buffer) + assert isinstance(out_buffer, ctypes.Array) + + returned_bytes = ctypes.wintypes.DWORD() + + res = DeviceIoControl( + device, + io_control_code, + in_buffer, + in_buffer_size, + out_buffer, + out_buffer_size, + returned_bytes, + overlapped, + ) + + handle_nonzero_success(res) + handle_nonzero_success(returned_bytes) + + return out_buffer[: returned_bytes.value] + + +# Fake the jaraco api +class api: + CreateFile = CreateFile + CloseHandle = CloseHandle + GetFileInformationByHandle = GetFileInformationByHandle + + BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION + FILE_FLAG_BACKUP_SEMANTICS = FILE_FLAG_BACKUP_SEMANTICS + FILE_FLAG_OPEN_REPARSE_POINT = FILE_FLAG_OPEN_REPARSE_POINT + FILE_SHARE_READ = FILE_SHARE_READ + FSCTL_GET_REPARSE_POINT = FSCTL_GET_REPARSE_POINT + GENERIC_READ = GENERIC_READ + INVALID_HANDLE_VALUE = INVALID_HANDLE_VALUE + IO_REPARSE_TAG_SYMLINK = IO_REPARSE_TAG_SYMLINK + OPEN_EXISTING = OPEN_EXISTING + REPARSE_DATA_BUFFER = REPARSE_DATA_BUFFER + + +class reparse: + DeviceIoControl = _reparse_DeviceIoControl diff --git a/ubelt/_win32_links.py b/ubelt/_win32_links.py index 143108eb..1c948ada 100644 --- a/ubelt/_win32_links.py +++ b/ubelt/_win32_links.py @@ -1,6 +1,12 @@ """ For dealing with symlinks, junctions, and hard-links on windows. +Note: + The termonology used here was written before I really understood the + difference between symlinks, hardlinks, and junctions. As such it may be + inconsistent or incorrect in some places. This might be fixed in the + future. + References: .. [SO18883892] https://stackoverflow.com/questions/18883892/batch-file-windows-cmd-exe-test-if-a-directory-is-a-link-symlink .. [SO21561850] https://stackoverflow.com/questions/21561850/python-test-for-junction-point-target @@ -15,6 +21,7 @@ """ import os import warnings +import platform from os.path import exists from os.path import join from ubelt import util_io @@ -22,8 +29,11 @@ import sys if sys.platform.startswith('win32'): - import jaraco.windows.filesystem as jwfs - + try: + import jaraco.windows.filesystem as jwfs + except ImportError: + # Use vendored subset of jaraco.windows + from ubelt import _win32_jaraco as jwfs __win32_can_symlink__ = None # type: bool | None @@ -38,7 +48,7 @@ def _win32_can_symlink(verbose=0, force=0, testing=0): Example: >>> # xdoctest: +REQUIRES(WIN32) >>> import ubelt as ub - >>> _win32_can_symlink(verbose=1, force=1, testing=1) + >>> _win32_can_symlink(verbose=3, force=1, testing=1) """ global __win32_can_symlink__ if verbose: @@ -82,9 +92,9 @@ def _win32_can_symlink(verbose=0, force=0, testing=0): util_io.touch(broken_fpath) try: - _win32_symlink(dpath, dlink) + _win32_symlink(dpath, dlink, verbose=verbose) if testing: - _win32_symlink(broken_dpath, join(tempdir, 'broken_dlink')) + _win32_symlink(broken_dpath, join(tempdir, 'broken_dlink'), verbose=verbose) can_symlink_directories = os.path.islink(dlink) except OSError: can_symlink_directories = False @@ -92,9 +102,9 @@ def _win32_can_symlink(verbose=0, force=0, testing=0): print('can_symlink_directories = {!r}'.format(can_symlink_directories)) try: - _win32_symlink(fpath, flink) + _win32_symlink(fpath, flink, verbose=verbose) if testing: - _win32_symlink(broken_fpath, join(tempdir, 'broken_flink')) + _win32_symlink(broken_fpath, join(tempdir, 'broken_flink'), verbose=verbose) can_symlink_files = os.path.islink(flink) # os.path.islink(flink) except OSError: @@ -109,15 +119,28 @@ def _win32_can_symlink(verbose=0, force=0, testing=0): try: # test that we can create junctions, even if symlinks are disabled - djunc = _win32_junction(dpath, join(tempdir, 'djunc')) - fjunc = _win32_junction(fpath, join(tempdir, 'fjunc.txt')) + if verbose: + print('Testing that we can create junctions, ' + 'even if symlinks are disabled') + # from ubelt import util_links + # util_links._dirstats(tempdir) + # print('^^ before ^^') + + djunc = _win32_junction(dpath, join(tempdir, 'djunc'), verbose=verbose) + fjunc = _win32_junction(fpath, join(tempdir, 'fjunc.txt'), verbose=verbose) if testing: - _win32_junction(broken_dpath, join(tempdir, 'broken_djunc')) - _win32_junction(broken_fpath, join(tempdir, 'broken_fjunc.txt')) + _win32_junction(broken_dpath, join(tempdir, 'broken_djunc'), verbose=verbose) + _win32_junction(broken_fpath, join(tempdir, 'broken_fjunc.txt'), verbose=verbose) if not _win32_is_junction(djunc): - raise AssertionError('expected junction') + print(f'Error: djunc={djunc} claims to not be a junction') + from ubelt import util_links + util_links._dirstats(tempdir) + raise AssertionError(f'expected djunc={djunc} to be a junction') if not _win32_is_hardlinked(fpath, fjunc): - raise AssertionError('expected hardlink') + print(f'Error: fjunc={fjunc} claims to not be a hardlink') + from ubelt import util_links + util_links._dirstats(tempdir) + raise AssertionError(f'expected fjunc={fjunc} to be a hardlink') except Exception: warnings.warn('We cannot create junctions either!') raise @@ -230,6 +253,9 @@ def _win32_symlink(path, link, verbose=0): specially enabled symlink permissions. On Windows 10 enabling developer mode should give you these permissions. """ + if verbose >= 3: + print(f'_win32_symlink {link} -> {path}') + from ubelt import util_cmd if os.path.isdir(path): # directory symbolic link @@ -247,14 +273,15 @@ def _win32_symlink(path, link, verbose=0): command = 'mklink "{}" "{}"'.format(link, path) if command is not None: - info = util_cmd.cmd(command, shell=True) + cmd_verbose = 3 * verbose >= 3 + info = util_cmd.cmd(command, shell=True, verbose=cmd_verbose) if info['ret'] != 0: - from ubelt import util_format + from ubelt import util_repr permission_msg = 'You do not have sufficient privledge' if permission_msg not in info['err']: print('Failed command:') print(info['command']) - print(util_format.repr2(info, nl=1)) + print(util_repr.urepr(info, nl=1)) raise OSError(str(info)) return link @@ -295,11 +322,14 @@ def _win32_junction(path, link, verbose=0): path = os.path.abspath(path) link = os.path.abspath(link) + if verbose >= 3: + print(f'_win32_junction {link} -> {path}') + from ubelt import util_cmd if os.path.isdir(path): # try using a junction (soft link) if verbose: - print('... as soft link') + print('... as soft link (junction)') # TODO: what is the windows api for this? command = 'mklink /J "{}" "{}"'.format(link, path) @@ -316,12 +346,13 @@ def _win32_junction(path, link, verbose=0): command = None if command is not None: - info = util_cmd.cmd(command, shell=True) + cmd_verbose = 3 * verbose >= 3 + info = util_cmd.cmd(command, shell=True, verbose=cmd_verbose) if info['ret'] != 0: - from ubelt import util_format + from ubelt import util_repr print('Failed command:') print(info['command']) - print(util_format.repr2(info, nl=1)) + print(util_repr.urepr(info, nl=1)) raise OSError(str(info)) return link @@ -330,15 +361,23 @@ def _win32_is_junction(path): """ Determines if a path is a win32 junction + Note: + on PyPy this is bugged and will currently return True for a symlinked + directory. + + Returns: + bool: + Example: - >>> # xdoc: +REQUIRES(WIN32) + >>> # xdoctest: +REQUIRES(WIN32) + >>> from ubelt._win32_links import _win32_junction, _win32_is_junction >>> import ubelt as ub >>> root = ub.Path.appdir('ubelt', 'win32_junction').ensuredir() >>> ub.delete(root) >>> ub.ensuredir(root) - >>> dpath = join(root, 'dpath') - >>> djunc = join(root, 'djunc') - >>> ub.ensuredir(dpath) + >>> dpath = root / 'dpath' + >>> djunc = root / 'djunc' + >>> dpath.ensuredir() >>> _win32_junction(dpath, djunc) >>> assert _win32_is_junction(djunc) is True >>> assert _win32_is_junction(dpath) is False @@ -350,7 +389,36 @@ def _win32_is_junction(path): if not os.path.islink(path): return True return False - return jwfs.is_reparse_point(path) and not os.path.islink(path) + + if platform.python_implementation() == 'PyPy': + # Workaround for pypy where os.path.islink will return True + # for a junction. Can we just rely on it being a reparse point? + # https://github.com/pypy/pypy/issues/4976 + return _is_reparse_point(path) + else: + return _is_reparse_point(path) and not os.path.islink(path) + + +def _is_reparse_point(path): + """ + Check if a directory is a reparse point in windows. + + Note: a reparse point seems like it could be a junction or symlink. + + .. [SO54678399] https://stackoverflow.com/a/54678399/887074 + """ + if jwfs is None: + raise ImportError('jaraco.windows.filesystem is required to run _is_reparse_point') + # if jwfs is not None: + return jwfs.is_reparse_point(os.fspath(path)) + # else: + # # Fallback without jaraco: TODO: test this is 1-to-1 + # # Seems to break on pypy? + # import subprocess + # child = subprocess.Popen(f'fsutil reparsepoint query "{path}"', + # stdout=subprocess.PIPE) + # child.communicate()[0] + # return child.returncode == 0 def _win32_read_junction(path): @@ -372,7 +440,11 @@ def _win32_read_junction(path): >>> pointed = _win32_read_junction(path) >>> print('pointed = {!r}'.format(pointed)) """ + import ctypes path = os.fspath(path) + if jwfs is None: + raise ImportError('jaraco.windows.filesystem is required to run _win32_read_junction') + if not jwfs.is_reparse_point(path): raise ValueError('not a junction') @@ -389,8 +461,8 @@ def _win32_read_junction(path): res = jwfs.reparse.DeviceIoControl( handle, jwfs.api.FSCTL_GET_REPARSE_POINT, None, 10240) - bytes = jwfs.create_string_buffer(res) - p_rdb = jwfs.cast(bytes, jwfs.POINTER(jwfs.api.REPARSE_DATA_BUFFER)) + bytes = ctypes.create_string_buffer(res) + p_rdb = ctypes.cast(bytes, ctypes.POINTER(jwfs.api.REPARSE_DATA_BUFFER)) rdb = p_rdb.contents if rdb.tag not in [2684354563, jwfs.api.IO_REPARSE_TAG_SYMLINK]: @@ -484,6 +556,9 @@ def _win32_is_hardlinked(fpath1, fpath2): >>> assert not _win32_is_hardlinked(fjunc2, fpath1) >>> assert not _win32_is_hardlinked(fjunc1, fpath2) """ + if jwfs is None: + raise ImportError('jaraco.windows.filesystem is required to run _win32_is_hardlinked') + # NOTE: jwf.samefile(fpath1, fpath2) seems to behave differently def get_read_handle(fpath): if os.path.isdir(fpath): @@ -527,10 +602,10 @@ def _win32_dir(path, star=''): wrapped = wrapper.format(command) info = util_cmd.cmd(wrapped, shell=True) if info['ret'] != 0: - from ubelt import util_format + from ubelt import util_repr print('Failed command:') print(info['command']) - print(util_format.repr2(info, nl=1)) + print(util_repr.urepr(info, nl=1)) raise OSError(str(info)) # parse the output of dir to get some info # Remove header and footer diff --git a/ubelt/util_cache.py b/ubelt/util_cache.py index b53b911b..573351d7 100644 --- a/ubelt/util_cache.py +++ b/ubelt/util_cache.py @@ -36,13 +36,13 @@ Example: >>> import ubelt as ub - >>> @ub.Cacher('name', depends={'dep1': 1, 'dep2': 2}) # boilerplate:1 + >>> @ub.Cacher('name', depends='set-of-deps') # boilerplate:1 >>> def func(): # boilerplate:2 >>> data = 'mydata' >>> return data # boilerplate:3 >>> data = func() # boilerplate:4 - >>> cacher = ub.Cacher('name', depends=['dependencies']) # boilerplate:1 + >>> cacher = ub.Cacher('name', depends='set-of-deps') # boilerplate:1 >>> data = cacher.tryload(on_error='clear') # boilerplate:2 >>> if data is None: # boilerplate:3 >>> data = 'mydata' diff --git a/ubelt/util_deprecate.py b/ubelt/util_deprecate.py index 4c74bc8f..770a8197 100644 --- a/ubelt/util_deprecate.py +++ b/ubelt/util_deprecate.py @@ -164,6 +164,8 @@ def schedule_deprecation(modname=None, name='?', type='?', migration='', module = sys.modules[modname] current = Version(module.__version__) else: + # TODO: use the inspect module to get the function / module this was + # called from and fill in unspecified values. current = 'unknown' def _handle_when(when, default): @@ -190,6 +192,7 @@ def _handle_when(when, default): remove_now, remove_str = _handle_when(remove, default=False) error_now, error_str = _handle_when(error, default=False) + # TODO: make the message more customizable. msg = ( 'The "{name}" {type} was deprecated{deprecate_str}, will cause ' 'an error{error_str} and will be removed{remove_str}. The current ' diff --git a/ubelt/util_import.py b/ubelt/util_import.py index 1c65dfa9..60d335d0 100644 --- a/ubelt/util_import.py +++ b/ubelt/util_import.py @@ -551,17 +551,21 @@ def check_dpath(dpath): # break with pytest anymore? Nope, pytest still doesn't work right # with it. for finder_fpath in new_editable_finder_paths: - mapping = _static_parse('MAPPING', finder_fpath) try: - target = dirname(mapping[_pkg_name]) - except KeyError: + mapping = _static_parse('MAPPING', finder_fpath) + except AttributeError: ... else: - if not exclude or normalize(target) not in real_exclude: # pragma: nobranch - modpath = check_dpath(target) - if modpath: # pragma: nobranch - found_modpath = modpath - break + try: + target = dirname(mapping[_pkg_name]) + except KeyError: + ... + else: + if not exclude or normalize(target) not in real_exclude: # pragma: nobranch + modpath = check_dpath(target) + if modpath: # pragma: nobranch + found_modpath = modpath + break if found_modpath is not None: break @@ -621,9 +625,10 @@ def _custom_import_modpath(modpath, index=-1): with PythonPathContext(dpath, index=index): module = import_module_from_name(modname) except Exception as ex: # nocover - msg_parts = [ - 'ERROR: Failed to import modname={} with modpath={}'.format( - modname, modpath) + msg_parts = [( + 'ERROR: Failed to import modname={} with modpath={} and ' + 'sys.path modified with {} at index={}').format( + modname, modpath, repr(dpath), index) ] msg_parts.append('Caused by: {}'.format(repr(ex))) raise RuntimeError('\n'.join(msg_parts)) @@ -646,11 +651,22 @@ def _importlib_import_modpath(modpath): # nocover return module +def _importlib_modname_to_modpath(modname): # nocover + import importlib.util + spec = importlib.util.find_spec(modname) + print(f'spec={spec}') + modpath = spec.origin.replace('.pyc', '.py') + return modpath + + def _pkgutil_modname_to_modpath(modname): # nocover """ faster version of :func:`_syspath_modname_to_modpath` using builtin python mechanisms, but unfortunately it doesn't play nice with pytest. + Note: + pkgutil.find_loader is deprecated in 3.12 and removed in 3.14 + Args: modname (str): the module name. @@ -717,7 +733,18 @@ def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None): >>> modpath = basename(modname_to_modpath('_ctypes')) >>> assert 'ctypes' in modpath """ - modpath = _syspath_modname_to_modpath(modname, sys_path) + if hide_main or sys_path: + modpath = _syspath_modname_to_modpath(modname, sys_path) + else: + # import xdev + # with xdev.embed_on_exception_context: + # try: + # modpath = _importlib_modname_to_modpath(modname) + # except Exception: + # modpath = _syspath_modname_to_modpath(modname, sys_path) + # modpath = _pkgutil_modname_to_modpath(modname, sys_path) + modpath = _syspath_modname_to_modpath(modname, sys_path) + if modpath is None: return None @@ -950,6 +977,13 @@ def _static_parse(varname, fpath): """ Statically parse the a constant variable from a python file + Args: + varname (str): variable name to extract + fpath (str | PathLike): path to python file to parse + + Returns: + Any: the static value + Example: >>> import ubelt as ub >>> from ubelt.util_import import _static_parse @@ -974,6 +1008,10 @@ def _static_parse(varname, fpath): >>> with pytest.raises(AttributeError): >>> fpath.write_text('a = list(range(10))') >>> assert _static_parse('c', fpath) is None + >>> if sys.version_info[0:2] >= (3, 6): + >>> # Test with type annotations + >>> fpath.write_text('b: int = 10') + >>> assert _static_parse('b', fpath) == 10 """ import ast @@ -986,9 +1024,16 @@ def _static_parse(varname, fpath): class StaticVisitor(ast.NodeVisitor): def visit_Assign(self, node): for target in node.targets: - if getattr(target, 'id', None) == varname: + target_id = getattr(target, 'id', None) + if target_id == varname: self.static_value = _parse_static_node_value(node.value) + def visit_AnnAssign(self, node): + target = node.target + target_id = getattr(target, 'id', None) + if target_id == varname: + self.static_value = _parse_static_node_value(node.value) + visitor = StaticVisitor() visitor.visit(pt) try: diff --git a/ubelt/util_indexable.py b/ubelt/util_indexable.py index bee5453c..1a5078a4 100644 --- a/ubelt/util_indexable.py +++ b/ubelt/util_indexable.py @@ -11,8 +11,33 @@ """ from math import isclose from collections.abc import Generator +from typing import NamedTuple, Tuple, Any # from collections.abc import Iterable +try: + from functools import cache +except ImportError: + from ubelt.util_memoize import memoize as cache + + +@cache +def _lazy_numpy(): + try: + import numpy as np + except ImportError: + return None + return np + + +class Difference(NamedTuple): + """ + A result class of indexable_diff that organizes what the difference between + the indexables is. + """ + path: Tuple + value1: Any + value2: Any + class IndexableWalker(Generator): """ @@ -327,7 +352,7 @@ def _walk(self, data=None, prefix=[]): if isinstance(value, self.indexable_cls): stack.append((value, path)) - def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, return_info=False): + def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, equal_nan=False, return_info=False): """ Walks through this and another nested data structures and checks if everything is roughly the same. @@ -344,6 +369,9 @@ def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, return_info=False): maximum difference for being considered "close", regardless of the magnitude of the input values + equal_nan (bool): + if True, numpy must be available, and consider nans as equal. + return_info (bool, default=False): if true, return extra info dict Returns: @@ -439,6 +467,8 @@ def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, return_info=False): walker2 = IndexableWalker(other, dict_cls=self.dict_cls, list_cls=self.list_cls) + _isclose_fn, _iskw = _make_isclose_fn(rel_tol, abs_tol, equal_nan) + flat_items1 = [ (path, value) for path, value in walker1 if not isinstance(value, walker1.indexable_cls) or len(value) == 0] @@ -463,11 +493,10 @@ def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, return_info=False): p2, v2 = t2 assert p1 == p2, 'paths to the nested items should be the same' - flag = (v1 == v2) - if not flag: - if isinstance(v1, float) and isinstance(v2, float): - if isclose(v1, v2, rel_tol=rel_tol, abs_tol=abs_tol): - flag = True + flag = (v1 == v2) or ( + isinstance(v1, float) and isinstance(v2, float) and + _isclose_fn(v1, v2, **_iskw) + ) if flag: passlist.append(p1) else: @@ -488,6 +517,142 @@ def allclose(self, other, rel_tol=1e-9, abs_tol=0.0, return_info=False): else: return final_flag + def diff(self, other, rel_tol=1e-9, abs_tol=0.0, equal_nan=False): + """ + Walks through two nested data structures finds differences in the + structures. + + Args: + other (IndexableWalker | List | Dict): + a nested indexable item to compare against. + + rel_tol (float): + maximum difference for being considered "close", relative to the + magnitude of the input values + + abs_tol (float): + maximum difference for being considered "close", regardless of the + magnitude of the input values + + equal_nan (bool): + if True, numpy must be available, and consider nans as equal. + + Returns: + dict: information about the diff with + "similarity": a score between 0 and 1 + "num_differences" being the number of paths not common plus the + number of common paths with differing values. + "unique1": being the paths that were unique to self + "unique2": being the paths that were unique to other + "faillist": a list 3-tuples of common path and differing values + "num_approximations": + is the number of approximately equal items (i.e. floats) there were + + Example: + >>> import ubelt as ub + >>> dct1 = { + >>> 'foo': [1.222222, 1.333], + >>> 'bar': 1, + >>> 'baz': [], + >>> 'top': [1, 2, 3], + >>> 'L0': {'L1': {'L2': {'K1': 'V1', 'K2': 'V2', 'D1': 1, 'D2': 2}}}, + >>> } + >>> dct2 = { + >>> 'foo': [1.22222, 1.333], + >>> 'bar': 1, + >>> 'baz': [], + >>> 'buz': {1: 2}, + >>> 'top': [1, 1, 2], + >>> 'L0': {'L1': {'L2': {'K1': 'V1', 'K2': 'V2', 'D1': 10, 'D2': 20}}}, + >>> } + >>> info = ub.IndexableWalker(dct1).diff(dct2) + >>> print(f'info = {ub.urepr(info, nl=2)}') + + Example: + >>> # xdoctest: +REQUIRES(module:numpy) + >>> import ubelt as ub + >>> import numpy as np + >>> a = np.random.rand(3, 5) + >>> b = a + 1 + >>> wa = ub.IndexableWalker(a, list_cls=(np.ndarray,)) + >>> wb = ub.IndexableWalker(b, list_cls=(np.ndarray,)) + >>> info = wa.diff(wb) + >>> print(f'info = {ub.urepr(info, nl=2)}') + >>> a = np.random.rand(3, 5) + >>> b = a.copy() + 1e-17 + >>> wa = ub.IndexableWalker([a], list_cls=(np.ndarray, list)) + >>> wb = ub.IndexableWalker([b], list_cls=(np.ndarray, list)) + >>> info = wa.diff(wb) + >>> print(f'info = {ub.urepr(info, nl=2)}') + """ + walker1 = self + if isinstance(other, IndexableWalker): + walker2 = other + else: + walker2 = IndexableWalker(other, dict_cls=self.dict_cls, + list_cls=self.list_cls) + # TODO: numpy optimizations + flat_items1 = { + tuple(path): value for path, value in walker1 + if not isinstance(value, walker1.indexable_cls) or len(value) == 0} + flat_items2 = { + tuple(path): value for path, value in walker2 + if not isinstance(value, walker1.indexable_cls) or len(value) == 0} + + common = flat_items1.keys() & flat_items2.keys() + unique1 = flat_items1.keys() - flat_items2.keys() + unique2 = flat_items2.keys() - flat_items1.keys() + + num_approximations = 0 + + _isclose_fn, _iskw = _make_isclose_fn(rel_tol, abs_tol, equal_nan) + + faillist = [] + passlist = [] + for key in common: + v1 = flat_items1[key] + v2 = flat_items2[key] + flag = (v1 == v2) + if not flag: + flag = ( + isinstance(v1, float) and isinstance(v2, float) and + _isclose_fn(v1, v2, **_iskw) + ) + num_approximations += flag + if flag: + passlist.append(key) + else: + faillist.append(Difference(key, v1, v2)) + + num_differences = len(unique1) + len(unique2) + len(faillist) + num_similarities = len(passlist) + + similarity = num_similarities / (num_similarities + num_differences) + info = { + 'similarity': similarity, + 'num_approximations': num_approximations, + 'num_differences': num_differences, + 'num_similarities': num_similarities, + 'unique1': unique1, + 'unique2': unique2, + 'faillist': faillist, + 'passlist': passlist, + } + return info + + +def _make_isclose_fn(rel_tol, abs_tol, equal_nan): + np = _lazy_numpy() + if np is None: + _isclose_fn = isclose + _iskw = dict(rel_tol=rel_tol, abs_tol=abs_tol) + if equal_nan: + raise NotImplementedError('requires numpy') + else: + _isclose_fn = np.isclose + _iskw = dict(rtol=rel_tol, atol=abs_tol, equal_nan=equal_nan) + return _isclose_fn, _iskw + def indexable_allclose(items1, items2, rel_tol=1e-9, abs_tol=0.0, return_info=False): """ @@ -549,8 +714,9 @@ def indexable_allclose(items1, items2, rel_tol=1e-9, abs_tol=0.0, return_info=Fa return_info=return_info) +# Nested = IndexableWalker # class Indexable(IndexableWalker): # """ -# In the future IndexableWalker may simply change to Indexable +# In the future IndexableWalker may simply change to Indexable or maybe Nested # """ # ... diff --git a/ubelt/util_io.py b/ubelt/util_io.py index ea7c5196..a0acba1a 100644 --- a/ubelt/util_io.py +++ b/ubelt/util_io.py @@ -38,7 +38,7 @@ def writeto(fpath, to_write, aslines=False, verbose=None): `https://pypy.org/compat.html`. This is an argument for keeping this function. - NOTE: With modern versions of Python, it is generally recommened to use + NOTE: With modern versions of Python, it is generally recommend to use :func:`pathlib.Path.write_text` instead. Although there does seem to be some corner case this handles better on win32, so maybe useful? @@ -269,9 +269,10 @@ def delete(path, verbose=False): elif os.path.isdir(path): if verbose: # nocover print('Deleting directory="{}"'.format(path)) - if sys.platform.startswith('win32'): # nocover + if sys.platform.startswith('win32') and sys.version_info[0:2] < (3, 8): # nocover # Workaround bug that prevents shutil from working if # the directory contains junctions + # https://bugs.python.org/issue36621 from ubelt import _win32_links _win32_links._win32_rmtree(path, verbose=verbose) else: diff --git a/ubelt/util_links.py b/ubelt/util_links.py index 422d68fe..a34ec701 100644 --- a/ubelt/util_links.py +++ b/ubelt/util_links.py @@ -7,6 +7,10 @@ works without difficulty. Example: + >>> import pytest + >>> import ubelt as ub + >>> if ub.WIN32: + >>> pytest.skip() # hack for windows for now. Todo cleaner xdoctest conditional >>> import ubelt as ub >>> from os.path import normpath, join >>> dpath = ub.Path.appdir('ubelt', normpath('demo/symlink')).ensuredir() @@ -56,6 +60,13 @@ def symlink(real_path, link_path, overwrite=False, verbose=0): Returns: str | PathLike: link path + Note: + In the future we may rework and rename this function to something like + ``link``, ``pathlink``, ``fslink``, etc... to indicate that it may + perform multiple types of links. We may also allow the user to specify + which type of link (e.g. symlink, hardlink, reflink, junction) they + would like to use. + Note: On systems that do not contain support for symlinks (e.g. some versions / configurations of Windows), this function will fall back on hard @@ -80,6 +91,10 @@ def symlink(real_path, link_path, overwrite=False, verbose=0): .. [WikiNTFSLinks] https://en.wikipedia.org/wiki/NTFS_links Example: + >>> import pytest + >>> import ubelt as ub + >>> if ub.WIN32: + >>> pytest.skip() # hack for windows for now. Todo cleaner xdoctest conditional >>> import ubelt as ub >>> dpath = ub.Path.appdir('ubelt', 'test_symlink0').delete().ensuredir() >>> real_path = (dpath / 'real_file.txt') @@ -90,6 +105,10 @@ def symlink(real_path, link_path, overwrite=False, verbose=0): >>> dpath.delete() # clenaup Example: + >>> import pytest + >>> import ubelt as ub + >>> if ub.WIN32: + >>> pytest.skip() # hack for windows for now. Todo cleaner xdoctest conditional >>> import ubelt as ub >>> from ubelt.util_links import _dirstats >>> dpath = ub.Path.appdir('ubelt', 'test_symlink1').delete().ensuredir() @@ -119,6 +138,10 @@ def symlink(real_path, link_path, overwrite=False, verbose=0): >>> assert not real_path.exists() Example: + >>> import pytest + >>> import ubelt as ub + >>> if ub.WIN32: + >>> pytest.skip() # hack for windows for now. Todo cleaner xdoctest conditional >>> # Specifying bad paths should error. >>> import ubelt as ub >>> import pytest @@ -206,6 +229,15 @@ def _readlink(link): if _win32_links: # nocover if _win32_links._win32_is_junction(link): + import platform + if platform.python_implementation() == 'PyPy': + # On PyPy this test can have a false positive + # for what should be a regular link. + path = os.readlink(link) + junction_prefix = '\\\\?\\' + if path.startswith(junction_prefix): + path = path[len(junction_prefix):] + return path return _win32_links._win32_read_junction(link) try: path = os.readlink(link) @@ -226,10 +258,6 @@ def _can_symlink(verbose=0): # nocover Return true if we have permission to create real symlinks. This check always returns True on non-win32 systems. If this check returns false, then we still may be able to use junctions. - - Example: - >>> # Script - >>> print(_can_symlink(verbose=1)) """ if _win32_links is not None: return _win32_links._win32_can_symlink(verbose) @@ -244,6 +272,10 @@ def _dirstats(dpath=None): # nocover The column prefixes stand for: (E - exists), (L - islink), (F - isfile), (D - isdir), (J - isjunction) + + Example: + >>> from ubelt.util_links import _dirstats + >>> _dirstats('.') """ from ubelt import util_colors if dpath is None: @@ -301,13 +333,22 @@ def _dirstats(dpath=None): # nocover # I get it, they are probably broken junctions, but common # That should probably be 00011 not 00000 path = util_colors.color_text(path, 'red') + elif ELFDJ == [1, 1, 0, 1, 1]: + # Agg, on windows pypy, it looks like junctions and links are + # harder to distinguish. See + # https://github.com/pypy/pypy/issues/4976 + path = util_colors.color_text(path, 'red') + elif ELFDJ == [1, 1, 1, 0, 1]: + # Again? on windows pypy, its a link/file/junction what? + path = util_colors.color_text(path, 'red') else: print('dpath = {!r}'.format(dpath)) - print('path = {!r}'.format(path)) + print('pathhttps://github.com/pypy/pypy/issues/4976 = {!r}'.format(path)) raise AssertionError(str(ELFDJ) + str(path)) line = '{E:d} {L:d} {F:d} {D:d} {J:d} - {path}'.format(**locals()) if os.path.islink(full_path): - line += ' -> ' + os.readlink(full_path) + # line += ' -> ' + os.readlink(full_path) + line += ' -> ' + _readlink(full_path) elif _win32_links is not None: if _win32_links._win32_is_junction(full_path): resolved = _win32_links._win32_read_junction(full_path) diff --git a/ubelt/util_path.py b/ubelt/util_path.py index b9a3ae95..0d933bac 100644 --- a/ubelt/util_path.py +++ b/ubelt/util_path.py @@ -35,6 +35,8 @@ import os import sys import pathlib +import platform +import stat import warnings from ubelt import util_io @@ -44,6 +46,8 @@ 'expandpath', 'ChDir', ] +WIN32 = sys.platform.startswith('win32') + def augpath(path, suffix='', prefix='', ext=None, tail='', base=None, dpath=None, relative=None, multidot=False): @@ -198,7 +202,7 @@ def userhome(username=None): if 'HOME' in os.environ: userhome_dpath = os.environ['HOME'] else: # nocover - if sys.platform.startswith('win32'): + if WIN32: # win32 fallback when HOME is not defined if 'USERPROFILE' in os.environ: userhome_dpath = os.environ['USERPROFILE'] @@ -213,7 +217,7 @@ def userhome(username=None): userhome_dpath = pwd.getpwuid(os.getuid()).pw_dir else: # A specific user directory was requested - if sys.platform.startswith('win32'): # nocover + if WIN32: # nocover # get the directory name for the current user c_users = dirname(userhome()) userhome_dpath = join(c_users, username) @@ -1170,7 +1174,25 @@ def chmod(self, mode, follow_symlinks=True): # """ # return self.chmod(mode, follow_symlinks=False) - def touch(self, mode=0o666, exist_ok=True): + # TODO: + # chainable symlink_to that returns the new link + # chainable hardlink_to that returns the new link + # probably can just uncomment when ready for a new feature + # def symlink_to(self, target, target_is_directory=False): + # """ + # Make this path a symlink pointing to the target path. + # """ + # super().symlink_to(target, target_is_directory=target_is_directory) + # return self + + # def hardlink_to(self, target): + # """ + # Make this path a hard link pointing to the same file as *target*. + # """ + # super().hardlink_to(target) + # return self + + def touch(self, mode=0o0666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. @@ -1371,13 +1393,23 @@ def _request_copy_function(self, follow_file_symlinks=True, Get a copy_function based on specified capabilities """ import shutil + # Note: Avoiding the use of the partial enables shutil optimizations from functools import partial if meta is None: - copy_function = partial(shutil.copyfile, follow_symlinks=follow_file_symlinks) + if follow_file_symlinks: + copy_function = shutil.copyfile + else: + copy_function = partial(shutil.copyfile, follow_symlinks=follow_file_symlinks) elif meta == 'stats': - copy_function = partial(shutil.copy2, follow_symlinks=follow_file_symlinks) + if follow_file_symlinks: + copy_function = shutil.copy2 + else: + copy_function = partial(shutil.copy2, follow_symlinks=follow_file_symlinks) elif meta == 'mode': - copy_function = partial(shutil.copy, follow_symlinks=follow_file_symlinks) + if follow_file_symlinks: + copy_function = shutil.copy + else: + copy_function = partial(shutil.copy, follow_symlinks=follow_file_symlinks) else: raise KeyError(meta) return copy_function @@ -1510,11 +1542,16 @@ def copy(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False, copy_function = self._request_copy_function( follow_file_symlinks=follow_file_symlinks, follow_dir_symlinks=follow_dir_symlinks, meta=meta) + + if WIN32 and platform.python_implementation() == 'PyPy': + _patch_win32_stats_on_pypy() + if self.is_dir(): if sys.version_info[0:2] < (3, 8): # nocover copytree = _compat_copytree else: copytree = shutil.copytree + dst = copytree( self, dst, copy_function=copy_function, symlinks=not follow_dir_symlinks, dirs_exist_ok=overwrite) @@ -1596,6 +1633,10 @@ def move(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False, raise FileExistsError( 'Moves are only allowed to locations that dont exist') import shutil + + if WIN32 and platform.python_implementation() == 'PyPy': + _patch_win32_stats_on_pypy() + copy_function = self._request_copy_function( follow_file_symlinks=follow_file_symlinks, follow_dir_symlinks=follow_dir_symlinks, meta=meta) @@ -1620,6 +1661,17 @@ def _parse_chmod_code(code): op -- specified as '+' to add, '-' to remove, or '=' to assign. val -- specified as 'r' for read, 'w' for write, or 'x' for execute. + Notes: + The perm symbol X shall represent the execute/search portion of the + file mode bits if the file is a directory or if the current + (unmodified) file mode bits have at least one of the execute bits + (S_IXUSR, S_IXGRP, or S_IXOTH) set. It shall be ignored if the file is + not a directory and none of the execute bits are set in the current + file mode bits. [USE416877]_. + + References: + ..[USE416877] https://unix.stackexchange.com/questions/416877/what-is-a-capital-x-in-posix-chmod + Example: >>> from ubelt.util_path import _parse_chmod_code >>> print(list(_parse_chmod_code('ugo+rw,+r,g=rwx'))) @@ -1666,13 +1718,17 @@ def _resolve_chmod_code(old_mode, code): Returns: int : new code + References: + ..[RHEL_SpecialFilePerms] https://www.youtube.com/watch?v=Dn6b-mIKHmM&t=1970s + Example: + >>> # test normal user / group / other, read / write / execute perms >>> from ubelt.util_path import _resolve_chmod_code >>> print(oct(_resolve_chmod_code(0, '+rwx'))) >>> print(oct(_resolve_chmod_code(0, 'ugo+rwx'))) >>> print(oct(_resolve_chmod_code(0, 'a-rwx'))) >>> print(oct(_resolve_chmod_code(0, 'u+rw,go+r,go-wx'))) - >>> print(oct(_resolve_chmod_code(0o777, 'u+rw,go+r,go-wx'))) + >>> print(oct(_resolve_chmod_code(0o0777, 'u+rw,go+r,go-wx'))) 0o777 0o777 0o0 @@ -1683,14 +1739,17 @@ def _resolve_chmod_code(old_mode, code): >>> print(oct(_resolve_chmod_code(0, 'u=rw'))) >>> with pytest.raises(ValueError): >>> _resolve_chmod_code(0, 'u?w') + + Example: + >>> # Test special suid, sgid, and sticky (svtx) codes + >>> from ubelt.util_path import _resolve_chmod_code + >>> print(oct(_resolve_chmod_code(0, 'u+s'))) + >>> print(oct(_resolve_chmod_code(0o7777, 'u-s'))) + 0o4000 + 0o3777 """ - import stat import itertools as it action_lut = { - # TODO: handle suid, sgid, and sticky? - # suid = stat.S_ISUID - # sgid = stat.S_ISGID - # sticky = stat.S_ISVTX 'ur' : stat.S_IRUSR, 'uw' : stat.S_IWUSR, 'ux' : stat.S_IXUSR, @@ -1702,6 +1761,11 @@ def _resolve_chmod_code(old_mode, code): 'or' : stat.S_IROTH, 'ow' : stat.S_IWOTH, 'ox' : stat.S_IXOTH, + + # Special UNIX permissions + 'us': stat.S_ISUID, # SUID (executes run as the file's owner) + 'gs': stat.S_ISGID, # SGID (executes run as the file's group) + 'ot': stat.S_ISVTX, # sticky (only owner can delete) } actions = _parse_chmod_code(code) new_mode = int(old_mode) # (could optimize to modify inplace if needed) @@ -1734,37 +1798,85 @@ def _encode_chmod_int(int_code): Currently unused, but may be useful in the future. + Args: + int_code (int): mode from st_stat + concise (bool): if True, uses concise representations of special perms + + Returns: + str: the permissions code + Example: >>> from ubelt.util_path import _encode_chmod_int >>> int_code = 0o744 >>> print(_encode_chmod_int(int_code)) u=rwx,g=r,o=r - """ - import stat - action_lut = { - 'ur' : stat.S_IRUSR, - 'uw' : stat.S_IWUSR, - 'ux' : stat.S_IXUSR, - - 'gr' : stat.S_IRGRP, - 'gw' : stat.S_IWGRP, - 'gx' : stat.S_IXGRP, - 'or' : stat.S_IROTH, - 'ow' : stat.S_IWOTH, - 'ox' : stat.S_IXOTH, - } - from collections import defaultdict + >>> int_code = 0o7777 + >>> print(_encode_chmod_int(int_code)) + u=rwxs,g=rwxs,o=rwxt + """ + from collections import defaultdict, OrderedDict + action_lut = OrderedDict([ + ('ur' , stat.S_IRUSR), + ('uw' , stat.S_IWUSR), + ('ux' , stat.S_IXUSR), + + ('gr' , stat.S_IRGRP), + ('gw' , stat.S_IWGRP), + ('gx' , stat.S_IXGRP), + + ('or' , stat.S_IROTH), + ('ow' , stat.S_IWOTH), + ('ox' , stat.S_IXOTH), + + # Special UNIX permissions + ('us', stat.S_ISUID), # SUID (executes run as the file's owner) + ('gs', stat.S_ISGID), # SGID (executes run as the file's group) + ('ot', stat.S_ISVTX), # sticky (only owner can delete) + ]) target_to_perms = defaultdict(list) for key, val in action_lut.items(): target, perm = key if int_code & val: target_to_perms[target].append(perm) + + # The following commented logic might be useful if we want to created the + # "dashed" ls representation of permissions, but that is not needed for + # chmod itself, so it is not necessary to implement here. + # if concise: + # special_chars = {'u': 's', 'g': 's', 'o': 't'} + # for k, s in special_chars.items(): + # if k in target_to_perms: + # vs = target_to_perms[k] + # # if the executable bit is not set, replace the lowercase + # # with a capital S (or T for sticky) + # if 'x' in vs: + # if s in vs: + # vs.remove('x') + # elif s in vs: + # vs.remove(s) + # vs.append(s.upper()) parts = [k + '=' + ''.join(vs) for k, vs in target_to_perms.items()] code = ','.join(parts) return code +def _patch_win32_stats_on_pypy(): + """ + Handle [PyPyIssue4953]_ [PyPyDiscuss4952]_. + + References: + [PyPyIssue4953] https://github.com/pypy/pypy/issues/4953#event-12838738353 + [PyPyDiscuss4952] https://github.com/orgs/pypy/discussions/4952#discussioncomment-9481845 + """ + if not hasattr(stat, 'IO_REPARSE_TAG_MOUNT_POINT'): + os.supports_follow_symlinks.add("stat") + os.supports_follow_symlinks.add(os.stat) + stat.IO_REPARSE_TAG_APPEXECLINK = 0x8000001b # windows + stat.IO_REPARSE_TAG_MOUNT_POINT = 0xa0000003 # windows + stat.IO_REPARSE_TAG_SYMLINK = 0xa000000c # windows + + if sys.version_info[0:2] < (3, 8): # nocover # Vendor in a nearly modern copytree for Python 3.6 and 3.7