Protect against None components of universal pathlib xcom backend #41921
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
1 task
potiuk
force-pushed
the
fix-none-values-on-basepath
branch
from
acc85c8
to
bb9a9a2
Compare
potiuk
commented
Sep 1, 2024
jscheffl
approved these changes
Sep 1, 2024
|
Regarding the tldr: the default and most custom implementations throw away extra keyword arguments. The only one in this list that doesn't is See below for details on the different rename implementations: fsspec subclasses: customized method report for 'mv'Click to see the environment used to generate the reportDefault implementation def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
"""Move file(s) from one location to another"""
if path1 == path2:
logger.debug("%s mv: The paths are the same, so no files were moved.", self)
else:
# explicitly raise exception to prevent data corruption
self.copy(
path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise"
)
self.rm(path1, recursive=recursive)These filesystem classes do not customize the method 'mv':
Subclasses customizing 'mv'HadoopFileSystem
@wrap_exceptions
def mv(self, path1, path2, **kwargs):
path1 = self._strip_protocol(path1).rstrip("/")
path2 = self._strip_protocol(path2).rstrip("/")
self.fs.move(path1, path2)AsyncLocalFileSystem
def mv(self, path1, path2, **kwargs):
path1 = self._strip_protocol(path1)
path2 = self._strip_protocol(path2)
shutil.move(path1, path2)DaskWorkerFileSystem
def mv(self, *args, **kwargs):
if self.worker:
self.fs.mv(*args, **kwargs)
else:
self.rfs.mv(*args, **kwargs).compute()DatabricksFileSystem
def mv(
self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs
):
"""
Move a source to a destination path.
A note from the original [databricks API manual]
(https://docs.databricks.com/dev-tools/api/latest/dbfs.html#move).
When moving a large number of files the API call will time out after
approximately 60s, potentially resulting in partially moved data.
Therefore, for operations that move more than 10k files, we strongly
discourage using the DBFS REST API.
Parameters
----------
source_path: str
From where to move (absolute path)
destination_path: str
To where to move (absolute path)
recursive: bool
Not implemented to far.
maxdepth:
Not implemented to far.
"""
if recursive:
raise NotImplementedError
if maxdepth:
raise NotImplementedError
try:
self._send_to_api(
method="post",
endpoint="move",
json={"source_path": source_path, "destination_path": destination_path},
)
except DatabricksException as e:
if e.error_code == "RESOURCE_DOES_NOT_EXIST":
raise FileNotFoundError(e.message)
elif e.error_code == "RESOURCE_ALREADY_EXISTS":
raise FileExistsError(e.message)
raise e
self.invalidate_cache(self._parent(source_path))
self.invalidate_cache(self._parent(destination_path))DirFileSystem
def mv(self, path1, path2, **kwargs):
return self.fs.mv(
self._join(path1),
self._join(path2),
**kwargs,
)LocalFileSystem
def mv(self, path1, path2, **kwargs):
path1 = self._strip_protocol(path1)
path2 = self._strip_protocol(path2)
shutil.move(path1, path2)FTPFileSystem
def mv(self, path1, path2, **kwargs):
path1 = self._strip_protocol(path1)
path2 = self._strip_protocol(path2)
self.ftp.rename(path1, path2)
self.invalidate_cache(self._parent(path1))
self.invalidate_cache(self._parent(path2))SFTPFileSystem
def mv(self, old, new):
logger.debug("Renaming %s into %s", old, new)
self.ftp.posix_rename(old, new)SMBFileSystem
def mv(self, path1, path2, recursive=None, maxdepth=None, **kwargs):
wpath1 = _as_unc_path(self.host, path1)
wpath2 = _as_unc_path(self.host, path2)
smbclient.rename(wpath1, wpath2, port=self._port, **kwargs)WebdavFileSystem
def mv(
self,
path1: str,
path2: str,
recursive: bool = False,
maxdepth: Optional[bool] = None,
**kwargs: Any,
) -> None:
"""Move a file/directory from one path to the other."""
path1 = self._strip_protocol(path1)
path2 = self._strip_protocol(path2)
if recursive and not maxdepth and self.isdir(path1):
return self.client.move(path1, path2)
if not recursive and self.isdir(path1):
return self.makedirs(path2)
super().mv(path1, path2, recursive=recursive, maxdepth=maxdepth, **kwargs)
return NoneWebHDFS
def mv(self, path1, path2, **kwargs):
self._call("RENAME", method="put", path=path1, destination=path2) |
|
Thanks @ap-- ! |
potiuk
added a commit
to potiuk/airflow
that referenced
this pull request
Sep 1, 2024
…ache#41921) fixes: apache#41723 (cherry picked from commit 7a75f0a)
This was referenced Sep 1, 2024
potiuk
added a commit
that referenced
this pull request
Sep 2, 2024
utkarsharma2
added
changelog:skip
utkarsharma2
pushed a commit
that referenced
this pull request
Sep 2, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
fixes: #41723
^ Add meaningful description above
Read the Pull Request Guidelines for more information.
In case of fundamental code changes, an Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in a newsfragment file, named
{pr_number}.significant.rstor{issue_number}.significant.rst, in newsfragments.