Skip to content

Commit

Permalink
feat: allows to provide returned keys for custom data loaders
Browse files Browse the repository at this point in the history
Users might want to return multiple DataFrames from a custom loading function,
even if that function only operates on singe file.
With the additional kwargs `return_keys`, the keys of the returned dictionary
containing those DataFrames can now be provided to allow correct tracing in such cases.
  • Loading branch information
MArpogaus committed Sep 13, 2024
1 parent ce7c485 commit 2aaf930
Showing 1 changed file with 15 additions and 3 deletions.
18 changes: 15 additions & 3 deletions src/dvc_stage/loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@
# author : Marcel Arpogaus <marcel dot arpogaus at gmail dot com>
#
# created : 2022-11-15 08:02:51 (Marcel Arpogaus)
# changed : 2023-02-16 12:57:26 (Marcel Arpogaus)
# changed : 2024-09-13 18:47:49 (Marcel Arpogaus)
# DESCRIPTION #################################################################
# ...
# LICENSE #####################################################################
# ...
###############################################################################
# REQUIRED MODULES ############################################################
"""loading module."""

import fnmatch
import logging
import os
Expand Down Expand Up @@ -77,7 +78,15 @@ def _get_data_key(path, key_map):


# PUBLIC FUNCTIONS ############################################################
def load_data(format, paths, key_map=None, import_from=None, quiet=False, **kwds):
def load_data(
format,
paths,
key_map=None,
import_from=None,
quiet=False,
return_keys: list = False,
**kwds,
):
"""
Load data from one or more files. Executes substage "loading".
Expand Down Expand Up @@ -125,7 +134,10 @@ def load_data(format, paths, key_map=None, import_from=None, quiet=False, **kwds
return data
else:
if format is None:
return None
if return_keys:
return dict.fromkeys(return_keys)
else:
return None
else:
__LOGGER__.debug(f"loading data from {paths}")
fn = _get_loading_function(format, import_from)
Expand Down

0 comments on commit 2aaf930

Please sign in to comment.