Download重构

model_zoo/bert/run_pretrain_trainer.py

@@ -60,7 +60,7 @@ class ModelArguments:
         default=80, metadata={"help": "The maximum total of masked tokens in input sequence"}
     )
 
-    to_static: strtobool = field(default=False, metadata={"help": "Enable training under @to_static."})
+    # to_static: strtobool = field(default=False, metadata={"help": "Enable training under @to_static."})
     profiler_options: str = field(
         default=None,
         metadata={"help": "Whether to use FusedTransformerEncoderLayer to replace a TransformerEncoderLayer or not."},

paddlenlp/experimental/model_utils.py

@@ -24,10 +24,9 @@
 from paddle.framework import core
 
 from paddlenlp.transformers import PretrainedModel
+from paddlenlp.utils.download import get_file
 
 # TODO(fangzeyang) Temporary fix and replace by paddle framework downloader later
-from paddlenlp.utils.downloader import COMMUNITY_MODEL_PREFIX, get_path_from_url
-from paddlenlp.utils.env import MODEL_HOME
 from paddlenlp.utils.log import logger
 
 __all__ = ["FasterPretrainedModel", "ActScalesLoader", "WeightScalesLoader"]
@@ -96,6 +95,11 @@
         pretrained_models = list(cls.pretrained_init_configuration.keys())
         resource_files = {}
         init_configuration = {}
+        pretrained_model_name_or_path = str(pretrained_model_name_or_path)
+        cache_dir = kwargs.pop("cache_dir", None)
+        from_hf_hub = kwargs.pop("from_hf_hub", False)
+        from_aistudio = kwargs.pop("from_aistudio", False)
+        subfolder = kwargs.pop("subfolder", "")
 
         # From built-in pretrained models
         if pretrained_model_name_or_path in pretrained_models:
@@ -106,40 +110,27 @@
         elif os.path.isdir(pretrained_model_name_or_path):
             for file_id, file_name in cls.resource_files_names.items():
                 full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
-                resource_files[file_id] = full_file_name
+                if os.path.isfile(full_file_name):
+                    resource_files[file_id] = full_file_name
             resource_files["model_config_file"] = os.path.join(pretrained_model_name_or_path, cls.model_config_file)
         else:
-            # Assuming from community-contributed pretrained models
             for file_id, file_name in cls.resource_files_names.items():
-                full_file_name = "/".join([COMMUNITY_MODEL_PREFIX, pretrained_model_name_or_path, file_name])
-                resource_files[file_id] = full_file_name
-            resource_files["model_config_file"] = "/".join(
-                [COMMUNITY_MODEL_PREFIX, pretrained_model_name_or_path, cls.model_config_file]
-            )
+                resource_files[file_id] = file_name
 
-        default_root = os.path.join(MODEL_HOME, pretrained_model_name_or_path)
+        # default_root = os.path.join(MODEL_HOME, pretrained_model_name_or_path)
         resolved_resource_files = {}
         for file_id, file_path in resource_files.items():
             if file_path is None or os.path.isfile(file_path):
                 resolved_resource_files[file_id] = file_path
                 continue
-            path = os.path.join(default_root, file_path.split("/")[-1])
-            if os.path.exists(path):
-                logger.info("Already cached %s" % path)
-                resolved_resource_files[file_id] = path
-            else:
-                logger.info("Downloading %s and saved to %s" % (file_path, default_root))
-                try:
-                    resolved_resource_files[file_id] = get_path_from_url(file_path, default_root)
-                except RuntimeError as err:
-                    logger.error(err)
-                    raise RuntimeError(
-                        f"Can't load weights for '{pretrained_model_name_or_path}'.\n"
-                        f"Please make sure that '{pretrained_model_name_or_path}' is:\n"
-                        "- a correct model-identifier of built-in pretrained models,\n"
-                        "- or a correct model-identifier of community-contributed pretrained models,\n"
-                        "- or the correct path to a directory containing relevant modeling files(model_weights and model_config).\n"
-                    )
+            resolved_resource_files[file_id] = get_file(
+                pretrained_model_name_or_path,
+                [file_path],
+                subfolder,
+                cache_dir=cache_dir,
+                from_aistudio=from_aistudio,
+                from_hf_hub=from_hf_hub,
+            )
 
         # Prepare model initialization kwargs
         # Did we saved some inputs and kwargs to reload ?

paddlenlp/experimental/transformers/chatglm/modeling.py

@@ -581,12 +581,10 @@
         self.lm_head = self.model.get_input_embeddings()
 
     @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path, from_hf_hub: bool = False, subfolder: str | None = None, *args, **kwargs
-    ):
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
         # TODO: Support safetensors loading.
         kwargs["use_safetensors"] = False
-        return super().from_pretrained(pretrained_model_name_or_path, from_hf_hub, subfolder, *args, **kwargs)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
 
     @classmethod
     def get_cache_kvs_shape(

paddlenlp/experimental/transformers/gpt/modeling.py

@@ -444,12 +444,10 @@
         self.gpt = GPTInferenceModel(config)
 
     @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path, from_hf_hub: bool = False, subfolder: str | None = None, *args, **kwargs
-    ):
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
         # TODO: Support safetensors loading.
         kwargs["use_safetensors"] = False
-        return super().from_pretrained(pretrained_model_name_or_path, from_hf_hub, subfolder, *args, **kwargs)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
 
     @classmethod
     def get_cache_kvs_shape(

paddlenlp/experimental/transformers/llama/modeling.py

@@ -865,12 +865,10 @@
         self.lm_head = LlamaLMHead(config)
 
     @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path, from_hf_hub: bool = False, subfolder: str | None = None, *args, **kwargs
-    ):
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
         # TODO: Support safetensors loading.
         kwargs["use_safetensors"] = False
-        return super().from_pretrained(pretrained_model_name_or_path, from_hf_hub, subfolder, *args, **kwargs)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
 
     @classmethod
     def get_cache_kvs_shape(
@@ -1106,17 +1104,15 @@
         return mappings
 
     @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path, from_hf_hub: bool = False, subfolder: str | None = None, *args, **kwargs
-    ):
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
         # TODO: Support safetensors loading.
         kwargs["use_safetensors"] = False
         from paddlenlp.transformers.utils import (
             ContextManagers,
             is_safetensors_available,
-            resolve_cache_dir,
         )
 
+        from_hf_hub = kwargs.pop("from_hf_hub", False)
         config = kwargs.pop("config", None)
         from_aistudio = kwargs.get("from_aistudio", False)
         subfolder = kwargs.get("subfolder", None)
@@ -1125,7 +1121,7 @@
         convert_from_torch = kwargs.pop("convert_from_torch", None)
         cache_dir = kwargs.pop("cache_dir", None)
 
-        cache_dir = resolve_cache_dir(pretrained_model_name_or_path, from_hf_hub, cache_dir)
+        # cache_dir = resolve_cache_dir(pretrained_model_name_or_path, from_hf_hub, cache_dir)
 
         init_contexts = []
         with ContextManagers(init_contexts):

paddlenlp/experimental/transformers/opt/modeling.py

@@ -327,12 +327,10 @@
         self.lm_head = OPTLMHead(config)
 
     @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path, from_hf_hub: bool = False, subfolder: str | None = None, *args, **kwargs
-    ):
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
         # TODO: Support safetensors loading.
         kwargs["use_safetensors"] = kwargs.get("use_safetensors", False)
-        return super().from_pretrained(pretrained_model_name_or_path, from_hf_hub, subfolder, *args, **kwargs)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
 
     @classmethod
     def get_cache_kvs_shape(

paddlenlp/generation/configuration_utils.py

@@ -24,18 +24,11 @@
 
 from paddlenlp import __version__
 from paddlenlp.transformers.configuration_utils import PretrainedConfig
-from paddlenlp.transformers.utils import resolve_cache_dir
+from paddlenlp.utils.download import get_file
 from paddlenlp.utils.log import logger
 
-from ..transformers.aistudio_utils import aistudio_download
 from ..utils import GENERATION_CONFIG_NAME
-from ..utils.downloader import (
-    COMMUNITY_MODEL_PREFIX,
-    get_path_from_url_with_filelock,
-    hf_file_exists,
-    is_url,
-    url_file_exists,
-)
+from ..utils.downloader import hf_file_exists
 
 DEFAULT_MAX_NEW_TOKENS = 20
 
@@ -413,52 +406,17 @@ def from_pretrained(
         if subfolder is None:
             subfolder = ""
 
-        cache_dir = resolve_cache_dir(from_hf_hub, from_aistudio, cache_dir)
-
-        # 1. get the configuration file from local file, eg: /cache/path/model_config.json
-        if os.path.isfile(pretrained_model_name_or_path):
-            resolved_config_file = pretrained_model_name_or_path
-
-        # 2. get the configuration file from url, eg: https://ip/path/to/model_config.json
-        elif is_url(pretrained_model_name_or_path):
-            resolved_config_file = get_path_from_url_with_filelock(
-                pretrained_model_name_or_path,
-                cache_dir=os.path.join(cache_dir, pretrained_model_name_or_path, subfolder),
-                check_exist=not force_download,
-            )
-        # 3. get the configuration file from local dir with default name, eg: /local/path
-        elif os.path.isdir(pretrained_model_name_or_path):
-            configuration_file = os.path.join(pretrained_model_name_or_path, subfolder, config_file_name)
-            if os.path.exists(configuration_file):
-                resolved_config_file = configuration_file
-            else:
-                # try to detect old-school config file
-                raise FileNotFoundError("please make sure there is `generation_config.json` under the dir")
-        # 4. get the configuration file from aistudio
-        elif from_aistudio:
-            resolved_config_file = aistudio_download(
-                repo_id=pretrained_model_name_or_path,
-                filename=config_file_name,
-                cache_dir=cache_dir,
-                subfolder=subfolder,
-            )
-        # 5. get the configuration file from HF hub
-        elif from_hf_hub:
-            resolved_config_file = resolve_hf_generation_config_path(
-                repo_id=pretrained_model_name_or_path, cache_dir=cache_dir, subfolder=subfolder
-            )
-        else:
-            url_list = [COMMUNITY_MODEL_PREFIX, pretrained_model_name_or_path, config_file_name]
-            cache_dir = os.path.join(cache_dir, pretrained_model_name_or_path, subfolder)
-            if subfolder != "":
-                url_list.insert(2, subfolder)
-            community_url = "/".join(url_list)
-            if url_file_exists(community_url):
-                resolved_config_file = get_path_from_url_with_filelock(
-                    community_url, cache_dir, check_exist=not force_download
-                )
-            else:
-                raise FileNotFoundError(f"configuration file<{GENERATION_CONFIG_NAME}> not found")
+        # cache_dir = resolve_cache_dir(from_hf_hub, from_aistudio, cache_dir)
+
+        resolved_config_file = get_file(
+            pretrained_model_name_or_path,
+            [config_file_name],
+            subfolder,
+            cache_dir=cache_dir,
+            force_download=force_download,
+            from_aistudio=from_aistudio,
+            from_hf_hub=from_hf_hub,
+        )
 
         try:
             logger.info(f"Loading configuration file {resolved_config_file}")

paddlenlp/transformers/auto/configuration.py

@@ -20,20 +20,11 @@
 from collections import defaultdict
 from typing import Dict, List, Type
 
-from huggingface_hub import hf_hub_download
-
-from ... import __version__
-from ...utils.downloader import (
-    COMMUNITY_MODEL_PREFIX,
-    get_path_from_url_with_filelock,
-    url_file_exists,
-)
+from ...utils.download import get_file
 from ...utils.import_utils import import_module
 from ...utils.log import logger
-from ..aistudio_utils import aistudio_download
 from ..configuration_utils import PretrainedConfig
 from ..model_utils import PretrainedModel
-from ..utils import resolve_cache_dir
 
 __all__ = [
     "AutoConfig",
@@ -170,13 +161,8 @@
             config = AutoConfig.from_pretrained("bert-base-uncased")
             config.save_pretrained('./bert-base-uncased')
         """
-        subfolder = kwargs.get("subfolder", "")
-        if subfolder is None:
-            subfolder = ""
-        from_aistudio = kwargs.pop("from_aistudio", False)
-        from_hf_hub = kwargs.pop("from_hf_hub", False)
-        cache_dir = kwargs.pop("cache_dir", None)
-        cache_dir = resolve_cache_dir(from_hf_hub=from_hf_hub, from_aistudio=from_aistudio, cache_dir=cache_dir)
+
+        # cache_dir = resolve_cache_dir(from_hf_hub=from_hf_hub, from_aistudio=from_aistudio, cache_dir=cache_dir)
 
         if not cls.name2class:
             cls.name2class = {}
@@ -192,72 +178,33 @@
                 pretrained_model_name_or_path, *model_args, **kwargs
             )
 
-        # From local dir path
-        elif os.path.isdir(pretrained_model_name_or_path):
-            config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.config_file)
-            if not os.path.exists(config_file):
-                # try to load legacy config file
-                legacy_config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.legacy_config_file)
-                if not os.path.exists(legacy_config_file):
-                    raise ValueError(
-                        f"config file<{cls.config_file}> or legacy config file<{cls.legacy_config_file}> not found"
-                    )
+        subfolder = kwargs.get("subfolder", "")
+        if subfolder is None:
+            subfolder = ""
+        from_aistudio = kwargs.pop("from_aistudio", False)
+        from_hf_hub = kwargs.pop("from_hf_hub", False)
+        cache_dir = kwargs.pop("cache_dir", None)
 
-                logger.warning(f"loading legacy config file<{cls.legacy_config_file}> ...")
-                config_file = legacy_config_file
+        config_file = get_file(
+            pretrained_model_name_or_path,
+            [cls.config_file, cls.legacy_config_file],
+            subfolder,
+            cache_dir=cache_dir,
+            from_hf_hub=from_hf_hub,
+            from_aistudio=from_aistudio,
+        )
 
+        if os.path.exists(config_file):
             config_class = cls._get_config_class_from_config(pretrained_model_name_or_path, config_file)
             logger.info("We are using %s to load '%s'." % (config_class, pretrained_model_name_or_path))
             if config_class is cls:
                 return cls.from_file(config_file)
-            return config_class.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif from_aistudio:
-            file = aistudio_download(
-                repo_id=pretrained_model_name_or_path,
-                filename=cls.config_file,
-                subfolder=subfolder,
-                cache_dir=cache_dir,
-            )
-            return cls.from_pretrained(os.path.dirname(file))
-        elif from_hf_hub:
-            file = hf_hub_download(
-                repo_id=pretrained_model_name_or_path,
-                filename=cls.config_file,
-                cache_dir=cache_dir,
-                subfolder=subfolder,
-                library_name="PaddleNLP",
-                library_version=__version__,
-            )
-            # from local dir path
-            return cls.from_pretrained(os.path.dirname(file))
-
-        # Assuming from community-contributed pretrained models
+            return config_class.from_pretrained(config_file, *model_args, **kwargs)
         else:
-            url_list = [COMMUNITY_MODEL_PREFIX, pretrained_model_name_or_path, cls.config_file]
-            legacy_url_list = [COMMUNITY_MODEL_PREFIX, pretrained_model_name_or_path, cls.legacy_config_file]
-            cache_dir = os.path.join(cache_dir, pretrained_model_name_or_path, subfolder)
-            if subfolder != "":
-                url_list.insert(2, subfolder)
-                legacy_url_list.insert(2, subfolder)
-            community_config_path = "/".join(url_list)
-            legacy_community_config_path = "/".join(legacy_url_list)
-
-            if not url_file_exists(community_config_path):
-                if not url_file_exists(legacy_community_config_path):
-                    raise RuntimeError(
-                        f"Can't load Config for '{pretrained_model_name_or_path}'.\n"
-                        f"Please make sure that '{pretrained_model_name_or_path}' is:\n"
-                        "- a correct model-identifier of built-in pretrained models,\n"
-                        "- or a correct model-identifier of community-contributed pretrained models,\n"
-                        "- or the correct path to a directory containing relevant config files.\n"
-                    )
-                logger.warning(f"loading legacy config file<{cls.legacy_config_file}> ...")
-                community_config_path = legacy_community_config_path
-
-            resolved_config_file = get_path_from_url_with_filelock(community_config_path, cache_dir)
-            config_class = cls._get_config_class_from_config(pretrained_model_name_or_path, resolved_config_file)
-            logger.info("We are using %s to load '%s'." % (config_class, pretrained_model_name_or_path))
-            if config_class is cls:
-                return cls.from_file(resolved_config_file, **kwargs)
-
-            return config_class.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            raise RuntimeError(
+                f"Can't load config for '{pretrained_model_name_or_path}'.\n"
+                f"Please make sure that '{pretrained_model_name_or_path}' is:\n"
+                "- a correct model-identifier of built-in pretrained models,\n"
+                "- or a correct model-identifier of community-contributed pretrained models,\n"
+                "- or the correct path to a directory containing relevant config files.\n"
+            )