Merge branch 'dev-fuse-qkv' of github.com:DrownFish19/PaddleNLP into …

…dev-fuse-qkv

paddlenlp/peft/lora/lora_config.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import math
 import os
 from dataclasses import asdict, dataclass, field
 from typing import List, Optional, Union
@@ -94,6 +95,15 @@ def __post_init__(self):
             )
             self.use_quick_lora = False
 
+    @property
+    def scaling(self):
+        if not self.rslora and not self.pissa:
+            return self.lora_alpha / self.r
+        elif self.pissa:
+            return 1.0
+        else:
+            return self.lora_alpha / math.sqrt(self.r)
+
     @property
     def __dict__(self):
         return asdict(self)
@@ -114,6 +124,7 @@ def save_pretrained(self, save_directory):
         os.makedirs(save_directory, exist_ok=True)
 
         output_dict = self.__dict__
+        output_dict["scaling"] = self.scaling
         output_path = os.path.join(save_directory, LORA_CONFIG_NAME)
 
         # save it
@@ -136,6 +147,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
             raise ValueError(f"Can't find lora_config.json at '{pretrained_model_name_or_path}'")
 
         loaded_attributes = cls.from_json_file(config_file)
+        loaded_attributes.pop("scaling", None)
 
         config = cls(**kwargs)
 

paddlenlp/trainer/plugins/timer.py

@@ -32,15 +32,15 @@ def __init__(self, name):
     def start(self):
         """Start the timer."""
         assert not self.started_, f"{self.name} timer has already started"
-        if "gpu" in paddle.device.get_device():
+        if "cpu" not in paddle.device.get_device():
             paddle.device.synchronize()
         self.start_time = time.time()
         self.started_ = True
 
     def stop(self):
         """Stop the timers."""
         assert self.started_, f"{self.name} timer is not started."
-        if "gpu" in paddle.device.get_device():
+        if "cpu" not in paddle.device.get_device():
             paddle.device.synchronize()
         self.elapsed_ += time.time() - self.start_time
         self.started_ = False

scripts/distribute/run_ci.sh

@@ -50,6 +50,7 @@ install_paddlenlp(){
     python -m pip uninstall paddlenlp -y
     rm -rf build/ && rm -rf paddlenlp.egg-info/ && rm -rf dist/
     python -m pip install --ignore-installed -r requirements.txt
+    python -m pip install --ignore-installed -r requirements-dev.txt
     python setup.py install
     python setup.py build_ext
     python setup.py bdist_wheel

scripts/regression/run_ci.sh

@@ -55,6 +55,7 @@ nlp_build (){
     rm -rf dist/
 
     python -m pip install -r requirements.txt
+    python -m pip install -r requirements-dev.txt
     python setup.py bdist_wheel
     # python -m pip install --ignore-installed  dist/p****.whl
 }

tests/test_tipc/auto_tuner/llama_finetune/benchmark_common/prepare.sh

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
-pip install regex
+python -m pip install -r ../requirements-dev.txt
 
 cd ../llm/
 rm -rf data

tests/test_tipc/auto_tuner/llama_pretrain/benchmark_common/prepare.sh

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
+python -m pip install -r ../requirements-dev.txt
 
 # install fused_ln custom ops
 cd ../model_zoo/gpt-3/external_ops/

tests/test_tipc/dygraph/ft/benchmark_common/prepare.sh

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
-pip install regex
+python -m pip install -r ../requirements-dev.txt
 
 cd ../llm
 

tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh

@@ -26,7 +26,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
-python -m pip install regex
+python -m pip install -r ../requirements-dev.txt
 # get data
 cd ../llm/gpt-3
 rm -rf data

tests/test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
+python -m pip install -r ../requirements-dev.txt
 
 # install fused_ln custom ops
 cd ../model_zoo/gpt-3/external_ops/

tests/test_tipc/dygraph/hybrid_parallelism/llama2/benchmark_common/prepare.sh

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
+python -m pip install -r ../requirements-dev.txt
 
 # install fused_ln custom ops
 cd ../model_zoo/gpt-3/external_ops/

tests/test_tipc/dygraph/hybrid_parallelism/qwen/benchmark_common/prepare.sh

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 python -m pip install -r ../requirements.txt
+python -m pip install -r ../requirements-dev.txt
 
 python -m pip install tiktoken
 

tests/test_tipc/llm/prepare.sh

@@ -14,6 +14,7 @@
 
 cd ..
 sed -i -e "s/paddlepaddle/#paddlepaddle/g" requirements-dev.txt
+sed -i -e "s/pip install paddlepaddle/#pip install paddlepaddle/g" Makefile
 
 make install