Skip to content

Commit

Permalink
[LLM benchmark]add qwen2.5-7b (#9524)
Browse files Browse the repository at this point in the history
* add qwen2.5-7b

* add qwen2.5-1.5b/14b

* add fix

* update data

* fix

* fix

* fix

* Update run_ci.sh

* update ips for dpo

* add llama2
  • Loading branch information
Liujie0926 authored Dec 13, 2024
1 parent 4102eb0 commit d3f3bbf
Show file tree
Hide file tree
Showing 34 changed files with 1,416 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-13b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=4 "
param+="sharding_parallel_degree=2 "
param+="sharding=stage2 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=4 "
param+="run_stage=dpo "
param+="run_mode=tp4_sd2_acc4_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-13b_dpo "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-13b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=1 "
param+="sharding_parallel_degree=8 "
param+="sharding=stage1 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=1 "
param+="run_stage=lora "
param+="run_mode=tp1_sd8_acc1_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-13b_lora "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-13b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=2 "
param+="sharding_parallel_degree=4 "
param+="sharding=stage2 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=2 "
param+="run_stage=sft "
param+="run_mode=tp2_sd4_acc2_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-13b_sft "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-7b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=4 "
param+="sharding_parallel_degree=2 "
param+="sharding=stage2 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=1 "
param+="run_stage=dpo "
param+="run_mode=tp4_sd2_acc1_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-7b_dpo "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-7b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=1 "
param+="pipeline_parallel_degree=1 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=1 "
param+="run_stage=lora "
param+="run_mode=tp1_pp1_acc1_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-7b_lora "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


param="model_name_or_path=meta-llama/Llama-2-7b "
param+="per_device_train_batch_size=1 "
param+="tensor_parallel_degree=2 "
param+="sharding_parallel_degree=4 "
param+="sharding=stage2 "
param+="recompute=true "
param+="recompute_granularity=full "
param+="gradient_accumulation_steps=1 "
param+="run_stage=sft "
param+="run_mode=tp2_sd4_acc1_dygraph "
param+="device_num=N1C8 "
param+="global_batch_size=16 "
param+="model_item=llama2-7b_sft "
param+="max_steps=150 "

cd ./tests
bash ./test_tipc/llm/llama2/benchmark_common/prepare.sh

bash -c "${param} bash ./test_tipc/llm/llama2/benchmark_common/run_benchmark.sh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"model_name_or_path": "meta-llama/Llama-2-13b",
"train_dataset_path": "./data/dpo_benchmark_train/train.json",
"dev_dataset_path": "./data/dpo_benchmark_train/dev.json",
"output_dir": "./checkpoints/dpo_ckpts",
"per_device_train_batch_size": 1,
"gradient_accumulation_steps": 4,
"per_device_eval_batch_size": 1,
"num_train_epochs": 1,
"learning_rate": 1e-06,
"warmup_steps": 10,
"logging_steps": 1,
"evaluation_strategy": "no",
"save_strategy": "no",
"max_seq_len": 8192,
"max_prompt_len": 4096,
"bf16": true,
"fp16_opt_level": "O2",
"do_train": true,
"do_eval": false,
"disable_tqdm": true,
"load_best_model_at_end": true,
"tensor_parallel_degree": 4,
"sharding_parallel_degree": 2,
"sharding": "stage2",
"use_flash_attention": true,
"flash_mask": true,
"recompute": true,
"recompute_granularity": "full",
"benchmark": true,
"unified_checkpoint": true,
"autotuner_benchmark":false,
"beta": 0.1,
"loss_type": "sigmoid",
"label_smoothing": 0.0
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"model_name_or_path": "meta-llama/Llama-2-13b",
"dataset_name_or_path": "./data/sft_benchmark_train/",
"output_dir": "./checkpoints/lora_ckpts",
"per_device_train_batch_size": 1,
"gradient_accumulation_steps": 1,
"per_device_eval_batch_size": 8,
"eval_accumulation_steps":16,
"num_train_epochs": 1,
"learning_rate": 3e-04,
"warmup_steps": 30,
"logging_steps": 1,
"evaluation_strategy": "no",
"save_strategy": "no",
"src_length": 4096,
"max_length": 8192,
"bf16": true,
"fp16_opt_level": "O2",
"do_train": true,
"do_eval": false,
"disable_tqdm": true,
"load_best_model_at_end": true,
"eval_with_do_generation": false,
"metric_for_best_model": "accuracy",
"recompute": true,
"recompute_granularity": "full",
"save_total_limit": 1,
"tensor_parallel_degree": 1,
"sharding_parallel_degree": 8,
"sharding": "stage1",
"pipeline_parallel_degree": 1,
"lora": true,
"unified_checkpoint": true,
"benchmark": true,
"zero_padding": true,
"flash_mask": true,
"use_flash_attention": true,
"pissa": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"model_name_or_path": "meta-llama/Llama-2-13b",
"dataset_name_or_path": "./data/sft_benchmark_train/",
"output_dir": "./checkpoints/sft_ckpts",
"per_device_train_batch_size": 1,
"gradient_accumulation_steps": 2,
"per_device_eval_batch_size": 8,
"eval_accumulation_steps":16,
"num_train_epochs": 1,
"learning_rate": 3e-05,
"warmup_steps": 30,
"logging_steps": 1,
"evaluation_strategy": "no",
"save_strategy": "no",
"src_length": 4096,
"max_length": 8192,
"bf16": true,
"fp16_opt_level": "O2",
"do_train": true,
"do_eval": false,
"disable_tqdm": true,
"load_best_model_at_end": true,
"eval_with_do_generation": false,
"metric_for_best_model": "accuracy",
"recompute": true,
"recompute_granularity": "full",
"save_total_limit": 1,
"benchmark": true,
"tensor_parallel_degree": 2,
"sharding_parallel_degree": 4,
"pipeline_parallel_degree": 1,
"sharding": "stage2",
"zero_padding": true,
"flash_mask": true,
"unified_checkpoint": true,
"use_flash_attention": true
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"model_name_or_path": "meta-llama/Llama-2-7b",
"train_dataset_path": "./data/dpo_benchmark_train/train.json",
"dev_dataset_path": "./data/dpo_benchmark_train/dev.json",
"output_dir": "./checkpoints/dpo_ckpts",
"per_device_train_batch_size": 1,
"gradient_accumulation_steps": 1,
"per_device_eval_batch_size": 1,
"num_train_epochs": 1,
"learning_rate": 1e-06,
"warmup_steps": 10,
"logging_steps": 1,
"evaluation_strategy": "no",
"save_strategy": "no",
"max_seq_len": 8192,
"max_prompt_len": 4096,
"bf16": true,
"fp16_opt_level": "O2",
"do_train": true,
"do_eval": false,
"disable_tqdm": true,
"load_best_model_at_end": true,
"tensor_parallel_degree": 4,
"sharding_parallel_degree": 2,
"sharding": "stage2",
"use_flash_attention": true,
"flash_mask": true,
"recompute": true,
"recompute_granularity": "full",
"benchmark": true,
"unified_checkpoint": true,
"autotuner_benchmark":false,
"beta": 0.1,
"loss_type": "sigmoid",
"label_smoothing": 0.0
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"model_name_or_path": "meta-llama/Llama-2-7b",
"dataset_name_or_path": "./data/sft_benchmark_train/",
"output_dir": "./checkpoints/lora_ckpts",
"per_device_train_batch_size": 1,
"gradient_accumulation_steps": 1,
"per_device_eval_batch_size": 8,
"eval_accumulation_steps":16,
"num_train_epochs": 1,
"learning_rate": 3e-04,
"warmup_steps": 30,
"logging_steps": 1,
"evaluation_strategy": "no",
"save_strategy": "no",
"src_length": 4096,
"max_length": 8192,
"bf16": true,
"fp16_opt_level": "O2",
"do_train": true,
"do_eval": false,
"disable_tqdm": true,
"load_best_model_at_end": true,
"eval_with_do_generation": false,
"metric_for_best_model": "accuracy",
"recompute": true,
"recompute_granularity": "full",
"save_total_limit": 1,
"tensor_parallel_degree": 1,
"pipeline_parallel_degree": 1,
"lora": true,
"unified_checkpoint": true,
"benchmark": true,
"zero_padding": true,
"flash_mask": true,
"use_flash_attention": true,
"pissa": false
}
Loading

0 comments on commit d3f3bbf

Please sign in to comment.