[Benchmark] Enable use_fast_layer_norm for llama2 benchmark (#8714)

PaddlePaddle · Jul 4, 2024 · 70564ba · 70564ba
1 parent fd01043
commit 70564ba
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 0 deletions.
diff --git a/...raph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json b/...raph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 0,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,

diff --git a/...raph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json b/...raph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 1,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,

diff --git a/...ygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json b/...ygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 0,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,